diff options
Diffstat (limited to 'mlir/lib/Conversion')
7 files changed, 75 insertions, 79 deletions
diff --git a/mlir/lib/Conversion/GPUToCUDA/ConvertKernelFuncToCubin.cpp b/mlir/lib/Conversion/GPUToCUDA/ConvertKernelFuncToCubin.cpp index 66a2e66f99a..b111c96313c 100644 --- a/mlir/lib/Conversion/GPUToCUDA/ConvertKernelFuncToCubin.cpp +++ b/mlir/lib/Conversion/GPUToCUDA/ConvertKernelFuncToCubin.cpp @@ -46,18 +46,15 @@ static constexpr const char *kCubinAnnotation = "nvvm.cubin"; /// IR and further to PTX. A user provided CubinGenerator compiles the PTX to /// GPU binary code, which is then attached as an attribute to the function. The /// function body is erased. -class GpuKernelToCubinPass : public ModulePass<GpuKernelToCubinPass> { +class GpuKernelToCubinPass + : public OperationPass<GpuKernelToCubinPass, gpu::GPUModuleOp> { public: GpuKernelToCubinPass( CubinGenerator cubinGenerator = compilePtxToCubinForTesting) : cubinGenerator(cubinGenerator) {} - void runOnModule() override { - ModuleOp module = getModule(); - if (!module.getAttrOfType<UnitAttr>( - gpu::GPUDialect::getKernelModuleAttrName()) || - !module.getName()) - return; + void runOnOperation() override { + gpu::GPUModuleOp module = getOperation(); // Make sure the NVPTX target is initialized. LLVMInitializeNVPTXTarget(); @@ -71,8 +68,8 @@ public: // Translate the module to CUBIN and attach the result as attribute to the // module. - if (auto cubinAttr = translateGpuModuleToCubinAnnotation( - *llvmModule, module.getLoc(), *module.getName())) + if (auto cubinAttr = translateGPUModuleToCubinAnnotation( + *llvmModule, module.getLoc(), module.getName())) module.setAttr(kCubinAnnotation, cubinAttr); else signalPassFailure(); @@ -92,7 +89,7 @@ private: StringRef name); /// Translates llvmModule to cubin and returns the result as attribute. - StringAttr translateGpuModuleToCubinAnnotation(llvm::Module &llvmModule, + StringAttr translateGPUModuleToCubinAnnotation(llvm::Module &llvmModule, Location loc, StringRef name); CubinGenerator cubinGenerator; @@ -149,7 +146,7 @@ OwnedCubin GpuKernelToCubinPass::convertModuleToCubin(llvm::Module &llvmModule, return cubinGenerator(ptx, loc, name); } -StringAttr GpuKernelToCubinPass::translateGpuModuleToCubinAnnotation( +StringAttr GpuKernelToCubinPass::translateGPUModuleToCubinAnnotation( llvm::Module &llvmModule, Location loc, StringRef name) { auto cubin = convertModuleToCubin(llvmModule, loc, name); if (!cubin) @@ -157,7 +154,7 @@ StringAttr GpuKernelToCubinPass::translateGpuModuleToCubinAnnotation( return StringAttr::get({cubin->data(), cubin->size()}, loc->getContext()); } -std::unique_ptr<OpPassBase<ModuleOp>> +std::unique_ptr<OpPassBase<gpu::GPUModuleOp>> mlir::createConvertGPUKernelToCubinPass(CubinGenerator cubinGenerator) { return std::make_unique<GpuKernelToCubinPass>(cubinGenerator); } diff --git a/mlir/lib/Conversion/GPUToCUDA/ConvertLaunchFuncToCudaCalls.cpp b/mlir/lib/Conversion/GPUToCUDA/ConvertLaunchFuncToCudaCalls.cpp index 41f69d6e21d..31024d2881b 100644 --- a/mlir/lib/Conversion/GPUToCUDA/ConvertLaunchFuncToCudaCalls.cpp +++ b/mlir/lib/Conversion/GPUToCUDA/ConvertLaunchFuncToCudaCalls.cpp @@ -132,9 +132,9 @@ public: // GPU kernel modules are no longer necessary since we have a global // constant with the CUBIN data. - for (auto m : llvm::make_early_inc_range(getModule().getOps<ModuleOp>())) - if (m.getAttrOfType<UnitAttr>(gpu::GPUDialect::getKernelModuleAttrName())) - m.erase(); + for (auto m : + llvm::make_early_inc_range(getModule().getOps<gpu::GPUModuleOp>())) + m.erase(); } private: @@ -343,8 +343,8 @@ void GpuLaunchFuncToCudaCallsPass::translateGpuLaunchCalls( builder.getI32IntegerAttr(0)); // Create an LLVM global with CUBIN extracted from the kernel annotation and // obtain a pointer to the first byte in it. - auto kernelModule = - getModule().lookupSymbol<ModuleOp>(launchOp.getKernelModuleName()); + auto kernelModule = getModule().lookupSymbol<gpu::GPUModuleOp>( + launchOp.getKernelModuleName()); assert(kernelModule && "expected a kernel module"); auto cubinAttr = kernelModule.getAttrOfType<StringAttr>(kCubinAnnotation); @@ -354,8 +354,7 @@ void GpuLaunchFuncToCudaCallsPass::translateGpuLaunchCalls( return signalPassFailure(); } - assert(kernelModule.getName() && "expected a named module"); - SmallString<128> nameBuffer(*kernelModule.getName()); + SmallString<128> nameBuffer(kernelModule.getName()); nameBuffer.append(kCubinStorageSuffix); Value data = LLVM::createGlobalString( loc, builder, nameBuffer.str(), cubinAttr.getValue(), diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp index e2b1e0e533c..84bc7ff1d5f 100644 --- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp +++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp @@ -200,7 +200,7 @@ private: auto type = operand.getType().cast<LLVM::LLVMType>(); // Create shared memory array to store the warp reduction. - auto module = operand.getDefiningOp()->getParentOfType<ModuleOp>(); + auto module = operand.getDefiningOp()->getParentOfType<gpu::GPUModuleOp>(); assert(module && "op must belong to a module"); Value sharedMemPtr = createSharedMemoryArray(loc, module, type, kWarpSize, rewriter); @@ -391,10 +391,10 @@ private: } /// Creates a global array stored in shared memory. - Value createSharedMemoryArray(Location loc, ModuleOp module, + Value createSharedMemoryArray(Location loc, gpu::GPUModuleOp module, LLVM::LLVMType elementType, int numElements, ConversionPatternRewriter &rewriter) const { - OpBuilder builder(module.getBodyRegion()); + OpBuilder builder(module.body()); auto arrayType = LLVM::LLVMType::getArrayTy(elementType, numElements); StringRef name = "reduce_buffer"; @@ -699,13 +699,11 @@ struct GPUReturnOpLowering : public LLVMOpLowering { /// /// This pass only handles device code and is not meant to be run on GPU host /// code. -class LowerGpuOpsToNVVMOpsPass : public ModulePass<LowerGpuOpsToNVVMOpsPass> { +class LowerGpuOpsToNVVMOpsPass + : public OperationPass<LowerGpuOpsToNVVMOpsPass, gpu::GPUModuleOp> { public: - void runOnModule() override { - ModuleOp m = getModule(); - if (!m.getAttrOfType<UnitAttr>(gpu::GPUDialect::getKernelModuleAttrName())) - return; - + void runOnOperation() override { + gpu::GPUModuleOp m = getOperation(); OwningRewritePatternList patterns; NVVMTypeConverter converter(m.getContext()); populateStdToLLVMConversionPatterns(converter, patterns); @@ -718,7 +716,7 @@ public: target.addLegalDialect<LLVM::LLVMDialect>(); target.addLegalDialect<NVVM::NVVMDialect>(); // TODO(csigg): Remove once we support replacing non-root ops. - target.addLegalOp<gpu::YieldOp>(); + target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>(); if (failed(applyPartialConversion(m, target, patterns, &converter))) signalPassFailure(); } @@ -750,7 +748,8 @@ void mlir::populateGpuToNVVMConversionPatterns( "__nv_exp"); } -std::unique_ptr<OpPassBase<ModuleOp>> mlir::createLowerGpuOpsToNVVMOpsPass() { +std::unique_ptr<OpPassBase<gpu::GPUModuleOp>> +mlir::createLowerGpuOpsToNVVMOpsPass() { return std::make_unique<LowerGpuOpsToNVVMOpsPass>(); } diff --git a/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt b/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt index be82894461d..adeb4e099ab 100644 --- a/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt +++ b/mlir/lib/Conversion/GPUToSPIRV/CMakeLists.txt @@ -1,8 +1,15 @@ +set(LLVM_TARGET_DEFINITIONS GPUToSPIRV.td) +mlir_tablegen(GPUToSPIRV.cpp.inc -gen-rewriters) +add_public_tablegen_target(MLIRGPUToSPIRVIncGen) + add_llvm_library(MLIRGPUtoSPIRVTransforms ConvertGPUToSPIRV.cpp ConvertGPUToSPIRVPass.cpp ) +add_dependencies(MLIRGPUtoSPIRVTransforms + MLIRGPUToSPIRVIncGen) + target_link_libraries(MLIRGPUtoSPIRVTransforms MLIRGPU MLIRIR diff --git a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp index 2fd8cedfd63..a90cea99be4 100644 --- a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp +++ b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp @@ -63,27 +63,13 @@ private: SmallVector<int32_t, 3> workGroupSizeAsInt32; }; -/// Pattern to convert a module with gpu.kernel_module attribute to a -/// spv.module. -class KernelModuleConversion final : public SPIRVOpLowering<ModuleOp> { +/// Pattern to convert a gpu.module to a spv.module. +class GPUModuleConversion final : public SPIRVOpLowering<gpu::GPUModuleOp> { public: - using SPIRVOpLowering<ModuleOp>::SPIRVOpLowering; + using SPIRVOpLowering<gpu::GPUModuleOp>::SPIRVOpLowering; PatternMatchResult - matchAndRewrite(ModuleOp moduleOp, ArrayRef<Value> operands, - ConversionPatternRewriter &rewriter) const override; -}; - -/// Pattern to convert a module terminator op to a terminator of spv.module op. -// TODO: Move this into DRR, but that requires ModuleTerminatorOp to be defined -// in ODS. -class KernelModuleTerminatorConversion final - : public SPIRVOpLowering<ModuleTerminatorOp> { -public: - using SPIRVOpLowering<ModuleTerminatorOp>::SPIRVOpLowering; - - PatternMatchResult - matchAndRewrite(ModuleTerminatorOp terminatorOp, ArrayRef<Value> operands, + matchAndRewrite(gpu::GPUModuleOp moduleOp, ArrayRef<Value> operands, ConversionPatternRewriter &rewriter) const override; }; @@ -284,16 +270,12 @@ KernelFnConversion::matchAndRewrite(gpu::GPUFuncOp funcOp, } //===----------------------------------------------------------------------===// -// ModuleOp with gpu.kernel_module. +// ModuleOp with gpu.module. //===----------------------------------------------------------------------===// -PatternMatchResult KernelModuleConversion::matchAndRewrite( - ModuleOp moduleOp, ArrayRef<Value> operands, +PatternMatchResult GPUModuleConversion::matchAndRewrite( + gpu::GPUModuleOp moduleOp, ArrayRef<Value> operands, ConversionPatternRewriter &rewriter) const { - if (!moduleOp.getAttrOfType<UnitAttr>( - gpu::GPUDialect::getKernelModuleAttrName())) { - return matchFailure(); - } // TODO : Generalize this to account for different extensions, // capabilities, extended_instruction_sets, other addressing models // and memory models. @@ -302,8 +284,8 @@ PatternMatchResult KernelModuleConversion::matchAndRewrite( spirv::MemoryModel::GLSL450, spirv::Capability::Shader, spirv::Extension::SPV_KHR_storage_buffer_storage_class); // Move the region from the module op into the SPIR-V module. - Region &spvModuleRegion = spvModule.getOperation()->getRegion(0); - rewriter.inlineRegionBefore(moduleOp.getBodyRegion(), spvModuleRegion, + Region &spvModuleRegion = spvModule.body(); + rewriter.inlineRegionBefore(moduleOp.body(), spvModuleRegion, spvModuleRegion.begin()); // The spv.module build method adds a block with a terminator. Remove that // block. The terminator of the module op in the remaining block will be @@ -314,17 +296,6 @@ PatternMatchResult KernelModuleConversion::matchAndRewrite( } //===----------------------------------------------------------------------===// -// ModuleTerminatorOp for gpu.kernel_module. -//===----------------------------------------------------------------------===// - -PatternMatchResult KernelModuleTerminatorConversion::matchAndRewrite( - ModuleTerminatorOp terminatorOp, ArrayRef<Value> operands, - ConversionPatternRewriter &rewriter) const { - rewriter.replaceOpWithNewOp<spirv::ModuleEndOp>(terminatorOp); - return matchSuccess(); -} - -//===----------------------------------------------------------------------===// // GPU return inside kernel functions to SPIR-V return. //===----------------------------------------------------------------------===// @@ -342,14 +313,18 @@ PatternMatchResult GPUReturnOpConversion::matchAndRewrite( // GPU To SPIRV Patterns. //===----------------------------------------------------------------------===// +namespace { +#include "GPUToSPIRV.cpp.inc" +} + void mlir::populateGPUToSPIRVPatterns(MLIRContext *context, SPIRVTypeConverter &typeConverter, OwningRewritePatternList &patterns, ArrayRef<int64_t> workGroupSize) { + populateWithGenerated(context, &patterns); patterns.insert<KernelFnConversion>(context, typeConverter, workGroupSize); patterns.insert< - GPUReturnOpConversion, ForOpConversion, KernelModuleConversion, - KernelModuleTerminatorConversion, + GPUReturnOpConversion, ForOpConversion, GPUModuleConversion, LaunchConfigConversion<gpu::BlockDimOp, spirv::BuiltIn::WorkgroupSize>, LaunchConfigConversion<gpu::BlockIdOp, spirv::BuiltIn::WorkgroupId>, LaunchConfigConversion<gpu::GridDimOp, spirv::BuiltIn::NumWorkgroups>, diff --git a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp index 68392c36765..bc8273ec2a9 100644 --- a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp +++ b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.cpp @@ -60,15 +60,12 @@ void GPUToSPIRVPass::runOnModule() { SmallVector<Operation *, 1> kernelModules; OpBuilder builder(context); - module.walk([&builder, &kernelModules](ModuleOp moduleOp) { - if (moduleOp.getAttrOfType<UnitAttr>( - gpu::GPUDialect::getKernelModuleAttrName())) { - // For each kernel module (should be only 1 for now, but that is not a - // requirement here), clone the module for conversion because the - // gpu.launch function still needs the kernel module. - builder.setInsertionPoint(moduleOp.getOperation()); - kernelModules.push_back(builder.clone(*moduleOp.getOperation())); - } + module.walk([&builder, &kernelModules](gpu::GPUModuleOp moduleOp) { + // For each kernel module (should be only 1 for now, but that is not a + // requirement here), clone the module for conversion because the + // gpu.launch function still needs the kernel module. + builder.setInsertionPoint(moduleOp.getOperation()); + kernelModules.push_back(builder.clone(*moduleOp.getOperation())); }); SPIRVTypeConverter typeConverter; diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.td b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.td new file mode 100644 index 00000000000..cfe9d26273c --- /dev/null +++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.td @@ -0,0 +1,22 @@ +//===-- GPUToSPIRV.td - GPU to SPIR-V Dialect Lowerings ----*- tablegen -*-===// +// +// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains patterns to lower GPU dialect ops to to SPIR-V ops. +// +//===----------------------------------------------------------------------===// + + +#ifndef CONVERT_GPU_TO_SPIRV +#define CONVERT_GPU_TO_SPIRV + +include "mlir/Dialect/GPU/GPUOps.td" +include "mlir/Dialect/SPIRV/SPIRVStructureOps.td" + +def : Pat<(GPU_ModuleEndOp), (SPV_ModuleEndOp)>; + +#endif // CONVERT_GPU_TO_SPIRV |
