From a79e5b36d621a39c84e1e1bce343673dec7c7eec Mon Sep 17 00:00:00 2001 From: zhongsanming Date: Wed, 18 Nov 2026 11:17:57 +0800 Subject: [PATCH 1/7] [Ascend] Consume FLIR TritonToLinalgIncubated Build the FLIR incubated TTIR->Linalg target before the Ascend libraries, link it from TritonAscend/DiscreteMaskAccessConversion, and remove the local third_party/ascend TritonToLinalg copy. Register the core TTIR passes needed by the pass tests in the self-contained triton-opt. --- third_party/ascend/CMakeLists.txt | 32 +- third_party/ascend/bin/triton-opt.cpp | 10 +- third_party/ascend/include/CMakeLists.txt | 1 - .../TritonToLinalg/ArgMinMaxConverter.h | 356 -- .../include/TritonToLinalg/BlockPtrAnalysis.h | 378 -- .../include/TritonToLinalg/CMakeLists.txt | 3 - .../CanonicalizeDebugLocationsPass.h | 50 - .../TritonToLinalg/ConversionPatterns.h | 133 - .../TritonToLinalg/DeduplicateDebugNopsPass.h | 43 - .../TritonToLinalg/DescriptorConverter.h | 105 - .../TritonToLinalg/DevicePrintOffsetRewrite.h | 36 - .../TritonToLinalg/FunctionConverter.h | 60 - .../include/TritonToLinalg/HoistBroadcast.h | 81 - .../include/TritonToLinalg/ImplicitPermute.h | 129 - .../TritonToLinalg/LoadStoreConverter.h | 315 -- .../TritonToLinalg/MarkTensorKindPass.h | 56 - .../include/TritonToLinalg/MaskAnalysis.h | 31 - .../ascend/include/TritonToLinalg/Passes.h | 37 - .../ascend/include/TritonToLinalg/Passes.td | 49 - .../TritonToLinalg/StridedAxisCoalescing.h | 30 - .../TritonToLinalg/StridedLoadStoreRewrite.h | 30 - .../TritonToLinalg/TileChunkCoalescing.h | 38 - .../TritonToLinalg/TritonOpConverter.h | 923 ---- .../TritonToLinalg/TritonToLinalgPass.h | 121 - .../include/TritonToLinalg/UseAnalysis.h | 144 - third_party/ascend/lib/CMakeLists.txt | 2 - .../CMakeLists.txt | 3 +- .../DiscreteMaskAccessConversionPass.cpp | 4 +- .../lib/TritonToLinalg/ArgMinMaxConverter.cpp | 237 - .../lib/TritonToLinalg/BlockPtrAnalysis.cpp | 3291 ------------ .../ascend/lib/TritonToLinalg/CMakeLists.txt | 40 - .../CanonicalizeDebugLocationsPass.cpp | 93 - .../DeduplicateDebugNopsPass.cpp | 135 - .../TritonToLinalg/DescriptorConverter.cpp | 566 --- .../DevicePrintOffsetRewrite.cpp | 591 --- .../lib/TritonToLinalg/FunctionConverter.cpp | 63 - .../lib/TritonToLinalg/HoistBroadcast.cpp | 227 - .../lib/TritonToLinalg/ImplicitPermute.cpp | 636 --- .../lib/TritonToLinalg/LoadStoreConverter.cpp | 1650 ------ .../lib/TritonToLinalg/MarkTensorKindPass.cpp | 179 - .../lib/TritonToLinalg/TritonOpConverter.cpp | 4435 ----------------- .../lib/TritonToLinalg/TritonToLinalgPass.cpp | 2471 --------- .../ascend/lib/TritonToLinalg/UseAnalysis.cpp | 636 --- .../UnstructureConversionPass.cpp | 2 +- third_party/ascend/triton_ascend.cc | 4 +- 45 files changed, 43 insertions(+), 18413 deletions(-) delete mode 100644 third_party/ascend/include/TritonToLinalg/ArgMinMaxConverter.h delete mode 100644 third_party/ascend/include/TritonToLinalg/BlockPtrAnalysis.h delete mode 100644 third_party/ascend/include/TritonToLinalg/CMakeLists.txt delete mode 100644 third_party/ascend/include/TritonToLinalg/CanonicalizeDebugLocationsPass.h delete mode 100644 third_party/ascend/include/TritonToLinalg/ConversionPatterns.h delete mode 100644 third_party/ascend/include/TritonToLinalg/DeduplicateDebugNopsPass.h delete mode 100644 third_party/ascend/include/TritonToLinalg/DescriptorConverter.h delete mode 100644 third_party/ascend/include/TritonToLinalg/DevicePrintOffsetRewrite.h delete mode 100644 third_party/ascend/include/TritonToLinalg/FunctionConverter.h delete mode 100644 third_party/ascend/include/TritonToLinalg/HoistBroadcast.h delete mode 100644 third_party/ascend/include/TritonToLinalg/ImplicitPermute.h delete mode 100644 third_party/ascend/include/TritonToLinalg/LoadStoreConverter.h delete mode 100644 third_party/ascend/include/TritonToLinalg/MarkTensorKindPass.h delete mode 100644 third_party/ascend/include/TritonToLinalg/MaskAnalysis.h delete mode 100644 third_party/ascend/include/TritonToLinalg/Passes.h delete mode 100644 third_party/ascend/include/TritonToLinalg/Passes.td delete mode 100644 third_party/ascend/include/TritonToLinalg/StridedAxisCoalescing.h delete mode 100644 third_party/ascend/include/TritonToLinalg/StridedLoadStoreRewrite.h delete mode 100644 third_party/ascend/include/TritonToLinalg/TileChunkCoalescing.h delete mode 100644 third_party/ascend/include/TritonToLinalg/TritonOpConverter.h delete mode 100644 third_party/ascend/include/TritonToLinalg/TritonToLinalgPass.h delete mode 100644 third_party/ascend/include/TritonToLinalg/UseAnalysis.h delete mode 100644 third_party/ascend/lib/TritonToLinalg/ArgMinMaxConverter.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/BlockPtrAnalysis.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/CMakeLists.txt delete mode 100644 third_party/ascend/lib/TritonToLinalg/CanonicalizeDebugLocationsPass.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/DeduplicateDebugNopsPass.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/DescriptorConverter.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/DevicePrintOffsetRewrite.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/FunctionConverter.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/HoistBroadcast.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/ImplicitPermute.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/LoadStoreConverter.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/MarkTensorKindPass.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/TritonOpConverter.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/TritonToLinalgPass.cpp delete mode 100644 third_party/ascend/lib/TritonToLinalg/UseAnalysis.cpp diff --git a/third_party/ascend/CMakeLists.txt b/third_party/ascend/CMakeLists.txt index ece7f09dee..49a621a83a 100644 --- a/third_party/ascend/CMakeLists.txt +++ b/third_party/ascend/CMakeLists.txt @@ -21,10 +21,39 @@ set(TRITON_DISTRIBUTED_ASCEND_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/Triton-dis include_directories(${ASCENDNPU_IR_SRC_DIR}/bishengir/include) include_directories(${ASCENDNPU_IR_BINARY_DIR}/bishengir/include) # Tablegen'd files +# Ascend's TritonToLinalg lowering lives in FLIR. Create the target before the +# Ascend libraries so DiscreteMaskAccessConversion can link it; its Phase-A +# Ascend dependencies are attached after add_subdirectory(lib) below. +if(FLAGTREE_BACKEND STREQUAL "ascend") + add_subdirectory( + ${PROJECT_SOURCE_DIR}/third_party/flir/include/incubated/Conversion/TritonToLinalgIncubated + ${PROJECT_BINARY_DIR}/third_party/flir/include/incubated/Conversion/TritonToLinalgIncubated) + add_subdirectory( + ${PROJECT_SOURCE_DIR}/third_party/flir/lib/Conversion/TritonToLinalgIncubated + ${PROJECT_BINARY_DIR}/third_party/flir/lib/Conversion/TritonToLinalgIncubated) +endif() + add_subdirectory(include) add_subdirectory(lib) add_subdirectory(bin) +if(FLAGTREE_BACKEND STREQUAL "ascend") + target_link_libraries(TritonToLinalgIncubated PUBLIC + BiShengIRHIVMDialect + BiShengIRHFusionDialect + BiShengIRAnnotationDialect + BiShengIRScopeDialect + TritonAscendIR + TritonStructuredIR + TritonMemoryAccessAnalysis + TritonToStructured + TritonToUnstructure + TritonToGraph + TritonControlFlowOpt + MLIRTritonNPUUtils + ) +endif() + if(TRITON_BUILD_TD) if(NOT EXISTS "${TRITON_DISTRIBUTED_ASCEND_SRC_DIR}/CMakeLists.txt") message(FATAL_ERROR "Triton-Distributed submodule is not initialized: ${TRITON_DISTRIBUTED_ASCEND_SRC_DIR}") @@ -123,7 +152,7 @@ add_triton_plugin(TritonAscend LINK_LIBS CommonIRIR TritonToGraph - TritonToLinalg + TritonToLinalgIncubated BiShengIRScopeDialect BiShengIRHIVMDialect MLIRMemRefDialect @@ -134,6 +163,7 @@ add_triton_plugin(TritonAscend target_compile_definitions(TritonAscend PRIVATE __TLE_DSA__) target_link_libraries(TritonAscend PRIVATE CommonIRToHIVM CommonIRTransforms) +add_dependencies(TritonAscend TritonToLinalgIncubatedConversionPassIncGen) if(TARGET CommonIRIR) add_dependencies(TritonAscend CommonIRIR) endif() diff --git a/third_party/ascend/bin/triton-opt.cpp b/third_party/ascend/bin/triton-opt.cpp index dda8bbb774..8cf4c3a3c5 100644 --- a/third_party/ascend/bin/triton-opt.cpp +++ b/third_party/ascend/bin/triton-opt.cpp @@ -40,6 +40,7 @@ #include "mlir/Tools/mlir-opt/MlirOptMain.h" #include "triton/Dialect/Triton/IR/Dialect.h" +#include "triton/Dialect/Triton/Transforms/Passes.h" #include "ascend/include/Dialect/TritonAscend/IR/TritonAscendDialect.h" #include "ascend/include/Dialect/TritonStructured/IR/TritonStructuredDialect.h" @@ -54,7 +55,7 @@ #include "ascend/include/TritonToHFusion/Passes.h" #include "ascend/include/TritonToHIVM/Passes.h" #include "ascend/include/TritonToLLVM/Passes.h" -#include "ascend/include/TritonToLinalg/Passes.h" +#include "incubated/Conversion/TritonToLinalgIncubated/Passes.h" #include "ascend/include/TritonToStructured/Passes.h" #include "ascend/include/TritonToUnstructure/Passes.h" @@ -74,8 +75,9 @@ static void registerAscendPasses() { // triton-to-structured, discrete-mask-access-conversion, // triton-to-annotation, triton-to-unstructure (+ bubble-up-operation), // triton-to-hivm, triton-to-hfusion, triton-to-llvm, commonir-to-hivm - // (+ reduce-sum-strength), triton-to-linalg, merge-concat-load-buffer - // (via TritonToGraph) and add_dynamic_cv_pipeline. + // (+ reduce-sum-strength), triton-to-linalg-incubated, + // merge-concat-load-buffer (via TritonToGraph) and add_dynamic_cv_pipeline. + mlir::triton::registerTritonPasses(); mlir::triton::registerTritonControlFlowOptPasses(); mlir::triton::registerTritonToStructuredPasses(); mlir::triton::registerDiscreteMaskAccessConversionPasses(); @@ -85,7 +87,7 @@ static void registerAscendPasses() { mlir::triton::registerTritonToHFusionPasses(); mlir::triton::registerTritonToLLVMPasses(); mlir::triton::registerCommonIRToHIVMPasses(); - mlir::triton::registerTritonToLinalgPasses(); + mlir::triton::Incubated::registerTritonToLinalgIncubatedPasses(); mlir::triton::registerAutoBlockifyPasses(); mlir::triton::cfg::registerTritonToGraphPasses(); mlir::triton::registerAddDynamicCVPipelinePasses(); diff --git a/third_party/ascend/include/CMakeLists.txt b/third_party/ascend/include/CMakeLists.txt index a2737f1c11..60f198550f 100644 --- a/third_party/ascend/include/CMakeLists.txt +++ b/third_party/ascend/include/CMakeLists.txt @@ -3,7 +3,6 @@ add_subdirectory(TritonToAnnotation) add_subdirectory(TritonToHFusion) add_subdirectory(TritonToHIVM) add_subdirectory(TritonToGraph) -add_subdirectory(TritonToLinalg) add_subdirectory(TritonControlFlowOpt) add_subdirectory(Utils) add_subdirectory(DiscreteMaskAccessConversion) diff --git a/third_party/ascend/include/TritonToLinalg/ArgMinMaxConverter.h b/third_party/ascend/include/TritonToLinalg/ArgMinMaxConverter.h deleted file mode 100644 index f09b23f360..0000000000 --- a/third_party/ascend/include/TritonToLinalg/ArgMinMaxConverter.h +++ /dev/null @@ -1,356 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_ARGMINMAXCONVERTER_H -#define TRITON_ADAPTER_ARGMINMAXCONVERTER_H - -#include "ConversionPatterns.h" -#include "ascend/include/Utils/Utils.h" - -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Transforms/DialectConversion.h" - -#define DEBUG_TYPE "triton-to-linalg" - -#include "llvm/Support/Debug.h" -#include "llvm/Support/LogicalResult.h" - -#include - -namespace TTOpConverters { -using namespace mlir; -using namespace triton; - -template -class ArgMinMaxBaseConverter : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult matchTieBreakResult(Value currValue, Value currIndex, - Value reduceValue, Value reduceIndex, - mlir::Block::iterator &it, - Value &tileBreakValue) const { - LLVM_DEBUG(llvm::dbgs() << "Matching: " << *it << "\n"); - auto eqCmpOp = dyn_cast(*it); - if (eqCmpOp) { - if (eqCmpOp.getPredicate() != arith::CmpFPredicate::OEQ || - currValue != eqCmpOp.getLhs() || reduceValue != eqCmpOp.getRhs()) { - return failure(); - } - } - - auto eqCmpIOp = dyn_cast(*it++); - if (eqCmpIOp) { - if (eqCmpIOp.getPredicate() != arith::CmpIPredicate::eq || - currValue != eqCmpIOp.getLhs() || reduceValue != eqCmpIOp.getRhs()) { - return failure(); - } - } - - LLVM_DEBUG(llvm::dbgs() << "Matching: " << *it << "\n"); - auto sltCmpOp = dyn_cast(*it++); - if (!sltCmpOp || sltCmpOp.getPredicate() != arith::CmpIPredicate::slt || - currIndex != sltCmpOp.getLhs() || reduceIndex != sltCmpOp.getRhs()) { - return failure(); - } - - // matching: %13 = arith.andi %11, %12 : i1 - LLVM_DEBUG(llvm::dbgs() << "Matching: " << *it << "\n"); - auto andOp = dyn_cast(*it++); - - Value cmpOp; - if (eqCmpOp) - cmpOp = eqCmpOp; - else - cmpOp = eqCmpIOp; - - if (!andOp || andOp.getLhs() != cmpOp || andOp.getRhs() != sltCmpOp) { - return failure(); - } - - tileBreakValue = andOp; - return success(); - } - - LogicalResult matchShouldUpdateValue(Value currValue, Value currIndex, - Value reduceValue, Value reduceIndex, - mlir::Block::iterator &it, - Value &shouldUpdate) const { - Value tieResult; - if (failed(matchTieBreakResult(currValue, currIndex, reduceValue, - reduceIndex, it, tieResult))) { - LLVM_DEBUG(llvm::dbgs() << "Tie break result match failed\n"); - return failure(); - } - - Value comparisonResult; - if (failed(T::matchComparisonResult(currValue, currIndex, reduceValue, - reduceIndex, it, comparisonResult))) { - LLVM_DEBUG(llvm::dbgs() << "Comparison result match failed\n"); - return failure(); - } - - LLVM_DEBUG(llvm::dbgs() << "Matching: " << *it << "\n"); - auto orOp = dyn_cast(*it++); - if (!orOp || orOp.getLhs() != comparisonResult || - orOp.getRhs() != tieResult) { - return failure(); - } - - shouldUpdate = orOp; - return success(); - } - - Value getInitTensor(ConversionPatternRewriter &rewriter, - ArrayRef shape, Value fillValue, - Location loc) const { - Value initTensor = - rewriter.create(loc, shape, fillValue.getType()); - return rewriter - .create(loc, ValueRange{fillValue}, - ValueRange{initTensor}) - .result(); - } - -public: - ArgMinMaxBaseConverter(MLIRContext *context) : OpConversionPattern(context) {} - - LogicalResult - matchAndRewrite(triton::ReduceOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - if (op.getBody()->getNumArguments() != 4) { - return failure(); - } - - auto block = op.getBody(); - auto ops = block->without_terminator(); - - Value currValue = block->getArgument(0); - Value currIndex = block->getArgument(1); - Value reduceValue = block->getArgument(2); - Value reduceIndex = block->getArgument(3); - - auto opsIt = ops.begin(); - Value shouldUpdate; - if (failed(matchShouldUpdateValue(currValue, currIndex, reduceValue, - reduceIndex, opsIt, shouldUpdate))) { - return failure(); - } - - LLVM_DEBUG(llvm::dbgs() << "Matching: " << *opsIt << "\n"); - auto valueSelectOp = dyn_cast(*opsIt++); - if (!valueSelectOp || valueSelectOp.getCondition() != shouldUpdate || - currValue != valueSelectOp.getTrueValue() || - reduceValue != valueSelectOp.getFalseValue()) { - return failure(); - } - - LLVM_DEBUG(llvm::dbgs() << "Matching: " << *opsIt << "\n"); - auto indexSelectOp = dyn_cast(*opsIt++); - if (!indexSelectOp || indexSelectOp.getCondition() != shouldUpdate || - currIndex != indexSelectOp.getTrueValue() || - reduceIndex != indexSelectOp.getFalseValue()) { - return failure(); - } - - LLVM_DEBUG(llvm::dbgs() << "Matching: " << *opsIt << "\n"); - auto termOp = dyn_cast(*opsIt++); - if (!(termOp && termOp == block->getTerminator() && - termOp.getOperands() == - ArrayRef{valueSelectOp, indexSelectOp})) { - return failure(); - } - - // Rewrite phase: perform the actual conversion - auto loc = op.getLoc(); - auto elemTypes = op.getElementTypes(); - - auto valueType = elemTypes[0]; - // tl.argmin reorder - bool isUnsigned = false; - if (isa(valueType)) { - arith::CmpFOp cmpFOp; - block->walk([&](arith::CmpFOp cmpOp) { - auto pred = cmpOp.getPredicate(); - if (pred == arith::CmpFPredicate::OEQ || - pred == arith::CmpFPredicate::ONE || - pred == arith::CmpFPredicate::UEQ || - pred == arith::CmpFPredicate::UNE) { - return WalkResult::advance(); - } else if (pred == arith::CmpFPredicate::OGT || - pred == arith::CmpFPredicate::OLT || - pred == arith::CmpFPredicate::UGT || - pred == arith::CmpFPredicate::ULT) { - cmpFOp = cmpOp; - return WalkResult::interrupt(); - } - return WalkResult::advance(); - }); - cmpFOp->moveBefore(block, block->getOperations().begin()); - } else if (isa(valueType)) { - arith::CmpIOp cmpIOp; - block->walk([&](arith::CmpIOp cmpOp) { - auto pred = cmpOp.getPredicate(); - if (pred == arith::CmpIPredicate::ugt || - pred == arith::CmpIPredicate::ult) { - isUnsigned = true; - } - if (pred == arith::CmpIPredicate::eq || - pred == arith::CmpIPredicate::ne) { - return WalkResult::advance(); - } else if (pred == arith::CmpIPredicate::sgt || - pred == arith::CmpIPredicate::slt || - pred == arith::CmpIPredicate::ugt || - pred == arith::CmpIPredicate::ult) { - if (cmpOp.getLhs() == block->getArgument(0) && - cmpOp.getRhs() == block->getArgument(2)) { - cmpIOp = cmpOp; - return WalkResult::interrupt(); - } - } - return WalkResult::advance(); - }); - cmpIOp->moveBefore(block, block->getOperations().begin()); - } - - TypedAttr valueAttr; - if (isa(valueType)) { - valueAttr = rewriter.getFloatAttr(valueType, T::getBaseReductionValue()); - } else if (isa(valueType)) { - if (isUnsigned) { - valueAttr = - rewriter.getIntegerAttr(valueType, T::getBaseReductionUIntValue()); - } else { - valueAttr = - rewriter.getIntegerAttr(valueType, T::getBaseReductionIntValue()); - } - } - - auto reduceWithIndexParams = getReduceWithIndexParams(op); - auto valuesAccBaseVal = - rewriter.create(loc, valueType, valueAttr); - int indicesInitValue = - (llvm::succeeded(reduceWithIndexParams) && - reduceWithIndexParams->tieBreakType == TieBreakType::RIGHT) - ? -1 - : std::numeric_limits::max(); - - auto indexType = elemTypes[1]; - auto indicesAccBaseVal = rewriter.create( - loc, indexType, rewriter.getIntegerAttr(indexType, indicesInitValue)); - - auto valueResultType = dyn_cast(op.getType(0)); - const auto isScalarReduce = valueResultType == nullptr; - SmallVector reductionResultShape{ - isScalarReduce ? SmallVector{} - : SmallVector(valueResultType.getShape())}; - - SmallVector outputs{ - getInitTensor(rewriter, reductionResultShape, valuesAccBaseVal, loc), - getInitTensor(rewriter, reductionResultShape, indicesAccBaseVal, loc)}; - - auto linalgOp = rewriter.create( - loc, adaptor.getOperands(), outputs, - SmallVector{adaptor.getAxis()}, - [&](OpBuilder &b, Location loc, ValueRange inputs) { - assert(inputs.size() == 4); - - auto tritonReduceBlock = op.getBody(); - IRMapping mapping; - mapping.map(tritonReduceBlock->getArguments(), inputs); - - for (auto &op : tritonReduceBlock->without_terminator()) { - b.clone(op, mapping); - } - - auto tritonYield = tritonReduceBlock->getTerminator(); - auto results = - llvm::map_to_vector(tritonYield->getOperands(), [&](Value val) { - return mapping.lookup(val); - }); - b.create(loc, results); - }); - - // before we rewrite the argmax reduce op, we know it has return value - // so addReduceWithIndexAttrIfNeeded won't fail - // but ignoring it will lead to compiling failure - if (llvm::succeeded(reduceWithIndexParams) && - reduceWithIndexParams->tieBreakType != TieBreakType::None) { - addReduceWithIndexAttr(*reduceWithIndexParams, rewriter, linalgOp); - } - - if (isScalarReduce) { - SmallVector reduceResults{ - rewriter.create( - loc, valueType, linalgOp.getResults()[0], ValueRange{}), - rewriter.create( - loc, indexType, linalgOp.getResults()[1], ValueRange{})}; - rewriter.replaceOp(op, reduceResults); - } else { - rewriter.replaceOp(op, linalgOp); - } - - return success(); - } -}; - -class ArgMinConverter : public ArgMinMaxBaseConverter { -public: - static LogicalResult matchComparisonResult(Value currValue, Value currIndex, - Value reduceValue, - Value reduceIndex, - mlir::Block::iterator &it, - Value &comparisonResult); - - static float getBaseReductionValue(); - - static int8_t getBaseReductionIntValue(); - static uint8_t getBaseReductionUIntValue(); - - ArgMinConverter(MLIRContext *context) : ArgMinMaxBaseConverter(context) {} -}; - -class ArgMaxConverter : public ArgMinMaxBaseConverter { -public: - static LogicalResult matchComparisonResult(Value currValue, Value currIndex, - Value reduceValue, - Value reduceIndex, - mlir::Block::iterator &it, - Value &comparisonResult); - - static float getBaseReductionValue(); - - static int8_t getBaseReductionIntValue(); - static uint8_t getBaseReductionUIntValue(); - - ArgMaxConverter(MLIRContext *context) : ArgMinMaxBaseConverter(context) {} -}; - -void populatePostConversionCanonicalizationPatterns( - RewritePatternSet &patterns); - -} // namespace TTOpConverters - -#endif diff --git a/third_party/ascend/include/TritonToLinalg/BlockPtrAnalysis.h b/third_party/ascend/include/TritonToLinalg/BlockPtrAnalysis.h deleted file mode 100644 index c1d799a1bc..0000000000 --- a/third_party/ascend/include/TritonToLinalg/BlockPtrAnalysis.h +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ANALYSIS_BLOCKPTRANALYSIS_H -#define TRITON_ANALYSIS_BLOCKPTRANALYSIS_H -#include "bishengir/Dialect/HIVM/IR/HIVM.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" - -#include "mlir/IR/Builders.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Value.h" -#include "triton/Dialect/Triton/IR/Dialect.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" - -#include -namespace mlir { - -class ConversionPatternRewriter; - -namespace triton { - -/// Temporary provenance for memref carriers materialized from complete scalar -/// pointer addresses. Address round-trip folding is valid only for casts with -/// this marker; ordinary HIVM PointerCast operations retain their baseline -/// descriptor semantics. TritonToLinalg removes the marker after conversion. -inline constexpr llvm::StringLiteral kScalarPointerCarrierAttr = - "ScalarPointerCarrier"; - -/// Returns true when a loop still carries Triton pointer state, or when a -/// loop-carried integer tensor participates in legacy address computation. -/// Ordinary memrefs, including fixed-layout memrefs created by -/// memref.reinterpret_cast, are complete SSA values and do not require the -/// legacy BlockData loop expansion solely because of their producer. -bool needsLegacyBlockDataLoopRewrite(LoopLikeOpInterface loopOp); - -enum class MemAccVal { Undefined = 0, StrucMemAcc = 1, UnstrucMemAcc = 2 }; - -/// Creates a verifier-valid HIVM pointer cast for a scalar Triton pointer -/// represented by an integer address. Triton scalar pointers do not carry an -/// extent, while their downstream carrier is normally `memref` and every -/// dynamic memref dimension requires a size operand. Use one element as the -/// conservative carrier extent; reinterpret-cast lowering replaces it with a -/// precise access range when a larger descriptor is materialized. -hivm::PointerCastOp createScalarPointerCast(OpBuilder &builder, Location loc, - MemRefType resultType, - Value address); - -struct MemAccType { - - MemAccVal value; - - explicit constexpr MemAccType(MemAccVal v = MemAccVal::Undefined) - : value(v) {} - - constexpr operator MemAccVal() const { return value; } - explicit operator bool() = delete; - - constexpr bool isUndefined() const { return value == MemAccVal::Undefined; } - constexpr bool isStructured() const { - return value == MemAccVal::StrucMemAcc; - } - constexpr bool isUnstructured() const { - return value == MemAccVal::UnstrucMemAcc; - } - - void merge(MemAccType &other) { - this->value = (this->value > other.value) ? this->value : other.value; - } - - std::string_view toString() const { - static constexpr std::string_view names[] = {"Undefined", "StrucMemAcc", - "UnstrucMemAcc"}; - return names[static_cast(value)]; - } -}; - -class BlockData { -public: - SmallVector &getOffsetsRef(); - SmallVector &getSizesRef(); - SmallVector &getStridesRef(); - Value &getSourceRef(); - OpFoldResult &getScalarRef(); - Type &getResElemTyRef(); - MemAccType &getMemAccTypeRef(); - - SmallVector getOffsets() const; - SmallVector getSizes() const; - SmallVector getStrides() const; - Type getResElemTy() const; - OpFoldResult getOffset(int) const; - OpFoldResult getSize(int) const; - OpFoldResult getStride(int) const; - OpFoldResult getScalar() const; - Value getSource() const; - MemAccType getMemAccType() const; - - bool isScalar() const; - bool isEmpty() const; - bool hasSource() const; - bool hasResElemTy() const; - void removeSource(); - - int64_t getRank() const; - FailureOr - getResultMemrefType(int64_t offset, ArrayRef resultShape) const; - - void addBlock(BlockData &lBlock, BlockData &rBlock, Location loc, - ConversionPatternRewriter &rewriter); - void subBlock(BlockData &lBlock, BlockData &rBlock, Location loc, - ConversionPatternRewriter &rewriter); - void mulBlock(BlockData &lBlock, BlockData &rBlock, Location loc, - ConversionPatternRewriter &rewriter); - void divBlock(BlockData &lBlock, BlockData &rBlock, Location loc, - ConversionPatternRewriter &rewriter); - - FailureOr - createCastOp(ArrayRef resultShape, const Location &loc, - OpBuilder &builder) const; - - void setResElemTy(const Type &); - void setSource(const Value &); - void setScalar(const OpFoldResult &); - void setOffsets(const SmallVector &); - void setStrides(const SmallVector &); - void setSizes(const SmallVector &); - void setMemAccTy(const MemAccType &); - void setMemAccVal(const MemAccVal); - - void dump() const; - -private: - SmallVector offsets; - SmallVector sizes; - SmallVector strides; - Value source; - // `Scalar` is a shortcut used when the entire blockdata describes a single - // scalar value - OpFoldResult scalar; - Type resElemTy; - MemAccType memAccTy; - - // Accumulate offsets of each dimension in BlockData to get a total offset - // from source ptr, which is used in memref::ReinterpretCastOp - OpFoldResult inferBlockOffset(const Location &loc, OpBuilder &builder) const; -}; - -class BlockDataParser { -public: - /// Materialize the rank-one memref used by a scalar load only when the - /// pointer producer has already been converted to a compatible memref. - /// Returning failure lets a consumer reject an unconverted producer without - /// asserting or creating partial IR. - static FailureOr getScalarMemRef(Value ptr, Value memref, - const Location &loc, - ConversionPatternRewriter &rewriter); - - static LogicalResult - parse(Value operand, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseAdd(arith::AddIOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseSub(arith::SubIOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseMul(arith::MulIOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseDiv(arith::DivSIOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseRem(arith::RemSIOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static void - parseUnrealizedCast(UnrealizedConversionCastOp op, BlockData &data, - const Location &loc, ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static void - parseMakeRange(triton::MakeRangeOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static void parseLinalgGenericFromMakeRange( - linalg::GenericOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseExpandDims(triton::ExpandDimsOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseBitcast(triton::BitcastOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseExtSI(arith::ExtSIOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseBroadcast(triton::BroadcastOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseSplat(triton::SplatOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static void - parseConstSplat(arith::ConstantOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - template - static std::enable_if_t || - std::is_same_v, - LogicalResult> - parseTensorPtr(T op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseAddPtr(triton::AddPtrOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseExtractSlice(tensor::ExtractSliceOp op, BlockData &data, - const Location &loc, ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static void - parseReinterpretCast(memref::ReinterpretCastOp op, BlockData &data, - const Location &loc, ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseReduce(triton::ReduceOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static void - parseAtomicRmw(triton::AtomicRMWOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseFill(linalg::FillOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static LogicalResult - parseSelect(arith::SelectOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - static void parseCustomOp(hivm::CustomOp op, BlockData &data, - const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known, - unsigned resultIdx); - - static LogicalResult - parseStructuredCustomOp(Operation *op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known, - unsigned resultIdx); - - static void rewriteStructuredCustomOp(hivm::CustomOp op, - hivm::CustomOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter); - - static void rewriteStructuredCustomOp(hivm::CustomMacroOp op, - hivm::CustomMacroOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter); - - static void rewriteStructuredCustomOp(Operation *op, - ConversionPatternRewriter &rewriter); - - static LogicalResult - rewriteAddPtr(triton::AddPtrOp op, triton::AddPtrOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known); - - static FailureOr - materializePointer(Value ptr, ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known); - - static LogicalResult - rewriteMakeTensorPtrOp(triton::MakeTensorPtrOp op, Value convertedBase, - ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known); - - static LogicalResult - rewriteAdvanceOp(triton::AdvanceOp op, ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known); - - static LogicalResult - rewriteCustomOp(hivm::CustomOp op, hivm::CustomOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known); - - template - static std::enable_if_t || - std::is_same_v, - LogicalResult> - rewriteTerminator(T op, ConversionPatternRewriter &rewriter, - const llvm::SmallDenseSet &blockArgIdxSet, - ArrayRef iterArgIdxMap, - const llvm::SmallDenseMap &known); - - /// @param known is mainly designed for `rewriteLoop`, and is just non-const - /// in `rewriteLoop`, `rewriteAddPtr` and `rewriteAdvance` - static LogicalResult - rewriteLoopOp(LoopLikeOpInterface op, ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known, - ArrayRef onlyIndexTensorSlots = {}); - - static LogicalResult rewriteAddPtrToUnstrucMemAcc( - triton::AddPtrOp op, triton::AddPtrOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter, BlockData &data); -}; - -template -void parseIndirectLoad(OpTy op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known, - unsigned resultIdx = 0); - -} // namespace triton - -} // namespace mlir - -#endif diff --git a/third_party/ascend/include/TritonToLinalg/CMakeLists.txt b/third_party/ascend/include/TritonToLinalg/CMakeLists.txt deleted file mode 100644 index d62f670bbe..0000000000 --- a/third_party/ascend/include/TritonToLinalg/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS Passes.td) -mlir_tablegen(Passes.h.inc -gen-pass-decls --name TritonToLinalg) -add_public_tablegen_target(TritonToLinalgConversionPassIncGen) diff --git a/third_party/ascend/include/TritonToLinalg/CanonicalizeDebugLocationsPass.h b/third_party/ascend/include/TritonToLinalg/CanonicalizeDebugLocationsPass.h deleted file mode 100644 index 854ad875ee..0000000000 --- a/third_party/ascend/include/TritonToLinalg/CanonicalizeDebugLocationsPass.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -//===----------------------------------------------------------------------===// -// CanonicalizeDebugLocationsPass -// -// Collapses call-site locations whose callee is an inlined Triton stdlib -// helper (under site-packages) down to their caller (user-file) frame, so the -// emitted DWARF references the user's source line instead of the library. -// -// Run BEFORE DeduplicateDebugNopsPass. Gated by -// LLVM_EXTRACT_DI_LOCAL_VARIABLES=1. -//===----------------------------------------------------------------------===// - -#ifndef TRITON_ASCEND_TRITONTOLINALG_CANONICALIZEDEBUGLOCATIONSPASS_H -#define TRITON_ASCEND_TRITONTOLINALG_CANONICALIZEDEBUGLOCATIONSPASS_H - -// This pass is added programmatically to the TritonToLinalg pipeline (see -// compiler.py) rather than registered through TableGen, so it has no -// Passes.td entry and no GEN_PASS_DECL include. -#include "mlir/Pass/Pass.h" -#include - -namespace mlir { -namespace triton { - -std::unique_ptr createCanonicalizeDebugLocationsPass(); - -} // namespace triton -} // namespace mlir - -#endif // TRITON_ASCEND_TRITONTOLINALG_CANONICALIZEDEBUGLOCATIONSPASS_H diff --git a/third_party/ascend/include/TritonToLinalg/ConversionPatterns.h b/third_party/ascend/include/TritonToLinalg/ConversionPatterns.h deleted file mode 100644 index 4199810dcf..0000000000 --- a/third_party/ascend/include/TritonToLinalg/ConversionPatterns.h +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef CONVERSIONPATTERNS_H -#define CONVERSIONPATTERNS_H - -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/Affine/IR/AffineOps.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Linalg/Passes.h" - -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/MathExtras.h" - -#include -#include -#include - -using namespace mlir; -using namespace triton; - -//===----------------------------------------------------------------------===// -// Utilities -//===----------------------------------------------------------------------===// - -static Value getScalarValue(Value operand, Location loc, - ConversionPatternRewriter &rewriter) { - SmallVector ops; - - auto reconstructScalarValue = [&](Value src) { - for (auto op = ops.rbegin(); op != ops.rend(); ++op) { - src = TypeSwitch(*op) - .Case([&](Operation *op) { - auto resType = op->getResults()[0].getType(); - if (auto shapedType = dyn_cast(resType)) { - resType = shapedType.getElementType(); - } - return rewriter.create(loc, resType, src); - }) - .Case([&](Operation *op) { - auto resType = op->getResults()[0].getType(); - if (auto shapedType = dyn_cast(resType)) { - resType = shapedType.getElementType(); - } - return rewriter.create(loc, resType, src); - }) - .Default([](Operation *op) { - llvm_unreachable("unsupported op in generating "); - return nullptr; - }); - } - return src; - }; - - while (true) { - if (!dyn_cast(operand.getType())) { - return reconstructScalarValue(operand); - } else if (auto op = operand.getDefiningOp()) { - if (auto attr = dyn_cast(op.getValue())) { - if (!attr.isSplat()) { - InFlightDiagnostic diag = emitError(loc) - << "other value used in masked load " - "produced by unsupported instruction"; - return nullptr; - } - auto elemValue = attr.getSplatValue(); - auto constOp = arith::ConstantOp::materialize( - rewriter, elemValue, attr.getElementType(), op.getLoc()); - return reconstructScalarValue(constOp.getResult()); - } - } else if (auto op = operand.getDefiningOp()) { - operand = op.getSrc(); - } else if (auto op = operand.getDefiningOp()) { - ops.push_back(op.getOperation()); - operand = op.getIn(); - } else if (auto op = operand.getDefiningOp()) { - ops.push_back(op.getOperation()); - operand = op.getIn(); - } else { - InFlightDiagnostic diag = emitError(loc) - << "other value used in masked load produced " - "by unsupported instruction"; - return nullptr; - } - } - return nullptr; -} - -static SmallVector getNParallelLoopsAttrs(unsigned n) { - return SmallVector(n, utils::IteratorType::parallel); -} - -// for IntLike and FloatLike types -static std::optional getBitWidth(Type a) { - if (auto type = dyn_cast(a)) { - auto elementType = type.getElementType(); - if (elementType.isIntOrFloat()) { - return type.getElementType().getIntOrFloatBitWidth(); - } - return std::nullopt; - } - - if (a.isIntOrFloat()) { - return a.getIntOrFloatBitWidth(); - } - return std::nullopt; -} -#endif // CONVERSIONPATTERNS_H diff --git a/third_party/ascend/include/TritonToLinalg/DeduplicateDebugNopsPass.h b/third_party/ascend/include/TritonToLinalg/DeduplicateDebugNopsPass.h deleted file mode 100644 index 0cc7762358..0000000000 --- a/third_party/ascend/include/TritonToLinalg/DeduplicateDebugNopsPass.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ASCEND_TRITONTOLINALG_DEDUPLICATE_DEBUG_NOPS_H -#define TRITON_ASCEND_TRITONTOLINALG_DEDUPLICATE_DEBUG_NOPS_H - -#include "mlir/Pass/Pass.h" -#include - -namespace mlir { -namespace triton { - -/// Creates a pass that removes duplicate `llvm.inline_asm "nop"` debug anchors, -/// keeping the first per source line so DWARF line tables have one PC per line. -/// Opt-in via `LLVM_EXTRACT_DI_LOCAL_VARIABLES=1`. -std::unique_ptr createDeduplicateDebugNopsPass(); - -#define GEN_PASS_DECL_DEDUPLICATEDEBUGNOPS -#include "ascend/include/TritonToLinalg/Passes.h.inc" - -} // namespace triton -} // namespace mlir - -#endif // TRITON_ASCEND_TRITONTOLINALG_DEDUPLICATE_DEBUG_NOPS_H diff --git a/third_party/ascend/include/TritonToLinalg/DescriptorConverter.h b/third_party/ascend/include/TritonToLinalg/DescriptorConverter.h deleted file mode 100644 index dd2aaeac3d..0000000000 --- a/third_party/ascend/include/TritonToLinalg/DescriptorConverter.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_DESCRIPTORCONVERTER_H -#define TRITON_ADAPTER_DESCRIPTORCONVERTER_H - -#include "TritonToLinalg/BlockPtrAnalysis.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Debug.h" - -namespace DescriptorConverter { -using namespace mlir; -using namespace triton; - -struct Descriptor { - Value base; - SmallVector shape; - SmallVector strides; - triton::PaddingOptionAttr padding; -}; - -bool hasATensorDescriptorType(mlir::TypeRange types); - -class DescriptorLoadConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::DescriptorLoadOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class DescriptorStoreConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::DescriptorStoreOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class DescriptorGatherConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::DescriptorGatherOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class DescriptorScatterConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::DescriptorScatterOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class DescriptorReduceConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::DescriptorReduceOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -} // end of namespace DescriptorConverter - -#endif // TRITON_ADAPTER_DESCRIPTORCONVERTER_H diff --git a/third_party/ascend/include/TritonToLinalg/DevicePrintOffsetRewrite.h b/third_party/ascend/include/TritonToLinalg/DevicePrintOffsetRewrite.h deleted file mode 100644 index 9d2f00fee3..0000000000 --- a/third_party/ascend/include/TritonToLinalg/DevicePrintOffsetRewrite.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ASCEND_TRITONTOLINALG_DEVICEPRINTOFFSETREWRITE_H -#define TRITON_ASCEND_TRITONTOLINALG_DEVICEPRINTOFFSETREWRITE_H - -#include "mlir/IR/BuiltinOps.h" - -namespace mlir { -namespace triton { - -void rewriteDevicePrintOffsets(ModuleOp moduleOp); - -} // namespace triton -} // namespace mlir - -#endif // TRITON_ASCEND_TRITONTOLINALG_DEVICEPRINTOFFSETREWRITE_H diff --git a/third_party/ascend/include/TritonToLinalg/FunctionConverter.h b/third_party/ascend/include/TritonToLinalg/FunctionConverter.h deleted file mode 100644 index c0f10e620d..0000000000 --- a/third_party/ascend/include/TritonToLinalg/FunctionConverter.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_FUNCTIONCONVERTER_H -#define TRITON_ADAPTER_FUNCTIONCONVERTER_H - -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Transforms/DialectConversion.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -namespace FunctionConverter { -using namespace mlir; -using namespace triton; - -class GetProgramIDConverter - : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - static uint32_t constexpr LAUNCH_GRID_RANK = - getMaxEnumValForProgramIDDim() + 1; - -public: - LogicalResult - matchAndRewrite(triton::GetProgramIdOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class GetNumProgramsConverter - : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - static uint32_t constexpr LAUNCH_GRID_RANK = - getMaxEnumValForProgramIDDim() + 1; - -public: - LogicalResult - matchAndRewrite(triton::GetNumProgramsOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; -} // namespace FunctionConverter -#endif diff --git a/third_party/ascend/include/TritonToLinalg/HoistBroadcast.h b/third_party/ascend/include/TritonToLinalg/HoistBroadcast.h deleted file mode 100644 index 1ddf20f2fb..0000000000 --- a/third_party/ascend/include/TritonToLinalg/HoistBroadcast.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_TRITONTOLINALG_HOISTBROADCAST_H -#define TRITON_ADAPTER_TRITONTOLINALG_HOISTBROADCAST_H - -#include "ascend/include/TritonToLinalg/BlockPtrAnalysis.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Debug.h" - -#define DEBUG_TYPE "triton-to-linalg" - -namespace HoistBroadcast { -using namespace mlir; -using namespace triton; - -class BroadcastConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::BroadcastOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class BroadcastHoister { -public: - BroadcastHoister(triton::BroadcastOp op); - LogicalResult parse(Value operand, const Location &loc, - ConversionPatternRewriter &rewriter); - LogicalResult parseAddptr(triton::AddPtrOp op, const Location &loc, - ConversionPatternRewriter &rewriter); - LogicalResult parseBroadcast(triton::BroadcastOp op, const Location &loc, - ConversionPatternRewriter &rewriter); - LogicalResult parseSplat(triton::SplatOp op, const Location &loc, - ConversionPatternRewriter &rewriter); - LogicalResult findSrc(Value operand); - LogicalResult replaceBroadcastOp(triton::BroadcastOp op, - ConversionPatternRewriter &rewriter); - bool canBroadcast(); - -private: - Value source; - triton::BroadcastOp opToHoist; - SmallVector tensorSizes; - llvm::SmallDenseMap broadcastMap; -}; -} // namespace HoistBroadcast - -#endif // TRITON_ADAPTER_TRITONTOLINALG_HOISTBROADCAST_H diff --git a/third_party/ascend/include/TritonToLinalg/ImplicitPermute.h b/third_party/ascend/include/TritonToLinalg/ImplicitPermute.h deleted file mode 100644 index b048fa4bd6..0000000000 --- a/third_party/ascend/include/TritonToLinalg/ImplicitPermute.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "TritonMemoryAccess/MemoryAccessTags.h" -#include "TritonToStructured/MaskAnalysis.h" -#include "TritonToStructured/PtrAnalysis.h" -#include "bishengir/Dialect/HIVM/IR/HIVM.h" -#include "mlir/Dialect/Arith/Utils/Utils.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/MLIRContext.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Value.h" -#include "mlir/Support/LogicalResult.h" -#include "mlir/Transforms/DialectConversion.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -namespace ImplicitPermute { - -using namespace mlir; -using namespace triton; - -// Tag attached to load/store/atomic ops that this pass rewrote, so downstream -// sub-steps (e.g. StridedLoadStoreRewrite) can detect "already handled by -// ImplicitPermute" and avoid double-processing. -inline constexpr const char *ImplicitPermuteHandledTAG = - mlir::triton::memory_access::ImplicitPermuteHandledTAG; - -class LoadConverter : public OpRewritePattern { -public: - explicit LoadConverter(MLIRContext *context) - : OpRewritePattern(context) {}; - - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(triton::LoadOp op, - PatternRewriter &rewriter) const override; -}; - -class StoreConverter : public OpRewritePattern { -public: - explicit StoreConverter(MLIRContext *context) - : OpRewritePattern(context) {}; - - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(triton::StoreOp op, - PatternRewriter &rewriter) const override; -}; - -class AtomicRMWConverter : public OpRewritePattern { -public: - explicit AtomicRMWConverter(MLIRContext *context) - : OpRewritePattern(context) {}; - - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(triton::AtomicRMWOp op, - PatternRewriter &rewriter) const override; -}; - -class AtomicCASConverter : public OpRewritePattern { -public: - explicit AtomicCASConverter(MLIRContext *context) - : OpRewritePattern(context) {}; - - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(triton::AtomicCASOp op, - PatternRewriter &rewriter) const override; -}; - -class MemOpTransformer { -public: - TritonToStructured::PtrState ptrState; - TritonToStructured::MaskState maskState; - - enum class MemType { load, store, deafaultType }; - - MemType currentType = MemType::deafaultType; - - MemOpTransformer(MemType memType) : currentType(memType) {} - - Value materializeImplicitPermute(Value srcTensor, const Location loc, - PatternRewriter &rewriter); - - Value createNewAddPtr(Value oldPtr, const Location loc, - PatternRewriter &rewriter); - - Value createNewAdvancePtr(Value oldPtr, const Location loc, - PatternRewriter &rewriter); - - Value createNewTensorPtr(Value oldPtr, const Location loc, - PatternRewriter &rewriter); - - Value createNewMask(Value oldPtr, const Location loc, - PatternRewriter &rewriter); - - Value createNewOther(Value oldOther, const Location loc, - PatternRewriter &rewriter); - - SmallVector - getBoundaryCheck(ArrayRef oldBoundaryCheck) const; - - bool applyPermuteOnMask(); -}; - -} // namespace ImplicitPermute diff --git a/third_party/ascend/include/TritonToLinalg/LoadStoreConverter.h b/third_party/ascend/include/TritonToLinalg/LoadStoreConverter.h deleted file mode 100644 index e60c03dcf1..0000000000 --- a/third_party/ascend/include/TritonToLinalg/LoadStoreConverter.h +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_LOADSTORECONVERTER_H -#define TRITON_ADAPTER_LOADSTORECONVERTER_H - -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/MLIRContext.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Value.h" -#include "mlir/Support/LogicalResult.h" -#include "mlir/Transforms/DialectConversion.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/Arith/Utils/Utils.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" - -#include "ascend/include/TritonToLinalg/BlockPtrAnalysis.h" - -namespace LoadStoreConverter { - -using namespace mlir; -using namespace triton; - -class AddPtrConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::AddPtrOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -/// Materialize pointer tensor expressions directly instead of routing them -/// through a synthetic AddPtrOp with a zero offset. A higher benefit than the -/// generic value converters ensures pointer layout is handled by -/// BlockDataParser. -template -class MemoryPointerConverter : public OpConversionPattern { -public: - explicit MemoryPointerConverter(MLIRContext *context) - : OpConversionPattern(context, PatternBenefit(2)) {} - - LogicalResult - matchAndRewrite(OpTy op, typename OpTy::Adaptor, - ConversionPatternRewriter &rewriter) const override { - auto resultTy = dyn_cast(op->getResult(0).getType()); - if (!resultTy || !isa(resultTy.getElementType())) - return failure(); - - llvm::SmallDenseMap known; - FailureOr memref = - BlockDataParser::materializePointer(op->getResult(0), rewriter, known); - if (failed(memref)) - return rewriter.notifyMatchFailure( - op, "unsupported pointer expression for direct materialization"); - - rewriter.replaceOp(op, *memref); - return success(); - } -}; - -class LoadConverter : public OpConversionPattern { -private: - void propagateWasBoolToInt8Attr(Operation *srcLoadOp, Operation *dstOp, - PatternRewriter &rewriter) const; - - LogicalResult toTensorAndReplace(triton::LoadOp &op, - RankedTensorType &tensorType, Value localMem, - bool mayImplicitTransposeWithLastAxis, - const Location &loc, - ConversionPatternRewriter &rewriter) const; - - LogicalResult checkModifiedByAddPtrConverter(triton::LoadOp &op) const; - - LogicalResult - continueModifyFromAddPtrConverter(triton::LoadOp &op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const; - - void - fillTensorWithOtherForMaskScenario(Value other, Value localMem, - ArrayRef maskDim, - ConversionPatternRewriter &rewriter) const; - - LogicalResult - replaceMaskedLoadWithTensorOther(triton::LoadOp op, Value alloc, - bool mayImplicitTransposeWithLastAxis, - ConversionPatternRewriter &rewriter) const; - -public: - explicit LoadConverter(MLIRContext *context); - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::LoadOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -// tempate class's impl must in header file -template -class LoadStoreCanonicalizer : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(OpTy op, - PatternRewriter &rewriter) const override { - Value ptrVal = op.getPtr(); - - // A uniform scalar pointer can be broadcast and immediately extracted at - // a memory boundary. Fold that round-trip here so scalar pointer - // conversion does not require a tensor-of-pointer materialization. - if (auto extractOp = ptrVal.getDefiningOp()) { - if (auto splatOp = - extractOp.getTensor().getDefiningOp()) { - Value scalarPointer = splatOp.getSrc(); - auto pointerType = - dyn_cast(scalarPointer.getType()); - if (pointerType && !isa(pointerType.getPointeeType()) && - scalarPointer.getType() == ptrVal.getType()) { - rewriter.modifyOpInPlace( - op, [&]() { op->replaceUsesOfWith(ptrVal, scalarPointer); }); - return success(); - } - } - } - - auto ptrTy = dyn_cast(ptrVal.getType()); - if (!ptrTy || !isa(ptrTy.getElementType())) - return failure(); - - // ReorderBroadcast may turn - // addptr(splat(base), splat(offset)) - // into - // splat(addptr(base, offset)). - // Recover the former shape at the memory boundary so AddPtrConverter can - // keep using the real (non-synthetic) offset as its lowering anchor. - auto splatOp = ptrVal.getDefiningOp(); - if (!splatOp) - return failure(); - - auto scalarAddPtr = splatOp.getSrc().getDefiningOp(); - if (!scalarAddPtr) - return failure(); - - Value ptrSplat = rewriter.create(op.getLoc(), ptrTy, - scalarAddPtr.getPtr()); - auto offsetTy = - ptrTy.cloneWith(std::nullopt, scalarAddPtr.getOffset().getType()); - Value offsetSplat = rewriter.create( - op.getLoc(), offsetTy, scalarAddPtr.getOffset()); - Value addptr = rewriter.create(op.getLoc(), ptrTy, - ptrSplat, offsetSplat); - - rewriter.modifyOpInPlace(op, - [&]() { op->replaceUsesOfWith(ptrVal, addptr); }); - return success(); - } -}; - -class ScalarStoreCanonicalizer : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(triton::StoreOp op, - PatternRewriter &rewriter) const override; -}; - -class StoreConverter : public OpConversionPattern { -public: - explicit StoreConverter(MLIRContext *context); - - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::StoreOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class ScalarAtomicRMWCanonicalizer - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(triton::AtomicRMWOp op, - PatternRewriter &rewriter) const override; -}; - -class ScalarAtomicCASCanonicalizer - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(triton::AtomicCASOp op, - PatternRewriter &rewriter) const override; -}; - -class AtomicCASConverter : public OpConversionPattern { -public: - explicit AtomicCASConverter(MLIRContext *context) - : OpConversionPattern(context) {} - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::AtomicCASOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class AtomicRMWConverter : public OpConversionPattern { -private: - Value createAtomicBinaryOps(OpBuilder &builder, Location loc, - triton::AtomicRMWOp op, Type elementType, - Value lhs, Value rhs) const { - auto rmwOp = op.getAtomicRmwOp(); - - // it has been confirmed in AtomicRMWConverter::matchAndRewrite - // that the ptr of op is of MemRefType - Value binaryOp; - if (rmwOp == triton::RMWOp::FADD) { - binaryOp = builder.create(loc, lhs, rhs); - } else if (rmwOp == triton::RMWOp::ADD) { - binaryOp = builder.create(loc, lhs, rhs); - } else if (rmwOp == triton::RMWOp::XOR) { - binaryOp = builder.create(loc, lhs, rhs); - } else if (rmwOp == triton::RMWOp::OR) { - binaryOp = builder.create(loc, lhs, rhs); - } else if (rmwOp == triton::RMWOp::AND) { - binaryOp = builder.create(loc, lhs, rhs); - } else if (rmwOp == triton::RMWOp::MAX) { - // Max/Min only support f32/i32 for now - // Other type is not supported because of semantic.py - if (isa(elementType)) { - binaryOp = builder.create(loc, lhs, rhs); - } else { - binaryOp = builder.create(loc, lhs, rhs); - } - } else if (rmwOp == triton::RMWOp::MIN) { - if (isa(elementType)) { - binaryOp = builder.create(loc, lhs, rhs); - } else { - binaryOp = builder.create(loc, lhs, rhs); - } - } else if (rmwOp == triton::RMWOp::XCHG) { - binaryOp = rhs; - } else if (rmwOp == triton::RMWOp::UMAX) { - binaryOp = builder.create(loc, lhs, rhs); - } else if (rmwOp == triton::RMWOp::UMIN) { - binaryOp = builder.create(loc, lhs, rhs); - } else { - op.emitOpError("unsupported atomic RMW operation: "); - llvm_unreachable( - "Not implemented. Support fadd, add, max, min for now !"); - } - return binaryOp; - } - - // used when handling scalar - // to verify whether we need to handle this scalar - bool isConstantMaskTrue(Value mask) const { - if (auto denseAttr = - mask.getDefiningOp()->getAttrOfType("value")) { - auto eleType = denseAttr.getType().getElementType(); - if (isa(eleType) && - cast(eleType).getWidth() == 1) { - auto values = denseAttr.getValues(); - return values[0]; - } - } - return false; - } - - DenseSet softwareAtomicKinds = { - triton::RMWOp::AND, triton::RMWOp::OR, triton::RMWOp::XOR}; - -public: - explicit AtomicRMWConverter(MLIRContext *context); - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::AtomicRMWOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class AtomicMaxMinCanonicalizer : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(triton::AtomicRMWOp op, - PatternRewriter &rewriter) const override; -}; - -class ReinterpretCastStrideCanonicalizer - : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(memref::ReinterpretCastOp op, - PatternRewriter &rewriter) const override; - static bool hasFixableZeroStride(memref::ReinterpretCastOp op); -}; - -} // namespace LoadStoreConverter -#endif diff --git a/third_party/ascend/include/TritonToLinalg/MarkTensorKindPass.h b/third_party/ascend/include/TritonToLinalg/MarkTensorKindPass.h deleted file mode 100644 index 1c46719ea1..0000000000 --- a/third_party/ascend/include/TritonToLinalg/MarkTensorKindPass.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_CONVERSION_MARKTENSORKINDPASS_H -#define TRITON_ADAPTER_CONVERSION_MARKTENSORKINDPASS_H - -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#define GEN_PASS_DEF_MARKTENSORKIND -#include "ascend/include/TritonToLinalg/Passes.h.inc" - -namespace mlir { -namespace triton { - -std::unique_ptr> createMarkTensorKindPass(); - -enum TensorKind { NONE = -1, INPUT = 0, OUTPUT = 1, INPUT_OUTPUT = 2 }; - -} // namespace triton -} // namespace mlir - -using namespace mlir; -using namespace triton; - -class MarkTensorKindPass - : public ::impl::MarkTensorKindBase { -public: - MarkTensorKindPass() = default; - - void runOnOperation() override; -}; - -#endif // TRITON_ADAPTER_CONVERSION_MARKTENSORKINDPASS_H diff --git a/third_party/ascend/include/TritonToLinalg/MaskAnalysis.h b/third_party/ascend/include/TritonToLinalg/MaskAnalysis.h deleted file mode 100644 index cf9ca27450..0000000000 --- a/third_party/ascend/include/TritonToLinalg/MaskAnalysis.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * Copyright (c) Microsoft Corporation, Meta Platforms. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ANALYSIS_MASKANALYSIS_H -#define TRITON_ANALYSIS_MASKANALYSIS_H - -// Compatibility shim: the public MaskState API is owned by the neutral -// TritonMemoryAccess layer so Graph can use it without depending on T2L. -#include "TritonMemoryAccess/LoadStoreMaskAnalysis.h" - -#endif // TRITON_ANALYSIS_MASKANALYSIS_H diff --git a/third_party/ascend/include/TritonToLinalg/Passes.h b/third_party/ascend/include/TritonToLinalg/Passes.h deleted file mode 100644 index 6fcc73a0ac..0000000000 --- a/third_party/ascend/include/TritonToLinalg/Passes.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_TRITON_TO_LINALG_CONVERSION_PASSES_H -#define TRITON_ADAPTER_TRITON_TO_LINALG_CONVERSION_PASSES_H - -#include "DeduplicateDebugNopsPass.h" -#include "MarkTensorKindPass.h" -#include "TritonToLinalgPass.h" - -namespace mlir::triton { - -#define GEN_PASS_REGISTRATION -#include "ascend/include/TritonToLinalg/Passes.h.inc" - -} // namespace mlir::triton - -#endif // TRITON_ADAPTER_TRITON_TO_LINALG_CONVERSION_PASSES_H diff --git a/third_party/ascend/include/TritonToLinalg/Passes.td b/third_party/ascend/include/TritonToLinalg/Passes.td deleted file mode 100644 index 5d480fd8f2..0000000000 --- a/third_party/ascend/include/TritonToLinalg/Passes.td +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef TRITON_TO_LINALG_CONVERSION_PASSES -#define TRITON_TO_LINALG_CONVERSION_PASSES - -include "mlir/Pass/PassBase.td" - -def TritonToLinalg : Pass<"triton-to-linalg", "mlir::ModuleOp"> { - let summary = "Convert Triton to Linalg dialect"; - let constructor = "triton::createTritonToLinalgPass()"; - let options = [ - Option<"globalKernel", "global-kernel", - "bool", /*default*/"true", - "generate a global kernel">, - Option<"namedOps", "named-ops", - "bool", /*default*/"true", - "use linalg named ops instead of linalg.generic">, - Option<"enableNd2nzOnVector", "enable-nd2nz-on-vector", - "bool", /*default*/"false", - "enable nd2nz on vector">, - Option<"enableSelectAnalysis", "enable-select-analysis", - "bool", /*default*/"true", - "enable select analysis">, - Option<"compileOn91095", "compile-on-910-95", - "bool", /*default*/"false", - "compile on 910_95">, - Option<"compileMode", "compile-mode", - "std::string", /*default*/"\"simd_simt_template\"", - "Compilation mode: simd, simd_simt_template (or unstructured_in_simt), simt_only"> - ]; -} - -def MarkTensorKind : Pass<"mark-tensor-kind", "mlir::ModuleOp"> { - let summary = "Mark tensor kind (INPUT/OUTPUT/INPUT_OUTPUT) on Triton function arguments"; - let constructor = "triton::createMarkTensorKindPass()"; -} - -def DeduplicateDebugNops : Pass<"deduplicate-debug-nops", "mlir::ModuleOp"> { - let summary = "Deduplicate llvm.inline_asm 'nop' debug-anchor ops by source line."; - let description = [{ - Removes redundant `llvm.inline_asm "nop"` ops that share a - `(filename, line)` source location within a function. Keeps the - first occurrence in IR walk order; erases the rest. - - Opt-in via env var `LLVM_EXTRACT_DI_LOCAL_VARIABLES=1`. With the - env var unset the pass is a no-op. - }]; - let constructor = "triton::createDeduplicateDebugNopsPass()"; -} - -#endif // TRITON_TO_LINALG_CONVERSION_PASSES diff --git a/third_party/ascend/include/TritonToLinalg/StridedAxisCoalescing.h b/third_party/ascend/include/TritonToLinalg/StridedAxisCoalescing.h deleted file mode 100644 index 07e905d82d..0000000000 --- a/third_party/ascend/include/TritonToLinalg/StridedAxisCoalescing.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ASCEND_STRIDED_AXIS_COALESCING_H -#define TRITON_ASCEND_STRIDED_AXIS_COALESCING_H - -// Compatibility shim. The implementation is owned by TritonToGraph and is -// scheduled only through its original T2L compatibility slot. -#include "TritonToGraph/LegacyMemoryAccess/StridedAxisCoalescing.h" - -#endif // TRITON_ASCEND_STRIDED_AXIS_COALESCING_H diff --git a/third_party/ascend/include/TritonToLinalg/StridedLoadStoreRewrite.h b/third_party/ascend/include/TritonToLinalg/StridedLoadStoreRewrite.h deleted file mode 100644 index 3dbef86850..0000000000 --- a/third_party/ascend/include/TritonToLinalg/StridedLoadStoreRewrite.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ASCEND_STRIDED_LOAD_STORE_REWRITE_H -#define TRITON_ASCEND_STRIDED_LOAD_STORE_REWRITE_H - -// Compatibility shim. The greedy pattern stage is owned by TritonToGraph and -// invoked only from the original T2L compatibility slot. -#include "TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h" - -#endif // TRITON_ASCEND_STRIDED_LOAD_STORE_REWRITE_H diff --git a/third_party/ascend/include/TritonToLinalg/TileChunkCoalescing.h b/third_party/ascend/include/TritonToLinalg/TileChunkCoalescing.h deleted file mode 100644 index 952052ecfa..0000000000 --- a/third_party/ascend/include/TritonToLinalg/TileChunkCoalescing.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ASCEND_TILE_CHUNK_COALESCING_H -#define TRITON_ASCEND_TILE_CHUNK_COALESCING_H - -#include "TritonToGraph/LegacyMemoryAccess/ChunkCoalescing.h" - -namespace TileChunkCoalescing { - -// Compatibility shim for clients of the pre-895 API. The implementation is -// the 895 ChunkCoalescing logic owned by TritonToGraph. -inline void rewriteTileChunkCoalesce(mlir::ModuleOp moduleOp) { - ChunkCoalescing::rewriteChunkCoalesce(moduleOp); -} - -} // namespace TileChunkCoalescing - -#endif // TRITON_ASCEND_TILE_CHUNK_COALESCING_H diff --git a/third_party/ascend/include/TritonToLinalg/TritonOpConverter.h b/third_party/ascend/include/TritonToLinalg/TritonOpConverter.h deleted file mode 100644 index 9f0fbf35b2..0000000000 --- a/third_party/ascend/include/TritonToLinalg/TritonOpConverter.h +++ /dev/null @@ -1,923 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_TRITONOPCONVERTER_H -#define TRITON_ADAPTER_TRITONOPCONVERTER_H - -#include "TritonToLinalg/TritonToLinalgPass.h" -#include "ascend/include/Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "ascend/include/TritonToLinalg/BlockPtrAnalysis.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Debug.h" - -#define DEBUG_TYPE "triton-to-linalg" - -namespace TTOpConverters { -using namespace mlir; -using namespace triton; - -static constexpr unsigned kFuncNameCap = 128; - -/* -Convert `tt.precise_div` operation to `arith.divf` operation. -tensor_x / tensor_y - -```ttir - %11 = tt.precise_divf %7, %10 : tensor<100xf32> -``` - -converts to: - -```mlir - %11 = arith.divf %7, %10 : tensor<100xf32> -``` -*/ -struct PreciseDivConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::PreciseDivFOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -/* -Convert `tt.fp_to_fp` operation with RTNE (default) rounding mode to -`arith.truncf` or `arith.extf` operation. - -For default RTNE rounding: -- downcast: tt.fp_to_fp -> arith.truncf -- upcast: tt.fp_to_fp -> arith.extf -- same-bitwidth conversions between distinct floating-point formats: - tt.fp_to_fp -> arith.extf (to f32) -> arith.truncf - -Only an operation whose complete input and result MLIR types are identical can -be eliminated as an identity conversion. - -Note: Non-RTNE rounding modes (e.g., RTZ) are handled by TritonToHFusion pass. -*/ -struct FpToFpCanonicalizer : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(triton::FpToFpOp op, - PatternRewriter &rewriter) const override; -}; - -class SelectCanonicalizer : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(arith::SelectOp op, - PatternRewriter &rewriter) const override; -}; - -/* - * Move tt.bitcast to a previous location if tt.bitcast is not directly applied - * on function arguments - */ -class BitcastCanonicalizer : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(triton::BitcastOp bitcastOp, - PatternRewriter &rewriter) const override; -}; - -template -class ScalarMathCanonicalizer : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MathOp op, - PatternRewriter &rewriter) const override { - if (op->getNumResults() != 1) { - return rewriter.notifyMatchFailure( - op, "ScalarMathCanonicalizer expects single scalar output."); - } - if (!op->getResult(0).getType().isIntOrIndexOrFloat()) { - return rewriter.notifyMatchFailure( - op, "ScalarMathCanonicalizer handles scalar load scene."); - } - if (auto linalgOp = op->template getParentOfType()) { - return rewriter.notifyMatchFailure( - op, "ScalarMathCanonicalizer handles op not within tt.reduce."); - } - if (auto linalgOp = op->template getParentOfType()) { - return rewriter.notifyMatchFailure( - op, "ScalarMathCanonicalizer handles op not within tt.scan."); - } - if (auto linalgOp = - op->template getParentOfType()) { - return rewriter.notifyMatchFailure( - op, - "ScalarMathCanonicalizer handles op not within tt.map_elementwise."); - } - auto loc = op.getLoc(); - llvm::SmallVector inputs; - for (auto input : op->getOperands()) { - auto blkTy = RankedTensorType::get({(int64_t)1}, input.getType()); - auto inputSplat = rewriter.create(loc, blkTy, input); - inputs.push_back(inputSplat.getResult()); - } - auto blkOp = rewriter.create(loc, inputs); - Value offset = - rewriter.create(loc, rewriter.getIndexAttr(0)); - auto extractOp = - rewriter.create(loc, blkOp.getResult(), offset); - rewriter.replaceOp(op, extractOp); - return success(); - } -}; - -/* - * Rewrite tt.make_tensor_ptr with non-contiguous order to - * tt.make_tensor_ptr + tt.load + tt.trans. - */ -class MakeTensorPtrCanonicalizer - : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(triton::MakeTensorPtrOp op, - PatternRewriter &rewriter) const override; -}; - -class ReduceSingleCanonicalizer : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(triton::ReduceOp reduceOp, - PatternRewriter &rewriter) const override; -}; - -class DenseConstantConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(arith::ConstantOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class MakeRangeConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::MakeRangeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class SplatConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::SplatOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class UnsplatConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::UnsplatOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class ReshapeConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::ReshapeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class ExpandDimsConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::ExpandDimsOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class ClampFConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ClampFOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class BroadcastConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::BroadcastOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -template -class ReductionOpBaseConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const final { - auto sourceType = - cast(adaptor.getOperands().front().getType()); - assert(sourceType.hasRank() && "Expected input is ranked"); - - int64_t axis = op.getAxis(); - assert(axis >= 0 && axis < sourceType.getRank() && - "Expected reduction axis is within operand's rank"); - - auto realReductionOps = this->getRealReductionOps(op); - if (realReductionOps.size() == 1) { - return this->convertToTargetOp(op, adaptor, rewriter); - } - return this->convertToTargetOpExtended(op, adaptor, rewriter); - } - -protected: - llvm::SmallVector getReductionOps(OpTy reductionOp) const { - auto reductionBody = reductionOp.getBody(); - return llvm::map_to_vector(reductionBody->without_terminator(), - [](Operation &op) { return &op; }); - } - - llvm::SmallVector getRealReductionOps(OpTy reductionOp) const { - auto *body = reductionOp.getBody(); - auto *terminator = body->getTerminator(); - - // Compute the backward slice from yielded values: only ops that - // actually contribute to the computation of the reduction result - // are considered. - llvm::DenseSet liveOps; - llvm::SmallVector worklist(terminator->getOperands()); - while (!worklist.empty()) { - Value val = worklist.pop_back_val(); - if (auto *defOp = val.getDefiningOp()) { - if (defOp->getBlock() == body && liveOps.insert(defOp).second) { - for (auto operand : defOp->getOperands()) - worklist.push_back(operand); - } - } - } - - // Count only live ops, excluding type conversions that serve as - // precision promotion or demotion (e.g. bf16 -> f32 -> bf16). - llvm::SmallVector realOps; - for (Operation &bodyOp : body->without_terminator()) { - if (!liveOps.contains(&bodyOp)) - continue; - if (isa(&bodyOp)) - continue; - realOps.push_back(&bodyOp); - } - return realOps; - } - - arith::ConstantOp - getMultiOpReductionBaseConstOp(ConversionPatternRewriter &rewriter, OpTy op, - Location loc, Type constantType) const { - // for multiop reduce of 1 element result is defined as exactly this element - // for multiop reduce of tensor with N elements default value - // is not involved in result computation - auto reductionOps = this->getReductionOps(op); - assert(reductionOps.size() == 1); - auto reductionOp = reductionOps.front(); - - assert(constantType.isIntOrFloat()); - - if (constantType.isInteger()) { - return rewriter.create( - loc, constantType, rewriter.getIntegerAttr(constantType, 0)); - } - return rewriter.create( - loc, constantType, rewriter.getFloatAttr(constantType, 0.f)); - } - - arith::ConstantOp getReductionBaseConstOp(ConversionPatternRewriter &rewriter, - Operation *reductionOp, - Type constantType) const { - const int64_t bitWidth = constantType.getIntOrFloatBitWidth(); - - auto attr = - llvm::TypeSwitch(reductionOp) - .Case([&](arith::AddFOp) { - return rewriter.getFloatAttr(constantType, 0.f); - }) - .Case([&](arith::AddIOp) { - return rewriter.getIntegerAttr(constantType, 0); - }) - .Case([&](arith::MulFOp) { - return rewriter.getFloatAttr(constantType, 1.f); - }) - .Case([&](arith::MulIOp) { - return rewriter.getIntegerAttr(constantType, 1); - }) - .template Case([&](auto) { - return rewriter.getFloatAttr( - constantType, -std::numeric_limits::infinity()); - }) - .template Case([&](auto) { - return rewriter.getFloatAttr( - constantType, std::numeric_limits::infinity()); - }) - .Case([&](arith::MinSIOp) { - return rewriter.getIntegerAttr(constantType, - llvm::maxIntN(bitWidth)); - }) - .Case([&](arith::MinUIOp) { - return rewriter.getIntegerAttr(constantType, - llvm::maxUIntN(bitWidth)); - }) - .Case([&](arith::MaxSIOp) { - return rewriter.getIntegerAttr(constantType, - llvm::minIntN(bitWidth)); - }) - .Case([&](arith::MaxUIOp) { - return rewriter.getIntegerAttr(constantType, 0); - }) - .Case([&](arith::OrIOp) { - return rewriter.getIntegerAttr(constantType, 0); - }) - .Case([&](arith::AndIOp) { - return rewriter.getIntegerAttr(constantType, 1); - }) - .Case([&](arith::XOrIOp) { - return rewriter.getIntegerAttr(constantType, 0); - }) - .Default([](Operation *op) { - op->dump(); - llvm_unreachable("Reduction op not supported yet"); - return nullptr; - }); - - return rewriter.create(reductionOp->getLoc(), - constantType, attr); - } - - bool requiresF32Conversion(const Type elemType, - Operation *reductionOp) const { - return isa(elemType) && - elemType.getIntOrFloatBitWidth() < - Float32Type::get(elemType.getContext()) - .getIntOrFloatBitWidth() && - (isa(reductionOp) || isa(reductionOp)); - } - - Value getReductionElement(Value lhs, Value rhs, const Location loc, - Operation *reductionOp, OpBuilder &b, - const bool convertLhsToF32Precision) const { - return llvm::TypeSwitch(reductionOp) - .template Case([&](auto reductionOp) { - if (convertLhsToF32Precision) { - lhs = b.create(loc, Float32Type::get(b.getContext()), - lhs); - } - return b.create(loc, lhs, rhs); - }) - .template Case([&](auto reductionOp) { - return b.create(loc, lhs, rhs); - }) - .Default([](Operation *op) { - op->dump(); - llvm_unreachable("Reduction op not yet supported"); - return nullptr; - }); - } - - virtual bool isReductionOpSupported(Operation *reductionOp) const = 0; - - virtual LogicalResult - convertToTargetOp(OpTy op, typename OpTy::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const = 0; - - virtual LogicalResult - convertToTargetOpExtended(OpTy op, typename OpTy::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const = 0; -}; - -class ReduceConverter : public ReductionOpBaseConverter { -public: - explicit ReduceConverter(MLIRContext *context) - : ReductionOpBaseConverter(context) {} - - using ReductionOpBaseConverter::ReductionOpBaseConverter; - -protected: - bool isReductionOpSupported(Operation *reductionOp) const override; - - static bool isMultiReductionOpSupported(Operation *reductionOp); - - Value cloneReduceOps(OpBuilder &builder, Value in, Value out, Value opIns, - Value opOuts, triton::ReduceOp op) const; - - void - checkIsNotCallOp(const llvm::SmallVector &reductionOps) const; - - bool isSCFOpReduce(const llvm::SmallVector &reductionOps) const; - - bool - isMultiOpReduce(const llvm::SmallVector &reductionOps) const; - - Value computeReduceResultWithCompileFlag( - OpBuilder &opBuilder, Location loc, Value lhs, Value rhs, Value source, - Value initTensor, triton::ReduceOp reductionOp, - bool compileOn91095Flag = false) const; - - LogicalResult - convertToTargetOp(triton::ReduceOp op, - typename triton::ReduceOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - - LogicalResult - convertToTargetOpExtended(triton::ReduceOp op, - typename triton::ReduceOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class ScanConverter : public ReductionOpBaseConverter { -public: - explicit ScanConverter(MLIRContext *context) - : ReductionOpBaseConverter(context) {} - - using ReductionOpBaseConverter::ReductionOpBaseConverter; - -protected: - bool isReductionOpSupported(Operation *reductionOp) const override; - - LogicalResult - convertToTargetOp(triton::ScanOp op, typename triton::ScanOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - - LogicalResult - convertToTargetOpExtended(triton::ScanOp op, - typename triton::ScanOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -/* - * Decompose tt.map_elementwise region body into tensor-level Named Ops. - * Each scalar op (arith, math, scf.if, etc.) is promoted to its tensor - * counterpart, producing a chain of independent Named Ops. - */ -class MapElementwiseDecomposeConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::MapElementwiseOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - -private: - // Promote a single scalar op to tensor-level and record the result in - // valueMap. Returns the tensor result values (one per op result). - SmallVector promoteOp(Operation *op, - llvm::DenseMap &valueMap, - OpBuilder &builder, Location loc, - ArrayRef tensorShape) const; - - // Promote ops in a region body (excluding terminator), return the yielded - // tensor values from the terminator. - SmallVector promoteRegionBody(Region ®ion, - llvm::DenseMap &valueMap, - OpBuilder &builder, Location loc, - ArrayRef tensorShape) const; -}; - -class ExternElementwiseClOpConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ExternElementwiseOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class UnrealizedCastConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(UnrealizedConversionCastOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class JoinConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::JoinOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class SplitConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::SplitOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class CatConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::CatOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class GatherConverter : public OpConversionPattern { -private: - static constexpr llvm::StringRef gatherFuncNameBase = "triton_gather"; - -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::GatherOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -// These predicates select only scalar !tt.ptr transports. A -// tensor<...x!tt.ptr> follows the separate tensor-pointer lowering. -bool hasScalarPointerResult(scf::IfOp op); -bool isScalarPointerSelect(arith::SelectOp op); - -// Marks an scf.if temporarily rebuilt by IfConverter. Its scalar-pointer -// results are represented as complete i64 addresses, so only its own yields -// require the matching pointer-to-address conversion. -inline constexpr llvm::StringLiteral kScalarPointerCarrierBoundaryAttr = - "ScalarPointerCarrierBoundary"; - -// Rebuild an scf.if with scalar-pointer results so the boundary carries -// complete i64 addresses and reconstructs memrefs only after the join. -// The original branch regions are moved into the new operation, preserving -// side effects and allowing the conversion driver to rewrite each scf.yield -// operand in place. -// -// Example: -// %base = scf.if %cond -> !tt.ptr { -// scf.yield %lhs : !tt.ptr -// } else { -// scf.yield %rhs : !tt.ptr -// } -// %ptr = tt.make_tensor_ptr %base, ... -// becomes an scf.if returning i64 plus one hivm.pointer_cast after the if. -class IfConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(scf::IfOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -// Convert a scalar-pointer select into a select over complete integer addresses -// and reconstruct one memref after the selection. This handles both BlockPtr -// bases and ordinary scalar pointers without asking the backend to merge two -// memory objects. -// -// Example: -// %base = arith.select %cond, %lhs, %rhs : !tt.ptr -// %ptr = tt.make_tensor_ptr %base, ... -// becomes: -// %lhs_addr = memref.extract_aligned_pointer_as_index %lhs -// %rhs_addr = memref.extract_aligned_pointer_as_index %rhs -// %selected_addr = arith.select %cond, %lhs_addr, %rhs_addr : i64 -// %base = hivm.pointer_cast %selected_addr : i64 to memref -class PointerSelectConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(arith::SelectOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -// Convert the yields of an IfConverter-created scf.if to its carrier result -// types. In particular, a yielded scalar pointer becomes its complete i64 -// address. Yields belonging to ordinary ifs or loops are intentionally left to -// their owning conversions. -class YieldConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(scf::YieldOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -template || - std::is_same_v>> -class LoopConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(LoopOpTy op, - typename OpConversionPattern::OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - // CFO-expanded descriptor loops already carry pointer-free policy values - // and remain structurally unchanged. This legacy BlockData rewrite is only - // valid for explicitly marked legacy loops. - if (!op->hasAttr("UnhandledLoopOp")) - return failure(); - llvm::SmallDenseMap known; - - rewriter.modifyOpInPlace(op, [&]() { op->removeAttr("UnhandledLoopOp"); }); - return BlockDataParser::rewriteLoopOp(op, rewriter, known, {}); - } -}; - -class AdvanceConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::AdvanceOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class MakeTensorPtrConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - explicit MakeTensorPtrConverter(MLIRContext *context) - : OpConversionPattern(context) {} - - LogicalResult - matchAndRewrite(triton::MakeTensorPtrOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class TransposeConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::TransOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class BitcastConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::BitcastOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class TritonMulhiuiConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::MulhiUIOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class TritonPreciseSqrtConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::PreciseSqrtOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class DeviceAssertConverter : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - -private: - static constexpr llvm::StringRef printFuncNameBase = "triton_assert"; - static constexpr llvm::StringRef msgAttrName = "msg"; - -public: - LogicalResult - matchAndRewrite(triton::AssertOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class DevicePrintConverter : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - -private: - static constexpr llvm::StringRef printFuncNameBase = "triton_print"; - static constexpr llvm::StringRef prefixAttrName = "prefix"; - static constexpr llvm::StringRef hexAttrName = "hex"; - -public: - LogicalResult - matchAndRewrite(triton::PrintOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -struct MatmulConverter : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::DotOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -struct DotConverter : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::ascend::DotOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -struct FlipOpConverter : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::ascend::FlipOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - - static constexpr StringRef baseFuncName = "triton_flip"; -}; - -struct SortOpConverter : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::ascend::SortOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -struct DotScaledConverter : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::DotScaledOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class PtrToIntConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::PtrToIntOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class IntToPtrConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::IntToPtrOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class IndexPutConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ascend::IndexPutOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - -private: - static constexpr llvm::StringRef funcNameBase = "triton_index_put"; -}; - -class GatherOutToUbConverter - : public OpConversionPattern { -public: - using OpConversionPattern< - triton::ascend::GatherOutToUbOp>::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ascend::GatherOutToUbOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - -private: - static constexpr llvm::StringRef funcNameBase = "triton__gather_out_to_ub"; -}; - -class ScatterUbToOutConverter - : public OpConversionPattern { -public: - using OpConversionPattern< - triton::ascend::ScatterUbToOutOp>::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ascend::ScatterUbToOutOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - -private: - static constexpr llvm::StringRef funcNameBase = "triton_scatter_ub_to_out"; -}; - -class IndirectLoadConverter - : public OpConversionPattern { -public: - using OpConversionPattern< - triton::ascend::IndirectLoadOp>::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ascend::IndirectLoadOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - -private: - static constexpr llvm::StringRef funcNameBase = "triton_indirect_load"; -}; - -class StrideLoadConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ascend::StrideLoadOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - -private: - static constexpr llvm::StringRef funcNameBase = "triton_stride_load"; -}; - -class StrideStoreConverter - : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ascend::StrideStoreOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - -private: - static constexpr llvm::StringRef funcNameBase = "triton_stride_store"; -}; - -class IndirectStoreConverter - : public OpConversionPattern { -public: - using OpConversionPattern< - triton::ascend::IndirectStoreOp>::OpConversionPattern; - LogicalResult - matchAndRewrite(triton::ascend::IndirectStoreOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; - -private: - static constexpr llvm::StringRef funcNameBase = "triton_indirect_store"; -}; - -class IndexSelectSimdConverter - : public OpConversionPattern { -public: - explicit IndexSelectSimdConverter(MLIRContext *context); - using OpConversionPattern< - triton::ascend::IndexSelectSimdOp>::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::ascend::IndexSelectSimdOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -class HistogramConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(triton::HistogramOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override; -}; - -} // end of namespace TTOpConverters - -#endif diff --git a/third_party/ascend/include/TritonToLinalg/TritonToLinalgPass.h b/third_party/ascend/include/TritonToLinalg/TritonToLinalgPass.h deleted file mode 100644 index c5440e2da2..0000000000 --- a/third_party/ascend/include/TritonToLinalg/TritonToLinalgPass.h +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ADAPTER_CONVERSION_TRITONTOLINALG_H -#define TRITON_ADAPTER_CONVERSION_TRITONTOLINALG_H - -#include "ascend/include/Utils/Utils.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include - -#define GEN_PASS_CLASSES -#include "ascend/include/TritonToLinalg/Passes.h.inc" - -extern int nd2nzFlag; -extern bool compileOn91095Flag; -extern bool existDotFlag; -extern mlir::triton::ascend::CompileMode compileModeFlag; - -namespace mlir { -namespace triton { - -std::unique_ptr> createTritonToLinalgPass(); - -std::unique_ptr> -createTritonToLinalgPass(bool, bool, bool, bool, bool, - const std::string &compileMode = "simd_simt_template"); - -} // namespace triton -} // namespace mlir - -using namespace mlir; -using namespace triton; -const std::string globalKernelAttr = "global_kernel"; -const std::string kernelMixModeName = "mix_mode"; -const std::string kernelParallelModeName = "parallel_mode"; - -class TritonTypeConverter : public mlir::TypeConverter { -public: - explicit TritonTypeConverter(); -}; - -class TritonToLinalgPass : public TritonToLinalgBase { - - static auto constexpr LAUNCH_GRID_RANK = getMaxEnumValForProgramIDDim() + 1; - static unsigned int constexpr TRITON_PROGRAM_INFO_ARG_COUNT = - LAUNCH_GRID_RANK * 2; - -private: - // grid构造 num_programs 3维, program_id 3维 - // remember 'xxxOp' is usually a Pointer, so that we can change target memory - // without giving a reference argument - void addProgramInfo(triton::FuncOp func, bool globalKernel); - - void convertTTFunc(triton::FuncOp func, const bool existDot, - const bool existSIMTOp); - - LogicalResult convertMultipleBlockControlFlow(Operation *funcOp, - OpBuilder &builder); - // 处理嵌套的if/else - scf::IfOp transformNestedIfElse(Operation &nestedBranch, OpBuilder &builder); - - void addDynamicLegal(ConversionTarget &target, - TritonTypeConverter &tritonTypeConverter); - - void - populateTritonToLinalgCanonicalizationPatterns(RewritePatternSet &patterns); - - void populateTritonToLinalgConversionPatterns(TypeConverter &typeConverter, - RewritePatternSet &patterns, - unsigned int launchGridRank); - - LogicalResult processDescriptorOperations(ModuleOp moduleOp); - LogicalResult processPtrBroadcastOperations(ModuleOp moduleOp); - LogicalResult processImplicitPermuteOperations(ModuleOp moduleOp); - LogicalResult processStridedLoadStoreRewriteOperations(ModuleOp moduleOp); - LogicalResult processLegalStrideOperations(ModuleOp moduleOp); - -public: - TritonToLinalgPass() = default; - - TritonToLinalgPass(bool globalKernel, bool namedOps, bool enableNd2nzOnVector, - bool enableSelectAnalysis, bool compileOn91095, - const std::string &compileMode = "simd_simt_template") { - this->globalKernel = globalKernel; - this->namedOps = namedOps; - this->enableNd2nzOnVector = enableNd2nzOnVector; - this->enableSelectAnalysis = enableSelectAnalysis; - this->compileOn91095 = compileOn91095; - this->compileMode = compileMode; - }; - - void getDependentDialects(DialectRegistry ®istry) const override; - - void runOnOperation() override; -}; - -#endif // TRITON_ADAPTER_CONVERSION_TRITONTOLINALG_H diff --git a/third_party/ascend/include/TritonToLinalg/UseAnalysis.h b/third_party/ascend/include/TritonToLinalg/UseAnalysis.h deleted file mode 100644 index c834f539a9..0000000000 --- a/third_party/ascend/include/TritonToLinalg/UseAnalysis.h +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ANALYSIS_USEANALYSIS_H -#define TRITON_ANALYSIS_USEANALYSIS_H - -#include "mlir/Analysis/DataFlow/SparseAnalysis.h" - -#include "ascend/include/Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "mlir/Transforms/DialectConversion.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -namespace mlir { -namespace triton { - -enum class UseType { - Undefined, // Initial state - DataUse, // value used for tensor computation only - MetaUse, // value used for metadata only - MixUse // value used for both tensor computation and metadata -}; - -struct UseInfo : public dataflow::AbstractSparseLattice { - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(UseInfo) - using AbstractSparseLattice::AbstractSparseLattice; - - // Lattice state transfer function - ChangeResult meetUseType(const UseType &other) { - if (other == UseType::Undefined) { - return ChangeResult::NoChange; - } - - switch (type) { - case UseType::Undefined: - type = other; - return ChangeResult::Change; - case UseType::DataUse: - case UseType::MetaUse: - if (type == other) { - return ChangeResult::NoChange; - } else { - type = UseType::MixUse; - return ChangeResult::Change; - } - case UseType::MixUse: - return ChangeResult::NoChange; - default: - llvm_unreachable("bad type"); - } - } - - ChangeResult meet(const AbstractSparseLattice &other) override { - auto rhs = reinterpret_cast(&other); - return meetUseType(rhs->type); - } - - void print(raw_ostream &os) const override { - switch (type) { - case UseType::DataUse: - os << "DataUse"; - break; - case UseType::MetaUse: - os << "MetaUse"; - break; - case UseType::MixUse: - os << "MixUse"; - break; - default: - os << "Undefined"; - } - } - - UseType type = UseType::Undefined; -}; - -class UseAnalysis : public dataflow::SparseBackwardDataFlowAnalysis { -public: - using SparseBackwardDataFlowAnalysis::SparseBackwardDataFlowAnalysis; - -#if LLVM_VERSION_MAJOR >= 20 - LogicalResult visitOperation(Operation *op, ArrayRef operands, - ArrayRef results) override; -#else - void visitOperation(Operation *op, ArrayRef operands, - ArrayRef results) override; -#endif - - void visitBranchOperand(OpOperand &operand) override { return; } - - void visitCallOperand(OpOperand &operand) override { return; } - - void setToExitState(UseInfo *lattice) override { - lattice->type = UseType::Undefined; - } - -private: - void propagateUse(UseInfo *lattice, const UseType &type) { - auto changed = lattice->meetUseType(type); - propagateIfChanged(lattice, changed); - } - - void propagateResults(UseInfo *lattice, ArrayRef results) { - auto changed = ChangeResult::NoChange; - for (auto result : results) { - changed |= lattice->meet(*result); - } - propagateIfChanged(lattice, changed); - } -}; - -class MetaUseEraser : public RewritePattern { -public: - MetaUseEraser(MLIRContext *context); - - LogicalResult matchAndRewrite(Operation *op, - PatternRewriter &rewriter) const final; -}; - -LogicalResult runUseAnalysis(triton::FuncOp &funcOp); - -} // namespace triton - -} // namespace mlir - -#endif // TRITON_CONVERSION_TRITONTOAFFINE_TRITONUSEANALYSIS_H diff --git a/third_party/ascend/lib/CMakeLists.txt b/third_party/ascend/lib/CMakeLists.txt index 63c3e2183f..f17cb15c6d 100644 --- a/third_party/ascend/lib/CMakeLists.txt +++ b/third_party/ascend/lib/CMakeLists.txt @@ -4,7 +4,6 @@ add_subdirectory(TritonMemoryAccess) add_subdirectory(Utils) add_subdirectory(TritonToStructured) add_subdirectory(TritonToGraph) -add_subdirectory(TritonToLinalg) add_subdirectory(TritonToAnnotation) add_subdirectory(TritonToHFusion) add_subdirectory(TritonToHIVM) @@ -22,7 +21,6 @@ if(TRITON_ENABLE_COVERAGE_HITEST) TritonToAnnotation TritonToHFusion TritonToHIVM - TritonToLinalg TritonControlFlowOpt TritonToLLVM TritonToStructured diff --git a/third_party/ascend/lib/DiscreteMaskAccessConversion/CMakeLists.txt b/third_party/ascend/lib/DiscreteMaskAccessConversion/CMakeLists.txt index 3452c257bd..9e45c0ed29 100644 --- a/third_party/ascend/lib/DiscreteMaskAccessConversion/CMakeLists.txt +++ b/third_party/ascend/lib/DiscreteMaskAccessConversion/CMakeLists.txt @@ -12,7 +12,6 @@ add_triton_library(DiscreteMaskAccessConversion MLIRTransforms MLIRSupport TritonIR - TritonToLinalg + TritonToLinalgIncubated TritonMemoryAccessAnalysis - TritonToLinalg ) diff --git a/third_party/ascend/lib/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.cpp b/third_party/ascend/lib/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.cpp index 156b6e90cc..33655cfac6 100644 --- a/third_party/ascend/lib/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.cpp +++ b/third_party/ascend/lib/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.cpp @@ -24,8 +24,8 @@ #include "ascend/include/DiscreteMaskAccessConversion/Passes.h" #include "TritonControlFlowOpt/ControlFlowRewrite.h" -#include "ascend/include/TritonToLinalg/LoadStoreConverter.h" -#include "ascend/include/TritonToLinalg/MaskAnalysis.h" +#include "incubated/Conversion/TritonToLinalgIncubated/LoadStoreConverter.h" +#include "incubated/Conversion/TritonToLinalgIncubated/MaskAnalysis.h" #include "ascend/include/TritonToStructured/MemOpConverter.h" #include "ascend/include/TritonToUnstructure/OffsetAnalysis.h" #include "bishengir/Dialect/HIVM/IR/HIVM.h" diff --git a/third_party/ascend/lib/TritonToLinalg/ArgMinMaxConverter.cpp b/third_party/ascend/lib/TritonToLinalg/ArgMinMaxConverter.cpp deleted file mode 100644 index 4362e11818..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/ArgMinMaxConverter.cpp +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/ArgMinMaxConverter.h" -#include -#include - -namespace TTOpConverters { -using namespace mlir; -using namespace triton; - -// ArgMinConverter functions -LogicalResult ArgMinConverter::matchComparisonResult( - Value currValue, Value currIndex, Value reduceValue, Value reduceIndex, - mlir::Block::iterator &it, Value &comparisonResult) { - LLVM_DEBUG(llvm::dbgs() << "Matching: " << *it << "\n"); - - auto cmpOp = dyn_cast(*it); - auto cmpIOp = dyn_cast(*it++); - if (!cmpOp && !cmpIOp) - return failure(); - - if (cmpOp) { - if (cmpOp.getPredicate() != arith::CmpFPredicate::OLT || - currValue != cmpOp.getLhs() || reduceValue != cmpOp.getRhs()) { - return failure(); - } - comparisonResult = cmpOp; - } - - if (cmpIOp) { - if ((cmpIOp.getPredicate() != arith::CmpIPredicate::slt && - cmpIOp.getPredicate() != arith::CmpIPredicate::ult) || - currValue != cmpIOp.getLhs() || reduceValue != cmpIOp.getRhs()) { - return failure(); - } - comparisonResult = cmpIOp; - } - - return success(); -} - -float ArgMinConverter::getBaseReductionValue() { - return std::numeric_limits::infinity(); -} - -int8_t ArgMinConverter::getBaseReductionIntValue() { - return std::numeric_limits::max(); -} -uint8_t ArgMinConverter::getBaseReductionUIntValue() { - return std::numeric_limits::max(); -} - -// ArgMaxConverter functions -LogicalResult ArgMaxConverter::matchComparisonResult( - Value currValue, Value currIndex, Value reduceValue, Value reduceIndex, - mlir::Block::iterator &it, Value &comparisonResult) { - auto cmpOp = dyn_cast(*it); - auto cmpIOp = dyn_cast(*it++); - if (!cmpOp && !cmpIOp) - return failure(); - - if (cmpOp) { - if (cmpOp.getPredicate() != arith::CmpFPredicate::OGT || - currValue != cmpOp.getLhs() || reduceValue != cmpOp.getRhs()) { - return failure(); - } - comparisonResult = cmpOp; - } - - if (cmpIOp) { - if ((cmpIOp.getPredicate() != arith::CmpIPredicate::sgt && - cmpIOp.getPredicate() != arith::CmpIPredicate::ugt) || - currValue != cmpIOp.getLhs() || reduceValue != cmpIOp.getRhs()) { - return failure(); - } - comparisonResult = cmpIOp; - } - - return success(); -} - -float ArgMaxConverter::getBaseReductionValue() { - return -std::numeric_limits::infinity(); -} - -int8_t ArgMaxConverter::getBaseReductionIntValue() { - return std::numeric_limits::min(); -} -uint8_t ArgMaxConverter::getBaseReductionUIntValue() { - return std::numeric_limits::min(); -} - -} // namespace TTOpConverters - -namespace { -struct FoldOneHotGatherAfterReduceWithIndex - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(linalg::ReduceOp op, - PatternRewriter &rewriter) const final { - if (op->getAttrOfType("reduce_mode")) - return failure(); - if (op.getNumResults() != 1 || op.getInputs().size() != 1) - return failure(); - if (!isAddFReduction(op)) - return failure(); - - auto mulOp = op.getInputs()[0].getDefiningOp(); - if (!mulOp) - return failure(); - - Value logits; - Value maskAsFloat; - if (mulOp.getRhs().getDefiningOp()) { - logits = mulOp.getLhs(); - maskAsFloat = mulOp.getRhs(); - } else if (mulOp.getLhs().getDefiningOp()) { - logits = mulOp.getRhs(); - maskAsFloat = mulOp.getLhs(); - } else { - return failure(); - } - - auto castOp = maskAsFloat.getDefiningOp(); - if (!castOp) - return failure(); - - auto cmpOp = castOp.getIn().getDefiningOp(); - if (!cmpOp || cmpOp.getPredicate() != arith::CmpIPredicate::eq) - return failure(); - - Value lhsBase = stripShapeAndBroadcast(cmpOp.getLhs()); - Value rhsBase = stripShapeAndBroadcast(cmpOp.getRhs()); - - linalg::ReduceOp reduceWithIndex = getMaxWithIndexFromIndexValue(lhsBase); - Value arangeBase = rhsBase; - if (!reduceWithIndex) { - reduceWithIndex = getMaxWithIndexFromIndexValue(rhsBase); - arangeBase = lhsBase; - } - if (!reduceWithIndex) - return failure(); - - if (reduceWithIndex.getNumResults() != 2 || - reduceWithIndex.getInputs().size() < 2) - return failure(); - - Value reduceIndexBase = - stripShapeAndBroadcast(reduceWithIndex.getInputs()[1]); - if (arangeBase != reduceIndexBase) - return failure(); - if (op.getDimensionsAttr() != reduceWithIndex.getDimensionsAttr()) - return failure(); - - Value maxInput = reduceWithIndex.getInputs()[0]; - auto selectOp = maxInput.getDefiningOp(); - if (selectOp && selectOp.getTrueValue() != logits && - selectOp.getFalseValue() != logits) - return failure(); - if (!selectOp && maxInput != logits) - return failure(); - - if (op.getResult(0).getType() != reduceWithIndex.getResult(0).getType()) - return failure(); - - rewriter.replaceOp(op, reduceWithIndex.getResult(0)); - return success(); - } - -private: - static bool isAddFReduction(linalg::ReduceOp op) { - Region ®ion = op.getRegion(); - if (!region.hasOneBlock()) - return false; - Block &block = region.front(); - auto yieldOp = dyn_cast(block.getTerminator()); - if (!yieldOp || yieldOp.getNumOperands() != 1) - return false; - return yieldOp.getOperand(0).getDefiningOp() != nullptr; - } - - static Value stripShapeAndBroadcast(Value value) { - while (Operation *def = value.getDefiningOp()) { - if (isa(def) || - def->getName().getStringRef() == "linalg.broadcast") { - if (def->getNumOperands() == 0) - break; - value = def->getOperand(0); - continue; - } - break; - } - return value; - } - - static linalg::ReduceOp getMaxWithIndexFromIndexValue(Value value) { - auto result = dyn_cast(value); - if (!result || result.getResultNumber() != 1) - return nullptr; - - auto reduceOp = dyn_cast(result.getOwner()); - if (!reduceOp) - return nullptr; - - auto reduceMode = reduceOp->getAttrOfType("reduce_mode"); - if (!reduceMode || reduceMode != "max_with_index") - return nullptr; - return reduceOp; - } -}; -} // namespace - -void TTOpConverters::populatePostConversionCanonicalizationPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} diff --git a/third_party/ascend/lib/TritonToLinalg/BlockPtrAnalysis.cpp b/third_party/ascend/lib/TritonToLinalg/BlockPtrAnalysis.cpp deleted file mode 100644 index 6f5b5c7e67..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/BlockPtrAnalysis.cpp +++ /dev/null @@ -1,3291 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/BlockPtrAnalysis.h" -#include "ascend/include/TritonControlFlowOpt/ControlFlowRewrite.h" -#include "ascend/include/TritonToLinalg/TritonToLinalgPass.h" -#include "ascend/include/Utils/DebugUtils.h" -#include "ascend/include/Utils/Utils.h" - -#include "bishengir/Dialect/Annotation/IR/Annotation.h" -#include "bishengir/Dialect/HIVM/IR/HIVM.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Arith/Utils/Utils.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/BuiltinTypeInterfaces.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/IRMapping.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/OperationSupport.h" -#include "mlir/IR/Types.h" -#include "mlir/IR/ValueRange.h" -#include "mlir/Transforms/DialectConversion.h" -#include "triton/Dialect/Triton/IR/Types.h" - -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormatVariadic.h" -#include -#include -#include - -#define DEBUG_TYPE "triton-block-ptr-analysis" -namespace mlir { -namespace triton { - -hivm::PointerCastOp createScalarPointerCast(OpBuilder &builder, Location loc, - MemRefType resultType, - Value address) { - SmallVector dynamicSizes; - if (resultType.getNumDynamicDims() != 0) { - Value defaultSize = builder.create(loc, 1); - dynamicSizes.assign(resultType.getNumDynamicDims(), defaultSize); - } - auto pointerCast = builder.create( - loc, resultType, ValueRange{address}, ValueRange{dynamicSizes}); - pointerCast->setAttr(kScalarPointerCarrierAttr, - UnitAttr::get(builder.getContext())); - return pointerCast; -} - -// Recognize original scalar-pointer producers whose converted result may act -// as a memref carrier. The parse site still requires BaseMemRefType, so merely -// appearing in this list never makes an unconverted pointer an opaque source. -static bool isScalarPointerTransport(Operation *op) { - return op && isa(op); -} - -// Returns true only for sources known to carry a complete scalar integer -// address. Before dialect conversion the source is tt.int_to_ptr; after -// IntToPtrConverter it is a memref-producing HIVM PointerCast with explicit -// ScalarPointerCarrier provenance. Ordinary PointerCast descriptors must keep -// using the existing unstructured fallback. -static bool isScalarPointerCarrierSource(Value source) { - if (!source) - return false; - if (source.getDefiningOp()) - return true; - auto pointerCast = source.getDefiningOp(); - return pointerCast && pointerCast->hasAttr(kScalarPointerCarrierAttr); -} - -// Dialect conversion may temporarily materialize a scalar Triton pointer from -// a memref carrier so that an operation which has not been rewritten yet can -// keep its original type. Scalar pointer transport must consume the carrier -// directly; otherwise the conversion driver is forced to leave a live -// memref-to-!tt.ptr cast next to tt.load/tt.store. Only peel the narrow, -// one-to-one UCC form whose input is already a BaseMemRefType. Unknown casts, -// multi-value casts, and non-memref inputs remain unsupported and therefore -// continue through the existing diagnostics. -static Value unwrapScalarPointerMemRefCarrier(Value value) { - while (auto castOp = value.getDefiningOp()) { - if (castOp.getInputs().size() != 1 || castOp.getOutputs().size() != 1) - break; - Value input = castOp.getInputs().front(); - Value output = castOp.getOutputs().front(); - if (!isa(input.getType()) || - !isa(output.getType())) - break; - value = input; - } - return value; -} - -// MemAccType selectMaxMemAccTy(const MemAccType &v1, const MemAccType &v2) { -// return (v1 > v2) ? v1 : v2; -// } - -SmallVector &BlockData::getOffsetsRef() { return this->offsets; } - -SmallVector &BlockData::getSizesRef() { return this->sizes; } - -SmallVector &BlockData::getStridesRef() { return this->strides; } - -Value &BlockData::getSourceRef() { return this->source; } - -OpFoldResult &BlockData::getScalarRef() { return this->scalar; } - -SmallVector BlockData::getOffsets() const { - return this->offsets; -} - -SmallVector BlockData::getSizes() const { return this->sizes; } - -SmallVector BlockData::getStrides() const { - return this->strides; -} - -OpFoldResult BlockData::getOffset(int index) const { - return this->offsets[index]; -} - -OpFoldResult BlockData::getSize(int index) const { return this->sizes[index]; } - -OpFoldResult BlockData::getStride(int index) const { - return this->strides[index]; -} - -OpFoldResult BlockData::getScalar() const { return this->scalar; } - -Value BlockData::getSource() const { return this->source; } - -MemAccType BlockData::getMemAccType() const { return this->memAccTy; }; - -MemAccType &BlockData::getMemAccTypeRef() { return this->memAccTy; }; - -bool BlockData::isScalar() const { return !(this->scalar).isNull(); } - -bool BlockData::isEmpty() const { - return !(this->getRank() || this->source || !(this->scalar).isNull()); -} - -bool BlockData::hasSource() const { return this->source != nullptr; } - -void BlockData::removeSource() { this->source = nullptr; }; - -bool BlockData::hasResElemTy() const { return this->resElemTy != nullptr; } - -Type &BlockData::getResElemTyRef() { return this->resElemTy; } - -Type BlockData::getResElemTy() const { return this->resElemTy; } - -int64_t BlockData::getRank() const { - assert(offsets.size() == sizes.size() && offsets.size() == strides.size()); - return this->offsets.size(); -} - -void BlockData::setResElemTy(const Type &Ty) { this->resElemTy = Ty; } - -void BlockData::setScalar(const OpFoldResult &scalar) { this->scalar = scalar; } - -void BlockData::setSource(const Value &src) { this->source = src; } - -void BlockData::setOffsets(const SmallVector &offsets) { - this->offsets = offsets; -} - -void BlockData::setStrides(const SmallVector &strides) { - this->strides = strides; -} - -void BlockData::setSizes(const SmallVector &szs) { - this->sizes = szs; -} - -void BlockData::setMemAccTy(const MemAccType &v) { this->memAccTy = v; } - -void BlockData::setMemAccVal(const MemAccVal v) { this->memAccTy.value = v; } - -OpFoldResult BlockData::inferBlockOffset(const Location &loc, - OpBuilder &builder) const { - OpFoldResult retOffset = builder.getIndexAttr(0); - for (auto ofr : offsets) { - retOffset = - addOpFoldResult(retOffset, ofr, loc, builder, builder.getIndexType()); - } - return retOffset; -} - -FailureOr -BlockData::getResultMemrefType(int64_t offset, - ArrayRef resultShape) const { - SmallVector staticStrides; - SmallVector dynamicStrides; - dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides); - - if (!this->source) - return failure(); - auto baseMemrefType = dyn_cast(this->source.getType()); - if (!baseMemrefType) - return failure(); - auto elementType = baseMemrefType.getElementType(); - auto layout = - StridedLayoutAttr::get(this->source.getContext(), offset, staticStrides); - return MemRefType::get(resultShape, elementType, layout); -} - -void BlockData::addBlock(BlockData &lBlock, BlockData &rBlock, Location loc, - ConversionPatternRewriter &rewriter) { - assert(this->isEmpty() && lBlock.getRank() == rBlock.getRank()); - // When both left block and right block have source, it is indirect load. - assert(!(lBlock.hasSource() && rBlock.hasSource()) && - "Don't support each BlockData has own base source pointer"); - this->source = - lBlock.hasSource() ? lBlock.getSourceRef() : rBlock.getSourceRef(); - - assert(!(lBlock.hasResElemTy() && rBlock.hasResElemTy())); - if (lBlock.hasResElemTy()) { - assert(lBlock.hasSource()); - this->resElemTy = lBlock.getResElemTyRef(); - } else if (rBlock.hasResElemTy()) { - assert(rBlock.hasSource()); - this->resElemTy = rBlock.getResElemTyRef(); - } - - // Acctually `scalar` should be accumulated into `offset` and `stride` finally - // In addBlock, just pass `scalar` when: - // 1. both lhs and rhs have `scalar` - // 2. otherwise, both lhs and rhs are scalar type with rank 0 - // Except above, original `scalar` has been fused into `offset` under add. - if (lBlock.isScalar() && rBlock.isScalar()) { - auto addScalar = - addOpFoldResult(lBlock.getScalarRef(), rBlock.getScalarRef(), loc, - rewriter, rewriter.getIndexType()); - this->scalar = addScalar; - } else if (lBlock.getRank() == 0) { - // When both lhs and rhs are scalar type with rank 0, just try passing - // potential `scalar` - this->scalar = - lBlock.isScalar() ? lBlock.getScalarRef() : rBlock.getScalarRef(); - } - - for (const auto &[lOffset, rOffset] : - llvm::zip(lBlock.getOffsetsRef(), rBlock.getOffsetsRef())) { - this->offsets.push_back(addOpFoldResult(lOffset, rOffset, loc, rewriter, - rewriter.getIndexType())); - } - - for (const auto &[lStride, rStride] : - llvm::zip(lBlock.getStridesRef(), rBlock.getStridesRef())) { - this->strides.push_back(addOpFoldResult(lStride, rStride, loc, rewriter, - rewriter.getIndexType())); - } - - // Both sizes are same implicitly under `add` - this->sizes = lBlock.getSizesRef(); - - this->getMemAccTypeRef().merge(lBlock.getMemAccTypeRef()); - this->getMemAccTypeRef().merge(rBlock.getMemAccTypeRef()); - // this->setMemAccTy(selectMaxMemAccTy(lBlock.getMemAccType(), - // rBlock.getMemAccType())); -} - -void BlockData::subBlock(BlockData &lBlock, BlockData &rBlock, Location loc, - ConversionPatternRewriter &rewriter) { - assert(this->isEmpty() && lBlock.getRank() == rBlock.getRank()); - - if (lBlock.isScalar() && rBlock.isScalar()) { - auto subScalar = subOpFoldResult(lBlock.getScalarRef(), - rBlock.getScalarRef(), loc, rewriter); - this->scalar = subScalar; - } else if (lBlock.getRank() == 0) { - // When both lhs and rhs are scalar type with rank 0, just try passing - // potential `scalar` - this->scalar = - lBlock.isScalar() ? lBlock.getScalarRef() : rBlock.getScalarRef(); - } - - for (const auto &[lOffset, rOffset] : - llvm::zip(lBlock.getOffsetsRef(), rBlock.getOffsetsRef())) { - this->offsets.push_back(subOpFoldResult(lOffset, rOffset, loc, rewriter)); - } - - for (const auto &[lStride, rStride] : - llvm::zip(lBlock.getStridesRef(), rBlock.getStridesRef())) { - this->strides.push_back(subOpFoldResult(lStride, rStride, loc, rewriter)); - } - - // Both sizes are same implicitly under `sub` - this->sizes = lBlock.getSizesRef(); - - this->getMemAccTypeRef().merge(lBlock.getMemAccTypeRef()); - this->getMemAccTypeRef().merge(rBlock.getMemAccTypeRef()); - // this->setMemAccTy(selectMaxMemAccTy(lBlock.getMemAccType(), - // rBlock.getMemAccType())); -} - -void BlockData::mulBlock(BlockData &lBlock, BlockData &rBlock, Location loc, - ConversionPatternRewriter &rewriter) { - assert(this->isEmpty() && lBlock.getRank() == rBlock.getRank()); - - assert(!(lBlock.hasSource() && rBlock.hasSource())); - - if (lBlock.isScalar() && rBlock.isScalar()) { - LLVM_DEBUG({ - llvm::dbgs() << "lBlock.scalar:" << lBlock.getScalar() - << " rBlbock.scalar:" << rBlock.getScalar() << "\n"; - }); - - auto scalar = mulOpFoldResult(lBlock.getScalar(), rBlock.getScalar(), loc, - rewriter, rewriter.getIndexType()); - this->scalar = scalar; - } - - // assert( - // (lBlock.isScalar() ^ rBlock.isScalar()) && - // "Currently only support one and only one scalar in function - // mulBlock()"); - - BlockData *lb = &lBlock; - BlockData *rb = &rBlock; - if (lb->isScalar()) { - std::swap(lb, rb); - } - - // In mulBlock, `scalar` will be accumulated into `offset` and `stride` - OpFoldResult rScalar = rb->getScalarRef(); - for (const auto &lOffset : lb->getOffsetsRef()) { - this->offsets.push_back(mulOpFoldResult(lOffset, rScalar, loc, rewriter, - rewriter.getIndexType())); - } - - for (const auto &lStride : lb->getStridesRef()) { - this->strides.push_back(mulOpFoldResult(lStride, rScalar, loc, rewriter, - rewriter.getIndexType())); - } - - this->sizes = lb->getSizesRef(); - - this->getMemAccTypeRef().merge(lBlock.getMemAccTypeRef()); - this->getMemAccTypeRef().merge(rBlock.getMemAccTypeRef()); - // this->setMemAccTy(selectMaxMemAccTy(lBlock.getMemAccType(), - // rBlock.getMemAccType())); -} - -void BlockData::divBlock(BlockData &lBlock, BlockData &rBlock, Location loc, - ConversionPatternRewriter &rewriter) { - assert(this->isEmpty() && lBlock.getRank() == rBlock.getRank()); - - assert(!(lBlock.hasSource() && rBlock.hasSource())); - assert(lBlock.isScalar() && rBlock.isScalar()); - - auto rScalar = rBlock.getScalar(); - this->scalar = divOpFoldResult(lBlock.getScalar(), rScalar, loc, rewriter); - - for (auto lOffset : lBlock.getOffsetsRef()) { - this->offsets.push_back(divOpFoldResult(lOffset, rScalar, loc, rewriter)); - } - - for (auto lStride : lBlock.getStridesRef()) { - this->strides.push_back(divOpFoldResult(lStride, rScalar, loc, rewriter)); - } - - this->sizes = lBlock.getSizesRef(); - - this->getMemAccTypeRef().merge(lBlock.getMemAccTypeRef()); - this->getMemAccTypeRef().merge(rBlock.getMemAccTypeRef()); - // this->setMemAccTy(selectMaxMemAccTy(lBlock.getMemAccType(), - // rBlock.getMemAccType())); -} - -FailureOr -BlockData::createCastOp(ArrayRef resultShape, const Location &loc, - OpBuilder &builder) const { - OpFoldResult resOffset = this->inferBlockOffset(loc, builder); - int64_t staticOffset = ShapedType::kDynamic; - if (isa(resOffset)) { - auto constantOffset = getConstantIntValue(resOffset); - if (!constantOffset) - return failure(); - staticOffset = *constantOffset; - } - FailureOr resultType = - this->getResultMemrefType(staticOffset, resultShape); - if (failed(resultType)) - return failure(); - - SmallVector strides(this->strides); - for (size_t i = 0; i < strides.size(); i++) { - if (resultShape[i] == 1) { - if (auto strideValue = dyn_cast(strides[i])) { - auto oneIdx = - builder.create(loc, builder.getIndexAttr(1)); - strides[i] = builder.create(loc, strideValue, oneIdx) - .getResult(); - } - } - } - - return builder.create( - loc, *resultType, this->source, resOffset, this->sizes, strides); -} - -void BlockData::dump() const { - llvm::outs() << "[INFO][BEG] BlockData info\n"; - llvm::outs() << "offsets has " << offsets.size() << " items\n"; - int cnt = 0; - for (auto it = offsets.begin(); it != offsets.end(); ++it) { - llvm::outs() << "offsets[" << cnt++ << "] = " << *it << "\n"; - } - llvm::outs() << "sizes has " << sizes.size() << " items\n"; - cnt = 0; - for (auto it = sizes.begin(); it != sizes.end(); ++it) { - llvm::outs() << "sizes[" << cnt++ << "] = " << *it << "\n"; - } - llvm::outs() << "strides has " << strides.size() << " items\n"; - cnt = 0; - for (auto it = strides.begin(); it != strides.end(); ++it) { - llvm::outs() << "strides[" << cnt++ << "] = " << *it << "\n"; - } - llvm::outs() << "source = " << source << "\n"; - llvm::outs() << "scalar = " << scalar << "\n"; - llvm::outs() << "resElemTy = " << resElemTy << "\n"; - llvm::outs() << "memAccTy = " << memAccTy.toString() << "\n"; - llvm::outs() << "[INFO][END] BlockData info\n"; -} - -FailureOr -BlockDataParser::getScalarMemRef(Value ptr, Value memref, const Location &loc, - ConversionPatternRewriter &rewriter) { - if (!ptr || !memref) - return failure(); - auto pointerType = dyn_cast(ptr.getType()); - if (!pointerType || isa(pointerType.getPointeeType())) - return failure(); - - memref = unwrapScalarPointerMemRefCarrier(memref); - - // A complete scalar address reconstructed after an SCF boundary is exposed - // as tt.int_to_ptr in the Triton IR. Its converted operand is already the - // canonical ScalarPointerCarrier memref; handle it before the legacy - // block-argument path so a direct load/store does not request a reverse - // memref-to-pointer materialization. - if (ptr.getDefiningOp()) { - if (!isa(memref.getType())) - return failure(); - // IntToPtrConverter's rank-1 carrier is still a dynamic base memref, not - // the canonical scalar view consumed by indirect loads. Normalize every - // int_to_ptr carrier to the one-element identity view below. - BlockData data; - data.setSource(memref); - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - auto castOp = data.createCastOp(SmallVector(1, 1), loc, rewriter); - if (failed(castOp)) - return failure(); - return (*castOp).getResult(); - } - - if (ptr.getDefiningOp()) { - if (auto castOp = memref.getDefiningOp()) - return castOp.getResult(); - if (auto pointerCast = memref.getDefiningOp(); - pointerCast && pointerCast->hasAttr(kScalarPointerCarrierAttr)) - return memref; - return failure(); - } - - // A scalar pointer produced by structured control flow or select is already - // represented by a converted memref. Give it the same one-element view used - // for a scalar block argument so direct tt.load/tt.store users can consume - // the transport result without attempting memref-to-pointer materialization. - if (auto definingOp = ptr.getDefiningOp(); - definingOp && isScalarPointerTransport(definingOp)) { - if (!isa(memref.getType())) - return failure(); - if (auto memrefType = dyn_cast(memref.getType()); - memrefType && memrefType.getRank() == 1) - return memref; - BlockData data; - data.setSource(memref); - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - auto castOp = data.createCastOp(SmallVector(1, 1), loc, rewriter); - if (failed(castOp)) - return failure(); - return (*castOp).getResult(); - } - - if (!isa(ptr) || !isa(memref.getType())) - return failure(); - - BlockData data; - data.setSource(memref); - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - auto castOp = data.createCastOp(SmallVector(1, 1), loc, rewriter); - if (failed(castOp)) - return failure(); - return (*castOp).getResult(); -} - -LogicalResult -BlockDataParser::parse(Value operand, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - if (known.find(operand) != known.end()) { - data = known.lookup(operand); - return success(); - } - - if (isa(operand.getType())) { - data.setScalar(getOpFoldResultOfLayoutInfo(operand, rewriter)); - return success(); - } - - // - if (isa(operand.getType())) { - // Just consider two state: ptr and ptr> - Value remappedPtr = rewriter.getRemappedValue(operand); - if (!remappedPtr) { - if (Operation *definingOp = operand.getDefiningOp()) - return definingOp->emitError("scalar pointer has no converted value"); - emitError(loc) << "scalar pointer block argument has no converted value: " - << operand; - return failure(); - } - // A not-yet-rewritten pointer user may be represented by a one-to-one - // UCC from a converted memref. Consume that memref as the transport - // source instead of asking the driver to materialize the pointer back. - remappedPtr = unwrapScalarPointerMemRefCarrier(remappedPtr); - if (auto op = operand.getDefiningOp()) { - if (auto addPtrOp = dyn_cast(op)) { - return parseAddPtr(addPtrOp, data, loc, rewriter, known); - } else if (auto bitcastOp = dyn_cast(op)) { - return parseBitcast(bitcastOp, data, loc, rewriter, known); - } else if (auto makeTensorPtrOp = dyn_cast(op)) { - return parseTensorPtr(makeTensorPtrOp, data, loc, rewriter, known); - } else if (auto advanceOp = dyn_cast(op)) { - // To support - // ptr_0 = tl.advance(ptr) - // ptr_1 = tl.advance(ptr_0) - return parseTensorPtr(advanceOp, data, loc, rewriter, known); - } else if (auto intToPtrOp = dyn_cast(op)) { - if (!isa(remappedPtr.getType())) { - return op->emitError( - "int_to_ptr did not convert to a memref carrier"); - } - data.setSource(remappedPtr); - // An address reconstructed from an i64 is a complete scalar pointer, - // but AddPtr still needs the ordinary one-element BlockData schema to - // form a memref.reinterpret_cast for a later load/store. Without - // this identity view the generic path sees an empty rank and may - // attempt to materialize a pointer from a non-memref source. - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - } else if (isDistributedTypeCustomOp(op)) { - data.setSource(remappedPtr); - } else if (isScalarPointerTransport(op)) { - if (!isa(remappedPtr.getType())) { - return op->emitError( - "scalar pointer transport did not convert to a memref"); - } - data.setSource(remappedPtr); - // Transport producers carry a complete scalar address but do not - // expose BlockData dimensions. Model the address as a one-element - // identity view so a following addptr can add a scalar offset without - // falling into the rank-mismatch/materialization failure path. - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - } else { - return op->emitError() - << "unsupported scalar pointer producer '" << op->getName() - << "' with original type " << operand.getType() - << " and converted type " << remappedPtr.getType(); - } - } else { - data.setSource(remappedPtr); - } - return success(); - } - - // not a scalar pointer - if (auto addOp = operand.getDefiningOp()) { - return parseAdd(addOp, data, loc, rewriter, known); - } else if (auto subOp = operand.getDefiningOp()) { - return parseSub(subOp, data, loc, rewriter, known); - } else if (auto mulOp = operand.getDefiningOp()) { - return parseMul(mulOp, data, loc, rewriter, known); - } else if (auto addPtrOp = operand.getDefiningOp()) { - return parseAddPtr(addPtrOp, data, loc, rewriter, known); - } else if (auto constOp = operand.getDefiningOp()) { - parseConstSplat(constOp, data, loc, rewriter, known); - return success(); - } else if (auto broadcastOp = operand.getDefiningOp()) { - return parseBroadcast(broadcastOp, data, loc, rewriter, known); - } else if (auto splatOp = operand.getDefiningOp()) { - return parseSplat(splatOp, data, loc, rewriter, known); - } else if (auto expandDimsOp = - operand.getDefiningOp()) { - return parseExpandDims(expandDimsOp, data, loc, rewriter, known); - } else if (auto remOp = operand.getDefiningOp()) { - return parseRem(remOp, data, loc, rewriter, known); - } else if (auto bitcastOp = operand.getDefiningOp()) { - return parseBitcast(bitcastOp, data, loc, rewriter, known); - } else if (auto extsiOp = operand.getDefiningOp()) { - return parseExtSI(extsiOp, data, loc, rewriter, known); - } else if (auto divOp = operand.getDefiningOp()) { - return parseDiv(divOp, data, loc, rewriter, known); - } else if (auto makeRangeOp = operand.getDefiningOp()) { - parseMakeRange(makeRangeOp, data, loc, rewriter, known); - return success(); - } else if (auto reduceOp = operand.getDefiningOp()) { - return parseReduce(reduceOp, data, loc, rewriter, known); - } else if (auto loadOp = operand.getDefiningOp()) { - parseIndirectLoad(loadOp, data, loc, rewriter, known); - return success(); - } else if (auto castOp = operand.getDefiningOp()) { - parseIndirectLoad(castOp, data, loc, rewriter, known); - return success(); - } else if (auto extractSliceOp = - operand.getDefiningOp()) { - return parseExtractSlice(extractSliceOp, data, loc, rewriter, known); - } else if (auto forOp = operand.getDefiningOp()) { - auto opResult = dyn_cast(operand); - if (!opResult) - return forOp.emitOpError("expected an OpResult while parsing its result"); - unsigned resultIdx = opResult.getResultNumber(); - parseIndirectLoad(forOp, data, loc, rewriter, known, resultIdx); - return success(); - } else if (auto tensorCastOp = operand.getDefiningOp()) { - // Used for identity operation. - return parse(tensorCastOp.getSource(), data, loc, rewriter, known); - } else if (auto fillOp = operand.getDefiningOp()) { - return parseFill(fillOp, data, loc, rewriter, known); - } else if (auto selectOp = operand.getDefiningOp()) { - if (auto resultType = dyn_cast(selectOp.getType()); - resultType && isa(resultType.getElementType())) - return selectOp.emitOpError( - "tensor-of-pointers select must be lowered before BlockData parsing"); - return parseSelect(selectOp, data, loc, rewriter, known); - } else if (isDistributedTypeCustomOp(operand.getDefiningOp())) { - auto opResult = dyn_cast(operand); - if (!opResult) - return emitError(loc, "expected a custom operation result"); - return parseStructuredCustomOp(operand.getDefiningOp(), data, loc, rewriter, - known, opResult.getResultNumber()); - } else if (auto genericOp = operand.getDefiningOp()) { - if (genericOp->hasAttr("tt.from_make_range")) { - parseLinalgGenericFromMakeRange(genericOp, data, loc, rewriter, known); - return success(); - } - return genericOp.emitOpError( - "cannot parse a generic operation without tt.from_make_range"); - } else if (auto atomicRMWOp = operand.getDefiningOp()) { - parseAtomicRmw(atomicRMWOp, data, loc, rewriter, known); - return success(); - } - - if (Operation *producer = operand.getDefiningOp()) - return producer->emitError() - << "unsupported BlockData producer '" << producer->getName() - << "' with result type " << operand.getType(); - emitError(loc) << "unsupported BlockData block argument of type " - << operand.getType(); - return failure(); -} - -void BlockDataParser::parseAtomicRmw( - triton::AtomicRMWOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - auto opRes = op->getResult(0); - auto opResTy = opRes.getType(); - std::vector resShape; - if (auto shapedResTy = dyn_cast(opResTy)) { - resShape = shapedResTy.getShape().vec(); - if (resShape.size() == 1 && resShape[0] == 1) { - Value zeroIdx = rewriter.create(loc, 0); - Value extracted = - rewriter.create(loc, opRes, ValueRange{zeroIdx}); - Value scalarIdx = rewriter.create( - loc, rewriter.getIndexType(), extracted); - data.setMemAccVal(MemAccVal::StrucMemAcc); - data.setScalar(scalarIdx); - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(0)); - data.getOffsetsRef().push_back(scalarIdx); - return; - } - // For now, we consider this is UnstrucMemAcc because we have no other info. - // Visiting other ops may change the type due to more info. - data.setMemAccVal(MemAccVal::UnstrucMemAcc); - } else { - data.setMemAccVal(MemAccVal::StrucMemAcc); - resShape.push_back(1); - } - for (auto &s : resShape) { - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(s)); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - } - data.setSource(opRes); -} - -LogicalResult -BlockDataParser::parseAdd(arith::AddIOp op, BlockData &data, - const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - BlockData lBlock, rBlock; - if (failed(parse(op.getLhs(), lBlock, loc, rewriter, known)) || - failed(parse(op.getRhs(), rBlock, loc, rewriter, known))) - return failure(); - data.addBlock(lBlock, rBlock, loc, rewriter); - return success(); -} - -LogicalResult -BlockDataParser::parseSub(arith::SubIOp op, BlockData &data, - const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - BlockData lBlock, rBlock; - if (failed(parse(op.getLhs(), lBlock, loc, rewriter, known)) || - failed(parse(op.getRhs(), rBlock, loc, rewriter, known))) - return failure(); - data.subBlock(lBlock, rBlock, loc, rewriter); - return success(); -} - -LogicalResult -BlockDataParser::parseMul(arith::MulIOp op, BlockData &data, - const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - BlockData lBlock, rBlock; - if (failed(parse(op.getLhs(), lBlock, loc, rewriter, known)) || - failed(parse(op.getRhs(), rBlock, loc, rewriter, known))) - return failure(); - - data.mulBlock(lBlock, rBlock, loc, rewriter); - return success(); -} - -LogicalResult -BlockDataParser::parseDiv(arith::DivSIOp op, BlockData &data, - const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - BlockData lBlock, rBlock; - if (failed(parse(op.getLhs(), lBlock, loc, rewriter, known)) || - failed(parse(op.getRhs(), rBlock, loc, rewriter, known))) - return failure(); - data.divBlock(lBlock, rBlock, loc, rewriter); - return success(); -} - -// TODO : support modulos -LogicalResult -BlockDataParser::parseRem(arith::RemSIOp op, BlockData &data, - const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - return op.emitOpError( - "address expressions with modulo are not supported by BlockDataParser"); -} - -void BlockDataParser::parseMakeRange( - triton::MakeRangeOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - insertDebugNop(op.getLoc(), rewriter); - assert(data.isEmpty()); - auto shape = dyn_cast(op.getType()).getShape(); - - auto start = op.getStart(); - auto end = op.getEnd(); - auto stride = (end >= start) && (end - start <= shape[0]); - assert(stride == 1 && - "make_range op should always return a tensor of stride 1"); - - data.getOffsetsRef().push_back(rewriter.getIndexAttr(start)); - data.getSizesRef().push_back(rewriter.getIndexAttr(shape[0])); - data.getStridesRef().push_back(rewriter.getIndexAttr(stride)); -} - -void BlockDataParser::parseLinalgGenericFromMakeRange( - linalg::GenericOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - assert(op->hasAttr("tt.from_make_range") && - "expected tt.from_make_range attribute"); - - auto offsetAttr = op->getAttr("tt.make_range_offset"); - auto sizeAttr = op->getAttr("tt.make_range_size"); - assert(offsetAttr && sizeAttr && - "tt.make_range_offset and tt.make_range_size required"); - - int64_t offset = cast(offsetAttr).getInt(); - int64_t size = cast(sizeAttr).getInt(); - - data.getOffsetsRef().push_back(rewriter.getIndexAttr(offset)); - data.getSizesRef().push_back(rewriter.getIndexAttr(size)); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); -} - -LogicalResult BlockDataParser::parseExpandDims( - triton::ExpandDimsOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - - if (failed(parse(op.getSrcMutable().get(), data, loc, rewriter, known))) - return failure(); - auto resShape = dyn_cast(op.getResult().getType()).getShape(); - auto axis = op.getAxis(); - - assert(resShape[axis] == 1 && - "The destiny shape of changed dimension should be 1"); - - data.getOffsetsRef().insert(data.getOffsetsRef().begin() + axis, - rewriter.getIndexAttr(0)); - data.getSizesRef().insert(data.getSizesRef().begin() + axis, - rewriter.getIndexAttr(1)); - data.getStridesRef().insert(data.getStridesRef().begin() + axis, - rewriter.getIndexAttr(0)); - return success(); -} - -LogicalResult BlockDataParser::parseExtractSlice( - tensor::ExtractSliceOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - const std::string scenarioMessages = - "PtsAnalysis supports indirectly block load in the " - "following scenario\n" - "B = tl.load(Aptr + Aoffset) # B is 1D tensor\n" - "s = tl.extract_slice(indices, offsets= (i,), sizes= " - "(1,), strides= (1,)) # s is a tensor<1x$dtype>\n" - "D = tl.load(Cptr + s + Coffset) # s is used as the " - "scalar offset\n"; // tensor<2x$dtype> will be support - // soon - - auto extract_src = op->getOperand(0); - BlockData srcBlock; - if (failed(parse(extract_src, srcBlock, loc, rewriter, known))) - return failure(); - if (!srcBlock.hasSource()) - return op.emitOpError(scenarioMessages); - // Use isa_and_nonnull for LLVM 21 compatibility - if (!isa_and_nonnull(srcBlock.getSource().getDefiningOp())) - return op.emitOpError(scenarioMessages); - - auto extract_result = op->getResult(0); - auto shaped_ty = dyn_cast(extract_result.getType()); - auto shape = shaped_ty.getShape(); - if (shape.size() > 1 || shape[0] > 1) - return op.emitOpError(scenarioMessages); - auto castOp = rewriter.create( - loc, RankedTensorType::get(shape, rewriter.getIndexType()), - extract_result); - auto offset = castOp.getResult(); - if (data.isEmpty()) { - data.getOffsetsRef().push_back(offset); - data.getSizesRef().push_back(rewriter.getIndexAttr(shape[0])); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - } else { - return op.emitOpError( - "extract_slice parsing with a pre-populated offset is unsupported"); - } - return success(); -} - -LogicalResult BlockDataParser::parseBitcast( - triton::BitcastOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - if (failed(parse(op.getSrc(), data, loc, rewriter, known))) - return failure(); - - auto resType = op.getResult().getType(); - Type resElemPointeeTy = nullptr; - if (auto resShapedTy = dyn_cast(resType)) { - auto resElemTy = resShapedTy.getElementType(); - resElemPointeeTy = - dyn_cast(resElemTy).getPointeeType(); - } else { - auto srcPointeeType = - cast(op.getSrc().getType()).getPointeeType(); - auto resPointeeType = cast(resType).getPointeeType(); - - // Handling special case - // If Op is MetaUse or src is i1 block argument and dst is i8, - // it should be converted to UnrealizedConversionCast - if (op->hasAttr("MetaUse") || - (isa(op.getSrc()) && - srcPointeeType == rewriter.getIntegerType(1) && - resPointeeType == rewriter.getIntegerType(8))) { - resElemPointeeTy = resPointeeType; - } else { - auto remappedValue = rewriter.getRemappedValue(op); - if (!remappedValue) - return op.emitOpError("bitcast result has no converted value"); - data.setSource(remappedValue); - LLVM_DEBUG({ - llvm::dbgs() << "Remapping bitcastOp:\n"; - llvm::dbgs() << op << "\nto \n"; - llvm::dbgs() << remappedValue << "\n"; - }); - } - } - data.setResElemTy(resElemPointeeTy); - return success(); -} - -LogicalResult BlockDataParser::parseExtSI( - arith::ExtSIOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - return parse(op.getIn(), data, loc, rewriter, known); -} - -LogicalResult BlockDataParser::parseBroadcast( - triton::BroadcastOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - - auto src = op.getSrcMutable().get(); - auto dst = op.getResult(); - assert(isa(src.getType()) && - "tt.broadcast's input should be a tensor"); - - auto srcShape = dyn_cast(src.getType()).getShape(); - auto dstShape = dyn_cast(dst.getType()).getShape(); - assert(srcShape.size() == dstShape.size() && - "rank of source shoule be equal to destnation"); - - if (failed(parse(src, data, loc, rewriter, known))) - return failure(); - - for (const auto &[idx, src_dst] : - llvm::enumerate(llvm::zip(srcShape, dstShape))) { - const auto &[srcAxis, dstAxis] = src_dst; - if (srcAxis == dstAxis) { - continue; - } - assert(srcAxis < dstAxis && - "srcShape of broadcastOp must be less than dstShape."); - data.getSizesRef()[idx] = rewriter.getIndexAttr(dstAxis); - } - return success(); -} - -LogicalResult BlockDataParser::parseSplat( - triton::SplatOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - auto src = op.getSrc(); - auto dst = op.getResult(); - auto dstShape = dyn_cast(dst.getType()).getShape(); - - if (failed(parse(src, data, loc, rewriter, known))) - return failure(); - - // A ScalarPointerCarrier stores the complete integer address passed to - // tt.int_to_ptr, so an addptr displacement exists only in BlockData and must - // survive the following splat. Ordinary scalar pointers already carry their - // displacement in the converted memref descriptor; retain the established - // behavior of resetting those offsets while constructing the tensor layout. - OpFoldResult splatOffset; - if (isa(src.getType()) && - isScalarPointerCarrierSource(data.getSource())) { - SmallVector pointerOffsets = data.getOffsets(); - if (pointerOffsets.size() > 1) - return op.emitOpError( - "scalar pointer carrier splat requires at most one BlockData offset"); - splatOffset = pointerOffsets.empty() - ? OpFoldResult(rewriter.getIndexAttr(0)) - : pointerOffsets.front(); - } else if (data.isScalar()) { - splatOffset = data.getScalarRef(); - } - - if (isa(src.getType()) || - isa(src.getType())) { - if (!data.isEmpty()) { - data.getOffsetsRef().clear(); - data.getSizesRef().clear(); - data.getStridesRef().clear(); - } - for (auto dstAxis : dstShape) { - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(dstAxis)); - data.getStridesRef().push_back(rewriter.getIndexAttr(0)); - } - } else { - return op.emitOpError("BlockDataParser does not support this splat source"); - } - if (!splatOffset.isNull()) - data.getOffsetsRef()[0] = splatOffset; - return success(); -} - -void BlockDataParser::parseConstSplat( - arith::ConstantOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - - DenseElementsAttr denseAttr = dyn_cast(op.getValue()); - assert(denseAttr && denseAttr.isSplat() && - isa(denseAttr.getElementType())); - - auto innerVal = denseAttr.getValues()[0].getValue(); - auto innerValIndexAttr = rewriter.getIndexAttr(innerVal.getSExtValue()); - - // for mul state - data.setScalar(innerValIndexAttr); - - auto resType = dyn_cast(op.getResult().getType()); - size_t loopLimit = resType.getShape().size(); - for (auto i = 0; i < loopLimit; i++) { - // Add original dense val to first dim offset for add state - if (i == 0) { - data.getOffsetsRef().push_back(innerValIndexAttr); - } else { - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - } - data.getSizesRef().push_back(rewriter.getIndexAttr(resType.getShape()[i])); - data.getStridesRef().push_back(rewriter.getIndexAttr(0)); - } -} - -template -std::enable_if_t || - std::is_same_v, - LogicalResult> -BlockDataParser::parseTensorPtr( - T op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - - Value remappedValue = rewriter.getRemappedValue(op); - if (!remappedValue) - return op.emitOpError("tensor pointer has no converted value"); - if (auto castOp = remappedValue.getDefiningOp()) { - parseReinterpretCast(castOp, data, loc, rewriter, known); - return success(); - } - return op.emitOpError( - "expected the converted tensor pointer to be a memref.reinterpret_cast"); -} - -LogicalResult BlockDataParser::parseAddPtr( - triton::AddPtrOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - - BlockData ptrBlock, offsetBlock; - if (failed(parse(op.getPtr(), ptrBlock, op.getLoc(), rewriter, known)) || - failed(parse(op.getOffset(), offsetBlock, op.getLoc(), rewriter, known))) - return failure(); - - if (!ptrBlock.hasSource()) - return op.emitOpError( - "could not resolve a source/base pointer for the addptr operand"); - // offset has source means offset is from tl.load and other ops(TODO) - if (offsetBlock.hasSource()) { - ptrBlock.setMemAccTy(offsetBlock.getMemAccType()); - offsetBlock.removeSource(); - } - - // handle for loop & scalar - if (ptrBlock.getRank() == 1 && offsetBlock.getRank() == 0) { - offsetBlock.getSizesRef().push_back(rewriter.getIndexAttr(1)); - offsetBlock.getOffsetsRef().push_back(offsetBlock.getScalarRef()); - offsetBlock.getStridesRef().push_back(rewriter.getIndexAttr(0)); - } - - if (ptrBlock.getRank() != offsetBlock.getRank()) - return op.emitOpError("pointer and offset BlockData ranks do not match"); - LLVM_DEBUG({ - auto &os = llvm::dbgs(); - os << "[parseAddPtr][BEG] =========================\n"; - os << "[parseAddPtr] op is " << op << "\n"; - for (int i = 0; i < ptrBlock.getRank(); i++) { - os << "ptrBlock.getOffsetsRef()[" << i - << "] = " << ptrBlock.getOffsetsRef()[i] << "\n"; - os << "ptrBlock.getSizesRef()[" << i - << "] = " << ptrBlock.getSizesRef()[i] << "\n"; - os << "ptrBlock.getStridesRef()[" << i - << "] = " << ptrBlock.getStridesRef()[i] << "\n"; - os << "offsetBlock.getOffsetsRef()[" << i - << "] = " << offsetBlock.getOffsetsRef()[i] << "\n"; - os << "offsetBlock.getSizesRef()[" << i - << "] = " << offsetBlock.getSizesRef()[i] << "\n"; - os << "offsetBlock.getStridesRef()[" << i - << "] = " << offsetBlock.getStridesRef()[i] << "\n"; - } - os << "[parseAddPtr][END] -------------------------\n"; - }); - data.addBlock(ptrBlock, offsetBlock, op.getLoc(), rewriter); - return success(); -} - -void BlockDataParser::parseReinterpretCast( - memref::ReinterpretCastOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - assert(data.isEmpty()); - - data.setOffsets(op.getMixedOffsets()); - data.setSizes(op.getMixedSizes()); - data.setStrides(op.getMixedStrides()); - data.setSource(op.getSource()); - - // In memref::ReinterpretCastOp, offset means the total of collapsing multiple - // dimensions, which corresponds to first dim offset in block data. - // Here populate the rest of the dimensions with zeroes. - assert(data.getOffsetsRef().size() == 1); - size_t loopLimit = data.getSizesRef().size(); - for (size_t i = 1; i < loopLimit; i++) { - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - } -} - -LogicalResult BlockDataParser::parseReduce( - triton::ReduceOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - - const std::string scenarioMessages = - "PtsAnalysis supports indirectly block load in the following scenario\n" - "B = tl.load(Aptr + Aoffset) # B is 1D tensor\n" - "s = tl.min(B) # s is a scalar\n" - "D = tl.load(Cptr + s + Coffset) # s is used as the scalar offset\n"; - - auto reduce_src = op->getOperand(0); - BlockData srcBlock; - if (failed(parse(reduce_src, srcBlock, loc, rewriter, known))) - return failure(); - if (!srcBlock.hasSource()) - return op.emitOpError(scenarioMessages); - // Use isa_and_nonnull for LLVM 21 compatibility - if (!isa_and_nonnull(srcBlock.getSource().getDefiningOp())) - return op.emitOpError(scenarioMessages); - - auto reduce_result = op->getResult(0); - auto shaped_ty = dyn_cast(reduce_result.getType()); - auto shape = shaped_ty.getShape(); - auto ops = llvm::map_to_vector(op.getBody()->without_terminator(), - [](Operation &op) { return &op; }); - // Support only the case: scalar = tl.load(1D tensor) - if (shape.size() != 1 || op.getAxis() != 0 || ops.size() != 1 || - !isa(ops.front())) - return op.emitOpError(scenarioMessages); - - auto castOp = rewriter.create( - loc, RankedTensorType::get(shape, rewriter.getIndexType()), - reduce_result); - auto offset = castOp.getResult(); - if (data.isEmpty()) { - data.getOffsetsRef().push_back(offset); - data.getSizesRef().push_back(rewriter.getIndexAttr(shape[0])); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - } else { - return op.emitOpError( - "reduce parsing with a pre-populated offset is unsupported"); - } - return success(); -} - -template -void parseIndirectLoad(OpTy op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known, - unsigned resultIdx) { - assert(resultIdx < op->getNumResults() && - "resultIdx out of range for parseIndirectLoad"); - auto opRes = op->getResult(resultIdx); - auto opResTy = opRes.getType(); - std::vector resShape; - if (auto shapedResTy = dyn_cast(opResTy)) { - // For now, we consider this is UnstrucMemAcc because we have no other info. - // Visiting other ops may change the type due to more info. - resShape = shapedResTy.getShape().vec(); - auto numOperands = 3; - if (resShape.size() == 1 && resShape[0] == 1 && - op->getNumOperands() == numOperands) { - Value zeroIdx = rewriter.create(loc, 0); - Value extracted = - rewriter.create(loc, opRes, ValueRange{zeroIdx}); - Value scalarIdx = rewriter.create( - loc, rewriter.getIndexType(), extracted); - data.setMemAccVal(MemAccVal::StrucMemAcc); - data.setScalar(scalarIdx); - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(0)); - data.getOffsetsRef().push_back(scalarIdx); - return; - } - data.setMemAccVal(MemAccVal::UnstrucMemAcc); - } else { - // scalar load means this is used as offset. It is StrucMemAcc. - data.setMemAccVal(MemAccVal::StrucMemAcc); - resShape.push_back(1); - } - for (auto &s : resShape) { - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(s)); - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - } - // set the source in BlockData so that we know an indirect-load op exists in - // the chain. - data.setSource(opRes); -} - -namespace { -template -LogicalResult -parseStructuredCustomOpImpl(CustomOpT op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known, - unsigned resultIdx) { - auto srcValArrayAttr = op->template getAttrOfType( - ConverterUtils::customSrcPtrIndexAttrName); - assert(srcValArrayAttr && - "structure hivm custom op should present src tensor"); - auto srcValArray = srcValArrayAttr.asArrayRef(); - assert(srcValArray[resultIdx] != -1 && - "tensor result should map to src tensor"); - if (failed(BlockDataParser::parse(op->getOperand(srcValArray[resultIdx]), - data, loc, rewriter, known))) - return failure(); - Value remappedResult = rewriter.getRemappedValue(op->getResult(resultIdx)); - if (!remappedResult) - return op.emitOpError("custom operation result has no converted value"); - data.setSource(remappedResult); - return success(); -} - -template -CustomOpT createRewrittenStructuredCustomOp( - ConversionPatternRewriter &rewriter, Location loc, - llvm::ArrayRef resultTypes, CustomOpT op, - typename CustomOpT::Adaptor &adaptor, ValueRange newOutputs) { - if constexpr (std::is_same_v) { - return rewriter.create( - loc, resultTypes, op.getName(), adaptor.getInputs(), newOutputs, - adaptor.getTempBuffers(), adaptor.getSyncRelatedArgs()); - } else { - return rewriter.create(loc, resultTypes, op.getName(), - adaptor.getInputs(), newOutputs, - adaptor.getTempBuffers()); - } -} - -template -void rewriteStructuredCustomOpImpl(CustomOpT op, - typename CustomOpT::Adaptor &adaptor, - ConversionPatternRewriter &rewriter) { - if (isDistributedTypeCustomOp(op)) { - auto ip = rewriter.saveInsertionPoint(); - rewriter.setInsertionPoint(op); - auto loc = op.getLoc(); - llvm::SmallVector newOutputs; - for (auto out : op.getOutputs()) { - auto tensorTy = llvm::cast(out.getType()); - if (llvm::isa(tensorTy.getElementType())) { - continue; - } - newOutputs.emplace_back(rewriter.getRemappedValue(out)); - } - llvm::SmallVector resultTypes; - for (auto ty : op->getResultTypes()) { - if (auto ptrTy = llvm::dyn_cast(ty)) { - resultTypes.emplace_back( - MemRefType::get({ShapedType::kDynamic}, ptrTy.getPointeeType())); - continue; - } - if (auto tensorTy = llvm::dyn_cast(ty)) { - if (auto ptrTy = llvm::dyn_cast( - tensorTy.getElementType())) { - resultTypes.emplace_back( - MemRefType::get(tensorTy.getShape(), ptrTy.getPointeeType())); - continue; - } - } - resultTypes.emplace_back(ty); - } - auto newOp = createRewrittenStructuredCustomOp(rewriter, loc, resultTypes, - op, adaptor, newOutputs); - auto operandSegmentSizesAttr = newOp->getAttr("operandSegmentSizes"); - newOp->setAttrs(op->getAttrs()); - newOp->setAttr("operandSegmentSizes", operandSegmentSizesAttr); - rewriter.replaceOp(op, newOp.getResults()); - rewriter.restoreInsertionPoint(ip); - } else { - SmallVector resultTypes(op->getResultTypes().begin(), - op->getResultTypes().end()); - auto newOp = createRewrittenStructuredCustomOp( - rewriter, op.getLoc(), resultTypes, op, adaptor, adaptor.getOutputs()); - auto operandSegmentSizesAttr = newOp->getAttr("operandSegmentSizes"); - newOp->setAttrs(op->getAttrs()); - newOp->setAttr("operandSegmentSizes", operandSegmentSizesAttr); - rewriter.replaceOp(op, newOp); - } -} -} // namespace - -void BlockDataParser::rewriteStructuredCustomOp( - hivm::CustomOp op, hivm::CustomOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter) { - rewriteStructuredCustomOpImpl(op, adaptor, rewriter); -} - -void BlockDataParser::rewriteStructuredCustomOp( - hivm::CustomMacroOp op, hivm::CustomMacroOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter) { - rewriteStructuredCustomOpImpl(op, adaptor, rewriter); -} - -LogicalResult BlockDataParser::parseStructuredCustomOp( - Operation *op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known, unsigned resultIdx) { - if (auto customOp = dyn_cast(op)) { - return parseStructuredCustomOpImpl(customOp, data, loc, rewriter, known, - resultIdx); - } else if (auto macroOp = dyn_cast(op)) { - return parseStructuredCustomOpImpl(macroOp, data, loc, rewriter, known, - resultIdx); - } - return op->emitError("expected a structured hivm custom operation"); -} - -void BlockDataParser::rewriteStructuredCustomOp( - Operation *op, ConversionPatternRewriter &rewriter) { - if (auto customOp = dyn_cast(op)) { - hivm::CustomOp::Adaptor adaptor(customOp); - rewriteStructuredCustomOpImpl(customOp, adaptor, rewriter); - } else if (auto macroOp = dyn_cast(op)) { - hivm::CustomMacroOp::Adaptor adaptor(macroOp); - rewriteStructuredCustomOpImpl(macroOp, adaptor, rewriter); - } else { - llvm_unreachable("expected hivm custom op"); - } -} - -LogicalResult -BlockDataParser::parseFill(linalg::FillOp op, BlockData &data, - const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - auto src = op.getInputs()[0]; - auto dst = op.getResult(0); - auto dstShape = dyn_cast(dst.getType()).getShape(); - - if (failed(parse(src, data, loc, rewriter, known))) - return failure(); - - if (isa(src.getType())) { - if (!data.isEmpty()) { - data.getOffsetsRef().clear(); - data.getSizesRef().clear(); - data.getStridesRef().clear(); - } - for (auto dstAxis : dstShape) { - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - data.getSizesRef().push_back(rewriter.getIndexAttr(dstAxis)); - data.getStridesRef().push_back(rewriter.getIndexAttr(0)); - } - } else { - return op.emitOpError("BlockDataParser does not support this fill pattern"); - } - if (data.isScalar()) { - data.getOffsetsRef()[0] = data.getScalarRef(); - } - return success(); -} - -LogicalResult BlockDataParser::parseSelect( - arith::SelectOp op, BlockData &data, const Location &loc, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - if (!data.isEmpty()) - return op.emitOpError( - "select parsing requires an empty BlockData destination"); - - auto res = op.getResult(); - auto resType = dyn_cast(res.getType()); - if (!resType || (!isa(resType.getElementType()) && - !isa(resType.getElementType()))) - return op.emitOpError( - "BlockData select requires a shaped integer or index result"); - - OpFoldResult indexOfr; - size_t loopLimit = resType.getShape().size(); - - Value cond = op.getCondition(); - bool condIsScalarI1 = isa(cond.getType()) && - cast(cond.getType()).getWidth() == 1 && - !isa(cond.getType()); - - auto trueConst = - dyn_cast_or_null(op.getTrueValue().getDefiningOp()); - auto falseConst = - dyn_cast_or_null(op.getFalseValue().getDefiningOp()); - auto trueDense = trueConst ? dyn_cast(trueConst.getValue()) - : DenseElementsAttr(); - auto falseDense = falseConst - ? dyn_cast(falseConst.getValue()) - : DenseElementsAttr(); - - bool denseConstCase = condIsScalarI1 && trueDense && falseDense && - trueDense.isSplat() && falseDense.isSplat(); - - if (denseConstCase) { - // if cond is scalar i1 and both true and false value are splat dense const, - // we can directly use the value of the dense const to create scalar select - // op. - Attribute trueFirst = *trueDense.value_begin(); - Attribute falseFirst = *falseDense.value_begin(); - - Value trueScalar = nullptr; - Value falseScalar = nullptr; - if (auto tInt = dyn_cast(trueFirst)) { - trueScalar = rewriter.create(loc, tInt).getResult(); - } else { - return op.emitOpError( - "BlockData select requires integer dense true values"); - } - - if (auto fInt = dyn_cast(falseFirst)) { - falseScalar = rewriter.create(loc, fInt).getResult(); - } else { - return op.emitOpError( - "BlockData select requires integer dense false values"); - } - - if (trueScalar.getType() != falseScalar.getType()) - return op.emitOpError( - "scalarized BlockData select values must have the same type"); - - auto scalarSelect = rewriter.create( - loc, trueScalar.getType(), cond, trueScalar, falseScalar); - - indexOfr = getOpFoldResultOfLayoutInfo(scalarSelect.getResult(), rewriter); - } else { - if (!llvm::all_of(resType.getShape(), [](int64_t dim) { return dim == 1; })) - return op.emitOpError( - "BlockData select supports non-splat values only when every result " - "dimension is one"); - - SmallVector indices; - indices.reserve(loopLimit); - for (size_t i = 0; i < loopLimit; ++i) { - indices.push_back(rewriter.create(loc, 0)); - } - - auto extractOp = rewriter.create(loc, res, indices); - indexOfr = extractOp.getResult(); - if (isa(extractOp.getType())) { - indexOfr = getOpFoldResultOfLayoutInfo(extractOp.getResult(), rewriter); - } - } - - // Set scalar for mul state - data.setScalar(indexOfr); - - for (size_t i = 0; i < loopLimit; ++i) { - // Add scalar to first dim offset for add state - if (i == 0) { - data.getOffsetsRef().push_back(indexOfr); - } else { - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - } - data.getSizesRef().push_back(rewriter.getIndexAttr(resType.getShape()[i])); - data.getStridesRef().push_back(rewriter.getIndexAttr(0)); - } - return success(); -} - -LogicalResult -BlockDataParser::rewriteAddPtr(triton::AddPtrOp op, - triton::AddPtrOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known) { - ConversionPatternRewriter::InsertionGuard guard(rewriter); - rewriter.setInsertionPoint(op); - - Location offLoc = op.getLoc(); - if (Value off = op.getOffset()) - if (Operation *defOp = off.getDefiningOp()) - offLoc = defOp->getLoc(); - insertDebugNop(offLoc, rewriter); - BlockData data; - if (failed(parseAddPtr(op, data, op.getLoc(), rewriter, known))) - return failure(); - - if (data.getMemAccTypeRef().isUnstructured() && - !isScalarPointerCarrierSource(data.getSource())) { - // TODO: Based on more info, try to create a performant IR - if (failed(rewriteAddPtrToUnstrucMemAcc(op, adaptor, rewriter, data))) - return failure(); - LLVM_DEBUG({ llvm::dbgs() << *getModuleOpFromOperation(op) << "\n"; }); - return success(); - } - - if (data.getSizesRef().size() == 0) { - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(0)); - data.getOffsetsRef().push_back(data.getScalarRef()); - } - - ArrayRef resultShape; - // shape {1,} is stub for single ptr - SmallVector stubScalarTypeShape(1, 1); - if (auto shapedType = dyn_cast(op.getResult().getType())) { - resultShape = shapedType.getShape(); - } else { - assert(data.getRank() == 1); - resultShape = stubScalarTypeShape; - } - - known[op.getResult()] = data; - - // If there are dimensions with size 1 and stride 0, replace 0 stride with the - // product of sizes of all lower dimensions. This avoids creating memref with - // zero stride. - // And here store the unmodified state into known ptrs, since any following - // pointer arithmetic operations should still use the original 0 stride. - auto inferedSize = 1; - auto hoistDim = op->getAttrOfType("hoist_dim"); - for (int i = data.getSizesRef().size() - 1; i >= 0; i--) { - auto strideConst = getConstantIntValue(data.getStridesRef()[i]); - auto sizeConst = getConstantIntValue(data.getSizesRef()[i]); - assert(sizeConst.has_value()); - bool shouldReplaceStride = - (sizeConst.value() == 1) || (hoistDim && hoistDim.getValue() == i); - if (shouldReplaceStride && strideConst && strideConst.value() == 0) { - data.getStridesRef()[i] = rewriter.getIndexAttr(inferedSize); - } - inferedSize *= sizeConst.value(); - } - - auto &offsets = data.getOffsetsRef(); - for (size_t i = 0; i < offsets.size(); ++i) { - if (auto constVal = getConstantIntValue(offsets[i])) { - if (constVal.value() < 0) { - LLVM_DEBUG({ - llvm::dbgs() << "[NegOffsetElim] Detected negative offset: " - << constVal.value() << " at dim " << i << "\n"; - }); - - Value negOffsetVal = rewriter.create( - op.getLoc(), constVal.value()); - offsets[i] = negOffsetVal; - } - } - } - - if (auto intToPtrOp = dyn_cast_or_null( - data.getSourceRef().getDefiningOp())) { - auto rtype = cast(intToPtrOp.getResult().getType()); - auto memrefType = - MemRefType::get({ShapedType::kDynamic}, rtype.getPointeeType()); - auto hivmPointCastOp = createScalarPointerCast( - rewriter, intToPtrOp.getLoc(), memrefType, intToPtrOp.getSrc()); - data.setSource(hivmPointCastOp.getResult()); - } - - if (data.hasResElemTy()) { - // Handle bitcast scenario - auto memrefType = dyn_cast(data.getSourceRef().getType()) - .cloneWith(std::nullopt, data.getResElemTyRef()); - UnrealizedConversionCastOp castOp = - rewriter.create( - op.getLoc(), memrefType, data.getSourceRef()); - data.setSource(castOp.getOutputs()[0]); - } - - // ToDo: need to handle module scenario - - FailureOr castOp = - data.createCastOp(resultShape, op.getLoc(), rewriter); - if (failed(castOp)) - return op.emitOpError( - "could not materialize a memref from the source type"); - Value src = (*castOp).getResult(); - LLVM_DEBUG({ - llvm::dbgs() << "cast MemRefType:\n"; - (*castOp).getOperation()->print(llvm::dbgs(), - OpPrintingFlags().printGenericOpForm()); - llvm::dbgs() << "\n"; - }); - - rewriter.replaceOp(op, src); - return success(); -} - -FailureOr BlockDataParser::materializePointer( - Value ptr, ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known) { - SmallVector resultShape; - if (auto resultTy = dyn_cast(ptr.getType())) { - if (!isa(resultTy.getElementType())) - return failure(); - resultShape.append(resultTy.getShape().begin(), resultTy.getShape().end()); - } else if (auto pointerTy = dyn_cast(ptr.getType())) { - // A pointer to a shaped value is a block pointer, not a scalar element - // pointer. It is materialized by the make_tensor_ptr path instead. - if (isa(pointerTy.getPointeeType())) - return failure(); - resultShape.push_back(1); - } else { - return failure(); - } - - Operation *defOp = ptr.getDefiningOp(); - if (!defOp) - return failure(); - - ConversionPatternRewriter::InsertionGuard guard(rewriter); - rewriter.setInsertionPoint(defOp); - - BlockData data; - if (failed(parse(ptr, data, ptr.getLoc(), rewriter, known)) || - !data.hasSource() || data.getMemAccType().isUnstructured()) - return failure(); - - if (data.getSizesRef().empty()) { - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - data.getStridesRef().push_back(rewriter.getIndexAttr(0)); - data.getOffsetsRef().push_back(data.getScalarRef().isNull() - ? OpFoldResult(rewriter.getIndexAttr(0)) - : data.getScalarRef()); - } - - if (data.getRank() != static_cast(resultShape.size())) - return failure(); - - known[ptr] = data; - - // A unit dimension with zero stride is represented as a normal contiguous - // unit dimension. Non-unit zero strides are intentional (for example a - // splatted base pointer) and must be preserved. - int64_t inferredSize = 1; - for (int64_t i = data.getRank() - 1; i >= 0; --i) { - auto strideConst = getConstantIntValue(data.getStridesRef()[i]); - auto sizeConst = getConstantIntValue(data.getSizesRef()[i]); - if (!sizeConst) - return failure(); - if (sizeConst.value() == 1 && strideConst && strideConst.value() == 0) - data.getStridesRef()[i] = rewriter.getIndexAttr(inferredSize); - inferredSize *= sizeConst.value(); - } - - // Keep negative static offsets as SSA values. This mirrors rewriteAddPtr and - // avoids unsigned attribute handling in later reinterpret_cast lowering. - auto &offsets = data.getOffsetsRef(); - for (OpFoldResult &offset : offsets) { - if (auto constVal = getConstantIntValue(offset); - constVal && constVal.value() < 0) { - offset = - rewriter - .create(ptr.getLoc(), constVal.value()) - .getResult(); - } - } - - if (auto intToPtrOp = dyn_cast_or_null( - data.getSourceRef().getDefiningOp())) { - auto pointerTy = - cast(intToPtrOp.getResult().getType()); - auto memrefTy = - MemRefType::get({ShapedType::kDynamic}, pointerTy.getPointeeType()); - auto pointerCast = createScalarPointerCast(rewriter, intToPtrOp.getLoc(), - memrefTy, intToPtrOp.getSrc()); - data.setSource(pointerCast.getResult()); - } - - if (data.hasResElemTy()) { - auto sourceTy = dyn_cast(data.getSourceRef().getType()); - if (!sourceTy) - return failure(); - auto castTy = sourceTy.cloneWith(std::nullopt, data.getResElemTyRef()); - if (sourceTy != castTy) { - auto cast = rewriter.create( - ptr.getLoc(), castTy, data.getSourceRef()); - data.setSource(cast.getResult(0)); - } - } - - FailureOr castOp = - data.createCastOp(resultShape, ptr.getLoc(), rewriter); - if (failed(castOp)) - return failure(); - return (*castOp).getResult(); -} - -static FailureOr -getBaseMemRefOffset(Value convertedBase, ConversionPatternRewriter &rewriter) { - auto memrefType = dyn_cast(convertedBase.getType()); - if (!memrefType) - return failure(); - // Preserve the existing foldable path for a directly converted tt.addptr. - // Reinterpret-casting a reinterpret cast does not compose offsets, so the - // first cast's absolute offset must be carried into the new descriptor. - if (auto baseRecast = - convertedBase.getDefiningOp()) - return baseRecast.getConstifiedMixedOffset(); - - auto stridedLayout = memrefType.getStridesAndOffset(); - int64_t staticOffset = stridedLayout.second; - if (!ShapedType::isDynamic(staticOffset)) - return OpFoldResult(rewriter.getIndexAttr(staticOffset)); - - // A control-flow-carried BlockPtr base uses the canonical identity-layout - // memref and therefore has static offset zero. Dynamic hidden offsets are not - // valid BlockPtr bases; their displacement belongs in descriptor offsets. - return failure(); -} - -LogicalResult BlockDataParser::rewriteCustomOp( - hivm::CustomOp op, hivm::CustomOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter, - const llvm::SmallDenseMap &known) { - ConversionPatternRewriter::InsertionGuard guard(rewriter); - rewriter.setInsertionPoint(op); - auto loc = op.getLoc(); - llvm::SmallVector newInputs; - llvm::SmallVector newOutputs; - auto convertIntToPtr = [&rewriter](BlockData &data) { - if (auto intToPtrOp = dyn_cast_or_null( - data.getSourceRef().getDefiningOp())) { - auto rtype = cast(intToPtrOp.getResult().getType()); - auto memrefType = - MemRefType::get({ShapedType::kDynamic}, rtype.getPointeeType()); - auto hivmPointCastOp = createScalarPointerCast( - rewriter, intToPtrOp.getLoc(), memrefType, intToPtrOp.getSrc()); - if (data.getSizesRef().size() == 0) { - data.getSizesRef().push_back(rewriter.getIndexAttr(1)); - if (data.getScalarRef().isNull()) { - data.getOffsetsRef().push_back(rewriter.getIndexAttr(0)); - } else { - data.getOffsetsRef().push_back(data.getScalarRef()); - } - data.getStridesRef().push_back(rewriter.getIndexAttr(1)); - } - data.setSource(hivmPointCastOp.getResult()); - } - }; - for (auto in : op.getInputs()) { - in = rewriter.getRemappedValue(in); - if (!in) - return op.emitOpError("custom operation input has no converted value"); - BlockData blockData; - auto curInput = in; - if (llvm::isa(in.getType())) { - if (failed(parse(in, blockData, loc, rewriter, known))) - return failure(); - convertIntToPtr(blockData); - FailureOr castOp = - blockData.createCastOp({ShapedType::kDynamic}, loc, rewriter); - if (failed(castOp)) - return failure(); - curInput = (*castOp).getResult(); - } else if (auto tensor = llvm::dyn_cast(in.getType())) { - if (llvm::isa(tensor.getElementType())) { - if (failed(parse(in, blockData, loc, rewriter, known))) - return failure(); - convertIntToPtr(blockData); - FailureOr castOp = - blockData.createCastOp(tensor.getShape(), loc, rewriter); - if (failed(castOp)) - return failure(); - curInput = (*castOp).getResult(); - } - } - newInputs.emplace_back(curInput); - } - for (auto out : op.getOutputs()) { - auto tensorTy = llvm::cast(out.getType()); - if (llvm::isa(tensorTy.getElementType())) { - // simd library shouldn't output tensor - // after rewrite, delete the tensor output value - continue; - } - newOutputs.emplace_back(rewriter.getRemappedValue(out)); - } - llvm::SmallVector resultTypes; - for (auto ty : op->getResultTypes()) { - if (auto ptrTy = llvm::dyn_cast(ty)) { - resultTypes.emplace_back( - MemRefType::get({ShapedType::kDynamic}, ptrTy.getPointeeType())); - continue; - } - if (auto tensorTy = llvm::dyn_cast(ty)) { - if (auto ptrTy = - llvm::dyn_cast(tensorTy.getElementType())) { - resultTypes.emplace_back( - MemRefType::get(tensorTy.getShape(), ptrTy.getPointeeType())); - continue; - } - } - resultTypes.emplace_back(ty); - } - auto newCustomOp = - rewriter.create(loc, resultTypes, op.getName(), newInputs, - newOutputs, adaptor.getTempBuffers()); - auto operandSegmentSizesAttr = newCustomOp->getAttr("operandSegmentSizes"); - newCustomOp->setAttrs(op->getAttrs()); - newCustomOp->setAttr("operandSegmentSizes", operandSegmentSizesAttr); - rewriter.replaceOp(op, newCustomOp.getResults()); - return success(); -} - -// Design for load/store boundary_check. -FailureOr -createRedundantOp(triton::MakeTensorPtrOp op, OpFoldResult sourceBaseOffset, - ConversionPatternRewriter &rewriter, BlockData &data) { - auto loc = op.getLoc(); - // to do boundary_check in tt.load, we need to keep the parent tensor's - // shape info in the IR. - // use the parent tensor's shape to create a cast - auto resultSizes = data.getSizes(); - auto resultOffsets = data.getOffsets(); - data.getSizesRef().clear(); - data.getOffsetsRef().clear(); - data.getSizesRef() = - std::move(llvm::map_to_vector(op.getShape(), [&](Value v) { - return getOpFoldResultOfLayoutInfo(v, rewriter); - })); - - // This redundant ReinterpretCastOp is to describe full tensor_ptr, so each - // dim offset from base is initialized as zero. - SmallVector curOffsets(op.getOffsets().size(), - rewriter.getIndexAttr(0)); - // Both the full-shape descriptor and the final block descriptor use the - // same absolute source offset. Reusing this value avoids both dropping it - // across SCF and accidentally composing it twice. - curOffsets.front() = sourceBaseOffset; - - for (auto offset : curOffsets) { - data.getOffsetsRef().push_back(offset); - } - - SmallVector staticShapes; - SmallVector dynamicShapes; - dispatchIndexOpFoldResults(data.getSizesRef(), dynamicShapes, staticShapes); - auto castOp = data.createCastOp(staticShapes, loc, rewriter); - // restore sizes and offsets - data.getSizesRef().clear(); - for (auto &s : resultSizes) { - data.getSizesRef().push_back(s); - } - data.getOffsetsRef().clear(); - for (auto &offset : resultOffsets) { - data.getOffsetsRef().push_back(offset); - } - return castOp; -} - -// Carries the converted runtime descriptor and records whether the resolver -// has already proven the complete ptr-to-ptr normalization contract. -// That exact fallback can initialize BlockData without consulting conversion -// state attached to the original bitcast result. -struct ResolvedMakeTensorPtrBase { - Value value; - bool normalizedI1ToI8 = false; -}; - -static FailureOr -resolveMakeTensorPtrBase(triton::MakeTensorPtrOp op, Value adaptorBase, - ConversionPatternRewriter &rewriter) { - if (adaptorBase && isa(adaptorBase.getType())) - return ResolvedMakeTensorPtrBase{adaptorBase}; - - auto bitcast = op.getBase().getDefiningOp(); - if (!bitcast) - return failure(); - - auto sourceArgument = dyn_cast(bitcast.getSrc()); - if (!sourceArgument) - return failure(); - - auto sourcePointer = - dyn_cast(bitcast.getSrc().getType()); - auto resultPointer = - dyn_cast(bitcast.getResult().getType()); - if (!sourcePointer || !resultPointer || - !sourcePointer.getPointeeType().isInteger(1) || - !resultPointer.getPointeeType().isInteger(8)) - return failure(); - - Value convertedSource = rewriter.getRemappedValue(bitcast.getSrc()); - if (!convertedSource) - return failure(); - auto sourceMemRef = dyn_cast(convertedSource.getType()); - auto expectedType = - MemRefType::get({ShapedType::kDynamic}, rewriter.getIntegerType(8)); - if (!sourceMemRef || sourceMemRef != expectedType) - return failure(); - - return ResolvedMakeTensorPtrBase{convertedSource, - /*normalizedI1ToI8=*/true}; -} - -LogicalResult BlockDataParser::rewriteMakeTensorPtrOp( - triton::MakeTensorPtrOp op, Value convertedBase, - ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known) { - FailureOr resolvedBase = - resolveMakeTensorPtrBase(op, convertedBase, rewriter); - if (failed(resolvedBase)) { - op.emitOpError("expected the converted base to be a memref descriptor"); - return failure(); - } - convertedBase = resolvedBase->value; - Location loc = op.getLoc(); - BlockData data; - - if (resolvedBase->normalizedI1ToI8) { - // The resolver has already established the complete normalized mask-base - // contract. Avoid querying the bitcast result mapping while dialect - // conversion may still be rewriting its owner block. - data.setSource(convertedBase); - data.setResElemTy(rewriter.getIntegerType(8)); - } else { - // Parse the original producer only for semantic information such as a - // bitcast element type. The runtime source always comes from the resolved - // converted base so SCF-selected memref descriptors are not bypassed. - if (failed( - BlockDataParser::parse(op.getBase(), data, loc, rewriter, known))) - return failure(); - } - if (!data.hasSource()) { - op.emitOpError("failed to resolve the converted scalar base"); - return failure(); - } - if (data.hasResElemTy()) { - auto sourceType = dyn_cast(convertedBase.getType()); - if (!sourceType) { - op.emitOpError("bitcast base did not resolve to a memref descriptor"); - return failure(); - } - auto memrefType = - sourceType.cloneWith(std::nullopt, data.getResElemTyRef()); - if (sourceType == memrefType) { - data.setSource(convertedBase); - } else { - UnrealizedConversionCastOp castOp = - rewriter.create(loc, memrefType, - convertedBase); - data.setSource(castOp.getOutputs()[0]); - } - } else { - data.setSource(convertedBase); - } - - data.getOffsetsRef() = - std::move(llvm::map_to_vector(op.getOffsets(), [&](Value v) { - auto zeroVal = rewriter.create( - loc, rewriter.getI32IntegerAttr(0)); - v = rewriter.create(loc, v, zeroVal); - return getOpFoldResultOfLayoutInfo(v, rewriter); - })); - data.getStridesRef() = - std::move(llvm::map_to_vector(op.getStrides(), [&](Value v) { - return getOpFoldResultOfLayoutInfo(v, rewriter); - })); - - SmallVector newOffsets; - for (auto [offset, stride] : - llvm::zip(data.getOffsetsRef(), data.getStridesRef())) - newOffsets.push_back(mulOpFoldResult(offset, stride, loc, rewriter, - rewriter.getIndexType())); - - if (newOffsets.empty()) { - op.emitOpError("expected at least one block pointer dimension"); - return failure(); - } - - FailureOr sourceBaseOffset = - getBaseMemRefOffset(convertedBase, rewriter); - if (failed(sourceBaseOffset)) { - op.emitOpError("could not extract the converted base offset"); - return failure(); - } - newOffsets.front() = addOpFoldResult(newOffsets.front(), *sourceBaseOffset, - loc, rewriter, rewriter.getIndexType()); - - data.getOffsetsRef().clear(); - - for (auto offset : newOffsets) { - data.getOffsetsRef().push_back(offset); - } - - ArrayRef resultShape; - auto pointerType = cast(op.getResult().getType()); - if (auto shapedType = dyn_cast(pointerType.getPointeeType())) { - resultShape = shapedType.getShape(); - data.getSizesRef().clear(); - for (auto dim_size : resultShape) { - data.getSizesRef().push_back( - IntegerAttr::get(IntegerType::get(op.getContext(), 64), dim_size)); - } - } else { - // scalar pointer, should produce a one dimensional memref - SmallVector scalarShape(1, 1); - resultShape = scalarShape; - assert(data.getRank() == 1); - } - - // special handling for davinci - // create redundant reinterpret_cast op for record shape info - FailureOr redundantOp = - createRedundantOp(op, *sourceBaseOffset, rewriter, data); - if (failed(redundantOp)) - return op.emitOpError("could not materialize the full tensor pointer"); - (*redundantOp)->setAttr("tensor_ptr_full_shape", rewriter.getUnitAttr()); - - // create reinterpret_cast op for the target block - data.setSource((*redundantOp).getResult()); - known[op.getResult()] = data; - FailureOr castOp = - data.createCastOp(resultShape, loc, rewriter); - if (failed(castOp)) - return op.emitOpError("could not materialize the block pointer"); - rewriter.replaceOp(op, (*castOp).getResult()); - - if (nd2nzFlag) { - auto basePtr = (*castOp).getResult(); - int original_rank = op.getShape().size() + 1; - std::string shapeStr; - - auto baseMemrefType = mlir::dyn_cast(basePtr.getType()); - assert(baseMemrefType && "basePtr is not a memref type"); - auto shape = baseMemrefType.getShape(); - - if (auto memrefType = mlir::dyn_cast(basePtr.getType())) { - for (auto dim : memrefType.getShape()) { - shapeStr += llvm::formatv("_{0}", dim); - } - } - std::string elemTypeName; - Type elemType = baseMemrefType.getElementType(); - if (auto intType = mlir::dyn_cast(elemType)) { - elemTypeName = llvm::formatv("i{0}", intType.getWidth()); - } else if (auto floatType = mlir::dyn_cast(elemType)) { - std::string floatTypeName; - llvm::raw_string_ostream os(floatTypeName); - floatType.print(os); - os.flush(); - elemTypeName = floatTypeName; - } else { - std::string typeName; - llvm::raw_string_ostream os(typeName); - elemType.print(os); - os.flush(); - elemTypeName = typeName; - } - - std::string memrefTypeStr; - llvm::raw_string_ostream os(memrefTypeStr); - baseMemrefType.print(os); - os.flush(); - - std::string laydbgsuffix; - for (char c : memrefTypeStr) { - if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || c == '_' || c == ',' || c == '[' || - c == ']') { - laydbgsuffix += c; - } - } - auto funcName = rewriter.getStringAttr( - llvm::formatv("__hmf_original_shape{0}d{1}_{2}_{3}", original_rank, - shapeStr, elemTypeName, laydbgsuffix)); - MemRefType targetMemrefType = MemRefType::get( - baseMemrefType.getShape(), baseMemrefType.getElementType(), - baseMemrefType.getLayout()); - const int vectorSize = 4; - SmallVector srcElemTys; - for (auto sz : op.getShape()) { - srcElemTys.push_back(sz.getType()); - } - srcElemTys.push_back(targetMemrefType); - Type dstElemTy = rewriter.getNoneType(); - FunctionType hintFuncType = - FunctionType::get(rewriter.getContext(), srcElemTys, {dstElemTy}); - - auto mod = SymbolTable::getNearestSymbolTable(op); - auto extFunc = dyn_cast_or_null( - SymbolTable::lookupSymbolIn(mod, funcName)); - SmallVector args; - for (auto sz : op.getShape()) { - args.push_back(sz); - } - args.push_back(basePtr); - if (!extFunc) { - OpBuilder::InsertionGuard guard(rewriter); - rewriter.setInsertionPointToStart(&mod->getRegion(0).front()); - extFunc = rewriter.create(rewriter.getUnknownLoc(), - funcName, hintFuncType); - extFunc.setPrivate(); - extFunc->setAttr(LLVM::LLVMDialect::getReadnoneAttrName(), - UnitAttr::get(rewriter.getContext())); - rewriter.setInsertionPoint(op); - } - rewriter.create(loc, funcName, dstElemTy, args); - } - - return success(); -} - -LogicalResult BlockDataParser::rewriteAdvanceOp( - triton::AdvanceOp op, ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known) { - OpBuilder::InsertionGuard insertionGuard(rewriter); - rewriter.setInsertionPoint(op); - auto loc = op.getLoc(); - - BlockData blockData; - if (failed(parse(op.getOperand(0), blockData, loc, rewriter, known))) - return failure(); - - // region [BUGFIX] Add the code block below following the same logic as - // 'BlockDataParser::rewriteAddPtr' function. - known[op.getResult()] = blockData; - auto inferedSize = 1; - for (int i = blockData.getSizesRef().size() - 1; i >= 0; i--) { - auto strideConst = getConstantIntValue(blockData.getStridesRef()[i]); - auto sizeConst = getConstantIntValue(blockData.getSizesRef()[i]); - assert(sizeConst.has_value()); - if (sizeConst.value() == 1 && strideConst && strideConst.value() == 0) { - blockData.getStridesRef()[i] = rewriter.getIndexAttr(inferedSize); - } - inferedSize *= sizeConst.value(); - } - // endregion - - SmallVector incrementOffsets = - llvm::map_to_vector(op.getOffsets(), [&](Value offset) { - return getOpFoldResultOfLayoutInfo(offset, rewriter); - }); - - SmallVector newOffsets; - for (const auto [increment, originalOffset, stride] : - llvm::zip(incrementOffsets, blockData.getOffsetsRef(), - blockData.getStridesRef())) { - auto curDimOffset = - addOpFoldResult(mulOpFoldResult(increment, stride, loc, rewriter, - rewriter.getIndexType()), - originalOffset, loc, rewriter, rewriter.getIndexType()); - - newOffsets.push_back(curDimOffset); - } - - blockData.getOffsetsRef().clear(); - - for (auto offset : newOffsets) - blockData.getOffsetsRef().push_back(offset); - - SmallVector scalarShape(1, 1); // Stub shape - ArrayRef resultShape; - auto pointerType = cast(op.getResult().getType()); - - if (auto shapedType = dyn_cast(pointerType.getPointeeType())) { - resultShape = shapedType.getShape(); - } else { - // scalar pointer, should produce a one dimensional memref - resultShape = scalarShape; - assert(blockData.getRank() == 1); - } - - FailureOr newOp = - blockData.createCastOp(resultShape, loc, rewriter); - if (failed(newOp)) - return op.emitOpError("could not materialize the advanced pointer"); - rewriter.replaceOp(op, (*newOp).getResult()); - - known[(*newOp).getResult()] = blockData; - return success(); -} - -static bool isIntegerTensorBlockDataValue(Value value) { - auto tensorType = dyn_cast(value.getType()); - return tensorType && isa(tensorType.getElementType()); -} - -static bool containsLegacyTritonPointer(Type type) { - if (isa(type)) - return true; - auto shapedType = dyn_cast(type); - return shapedType && isa(shapedType.getElementType()); -} - -// Before expanding a loop signature, reject backedges that are already known -// to be opaque to BlockData. Triton pointer values may still receive a legal -// converted mapping later, while complete memrefs require an explicit -// reinterpret-cast producer. In particular, this deliberately rejects an -// arbitrary fixed-layout memref returned by func.call. -static bool canAnalyzeLegacyBlockDataBackedge(LoopLikeOpInterface loopOp, - unsigned slot, Value value) { - if (containsLegacyTritonPointer(value.getType()) || - isIntegerTensorBlockDataValue(value) || - value.getDefiningOp()) - return true; - - // Forwarding the same loop-carried block argument preserves the descriptor - // state already parsed from the init. A captured function/block argument is - // not equivalent and remains opaque to this analysis. - if (slot < loopOp.getRegionIterArgs().size() && - value == loopOp.getRegionIterArgs()[slot]) - return true; - if (auto whileOp = dyn_cast(loopOp.getOperation())) { - return slot < whileOp.getAfterArguments().size() && - value == whileOp.getAfterArguments()[slot]; - } - return false; -} - -// Resolve a selected legacy BlockData state after dialect-conversion remapping. -// The legacy representation is intentionally limited to reinterpret-cast -// descriptors and integer tensors. An opaque memref returned by func.call is a -// complete value, not an analyzable descriptor producer, and must never be -// guessed across the call boundary. -static FailureOr -resolveBlockDataStateValue(Value originalValue, - ConversionPatternRewriter &rewriter) { - Value value = originalValue; - if (Value mappedValue = rewriter.getRemappedValue(originalValue)) { - if (originalValue.getDefiningOp() || - originalValue.getDefiningOp() || - originalValue.getDefiningOp()) { - if (!mappedValue.getDefiningOp()) - return failure(); - } else if (auto tensorType = dyn_cast(mappedValue.getType()); - tensorType && - isa(tensorType.getElementType())) { - return failure(); - } - value = mappedValue; - } - - if (value.getDefiningOp() || - isIntegerTensorBlockDataValue(value)) - return value; - return failure(); -} - -template -std::enable_if_t || - std::is_same_v, - LogicalResult> -BlockDataParser::rewriteTerminator( - T op, ConversionPatternRewriter &rewriter, - const llvm::SmallDenseSet &blockArgIdxSet, - ArrayRef iterArgIdxMap, - const llvm::SmallDenseMap &known) { - // Any inserted instruction should be before this yield - OpBuilder::InsertionGuard insertionGuard{rewriter}; - rewriter.setInsertionPoint(op); - - auto adaptor = typename T::Adaptor(op); - ValueRange args; - if constexpr (std::is_same_v) { - args = adaptor.getOperands(); - } else { - args = adaptor.getArgs(); - } - - SmallVector initArgState; - SmallVector operands; - - operands.reserve(op->getNumOperands()); - for (const auto &[oper, newIterArgIdx] : - llvm::zip_equal(args, iterArgIdxMap)) { - if (newIterArgIdx != -1) - operands.push_back(oper); - } - - // For each of the init arg that we added additional Values in for loop, we - // need to add corresponding Values as yield operands. The loop below gathers - // BlockData for those values. - for (auto [i, originalValue] : llvm::enumerate(args)) { - if (blockArgIdxSet.find(i) == blockArgIdxSet.end()) - continue; - - Value knownKey = rewriter.getRemappedValue(originalValue); - if (!knownKey) - knownKey = originalValue; - if (auto knownState = known.find(knownKey); knownState != known.end()) { - initArgState.push_back(knownState->second); - continue; - } - - FailureOr stateValue = - resolveBlockDataStateValue(originalValue, rewriter); - if (failed(stateValue)) { - InFlightDiagnostic diagnostic = op.emitError( - "legacy BlockData loop rewrite cannot analyze carried slot "); - diagnostic << i << " produced by "; - if (Operation *producer = originalValue.getDefiningOp()) - diagnostic << producer->getName(); - else - diagnostic << "a block argument"; - return failure(); - } - - Value v = *stateValue; - auto reintCastOp = v.getDefiningOp(); - - BlockData state; - if (reintCastOp) { - parseReinterpretCast(reintCastOp, state, op.getLoc(), rewriter, known); - } else { - if (failed(parse(v, state, op.getLoc(), rewriter, known))) - return failure(); - } - initArgState.push_back(state); - } - - // For each of the BlockData recorded in the last step, extract value - // that correspond to offset and stride for each dimension and append - // them to yield operands. - for (auto state : initArgState) { - for (auto offset : state.getOffsetsRef()) { - // offsets can be IntAttr zeroes, since reinterpret_cast collapses - // them for the input memref, and the for loop may not update - // offsets other than offsets[0]. Create constants Values for those - // zeroes. - if (isa(offset)) { - auto constOffset = cast(offset); - assert(isa(constOffset) && - dyn_cast(constOffset).getInt() == 0 && - "attribute offsets should be zeroes"); - auto constOp = rewriter.create( - op.getLoc(), rewriter.getIndexAttr(0)); - operands.push_back(constOp.getResult()); - } else { - operands.push_back(cast(offset)); - } - } - - auto sizesRef = state.getSizesRef(); - size_t dimIdx = 0; - for (OpFoldResult stride : state.getStridesRef()) { - if (isa(stride)) { - auto constStride = cast(stride); - assert(isa(constStride) && - "attribute strides should be IntegerAttr"); - - auto strideVal = dyn_cast(constStride).getInt(); - bool isSizeOne = - (dimIdx < sizesRef.size() && isa(sizesRef[dimIdx]) && - cast(cast(sizesRef[dimIdx])).getInt() == - 1); - assert((strideVal == 1 || (strideVal == 0 && isSizeOne)) && - "attribute strides should be ones"); - auto constOp = rewriter.create( - op.getLoc(), rewriter.getIndexAttr(1)); - operands.push_back(constOp.getResult()); - } else { - operands.push_back(cast(stride)); - } - dimIdx++; - } - } - - // Yield is a terminator op that must be at the end of the function - rewriter.setInsertionPointAfter(op); - Operation *newOp; - if constexpr (std::is_same_v) { - newOp = rewriter.replaceOpWithNewOp(op, operands); - } else { - newOp = rewriter.replaceOpWithNewOp(op, op.getCondition(), - operands); - } - - assert(op->getNumResults() == 0); - - LLVM_DEBUG({ - llvm::dbgs() << "new terminator: "; - newOp->print(llvm::dbgs(), OpPrintingFlags().printGenericOpForm()); - llvm::dbgs() << "\n"; - }); - return success(); -} - -// This function is util function for rewriteLoopOp that -// check if given regionIterArg is used with given condition -bool isUsedWithCondition(Value v, std::function cond, - int depth = 0, - llvm::SmallSetVector *visited = nullptr) { - llvm::SmallSetVector localVisited; - if (!visited) { - visited = &localVisited; - } - - if (visited->contains(v)) { - return false; - } - visited->insert(v); - - for (auto &use : v.getUses()) { - auto *user = use.getOwner(); - if (user->hasAttr(ConverterUtils::discreteAttrName) || - isa(user)) - continue; - if (cond(&use)) - return true; - if (auto loopOp = dyn_cast(user); - loopOp && !loopOp->hasAttr("ExtractedLoadOrStore")) { - Value tiedArg = loopOp.getTiedLoopRegionIterArg(&use); - if (tiedArg && isUsedWithCondition(tiedArg, cond, depth + 1, visited)) - return true; - } else if (auto yieldOp = dyn_cast(user); - yieldOp && !isa(user->getParentOp())) { - if (depth && isUsedWithCondition(yieldOp->getParentOp()->getResult( - use.getOperandNumber()), - cond, depth - 1, visited)) - return true; - } else if (auto conditionOp = dyn_cast(user); - conditionOp && use.getOperandNumber() > 0) { - auto whileOp = cast(conditionOp->getParentOp()); - if (depth && - isUsedWithCondition(whileOp->getResult(use.getOperandNumber() - 1), - cond, depth - 1, visited)) - return true; - if (isUsedWithCondition( - whileOp.getAfterArguments()[use.getOperandNumber() - 1], cond, - depth, visited)) - return true; - } - for (auto res : user->getResults()) { - if (isUsedWithCondition(res, cond, depth, visited)) - return true; - } - } - return false; -} - -// A loop-carried value may be consumed through a region argument, through a -// while after-argument, or only after the loop result. Check every semantic -// view of the same carried slot so an identity tensor.cast after the loop -// cannot hide an AddPtr/load/store use from the decomposition decision. -bool isLoopCarriedValueUsedWithCondition( - LoopLikeOpInterface loopOp, unsigned index, - const std::function &condition) { - if (index >= loopOp.getRegionIterArgs().size() || - index >= loopOp->getNumResults()) - return false; - if (isUsedWithCondition(loopOp.getRegionIterArgs()[index], condition)) - return true; - if (auto whileOp = dyn_cast(loopOp.getOperation())) { - if (index < whileOp.getAfterArguments().size() && - isUsedWithCondition(whileOp.getAfterArguments()[index], condition)) - return true; - } - return isUsedWithCondition(loopOp->getResult(index), condition); -} - -// The legacy BlockData route only needs address uses reached from inside the -// loop. Reuse the established traversal for the region arguments, but do not -// start another traversal from the loop result: a numerical carrier can feed -// an unrelated reduction after the loop and eventually reach an address use. -static bool isLoopCarriedAddressValueUsed( - LoopLikeOpInterface loopOp, unsigned index, - const std::function &condition) { - if (index >= loopOp.getRegionIterArgs().size() || - index >= loopOp->getNumResults()) - return false; - if (isUsedWithCondition(loopOp.getRegionIterArgs()[index], condition)) - return true; - if (auto whileOp = dyn_cast(loopOp.getOperation())) { - if (index < whileOp.getAfterArguments().size() && - isUsedWithCondition(whileOp.getAfterArguments()[index], condition)) - return true; - } - return false; -} - -bool needsLegacyBlockDataLoopRewrite(LoopLikeOpInterface loopOp) { - auto hasPointerValue = [&](auto values) { - return llvm::any_of(values, [&](Value value) { - return containsLegacyTritonPointer(value.getType()); - }); - }; - - auto isScalarPointerValue = [](Value value) { - auto pointerType = dyn_cast(value.getType()); - return pointerType && !isa(pointerType.getPointeeType()); - }; - auto isTensorPointerValue = [](Value value) { - auto pointerType = dyn_cast(value.getType()); - return pointerType && isa(pointerType.getPointeeType()); - }; - - SmallVector boundaryValues; - boundaryValues.append(loopOp.getInits().begin(), loopOp.getInits().end()); - boundaryValues.append(loopOp.getRegionIterArgs().begin(), - loopOp.getRegionIterArgs().end()); - boundaryValues.append(loopOp->getResults().begin(), - loopOp->getResults().end()); - if (auto whileOp = dyn_cast(loopOp.getOperation())) - boundaryValues.append(whileOp.getAfterArguments().begin(), - whileOp.getAfterArguments().end()); - - bool hasScalarPointer = llvm::any_of(boundaryValues, isScalarPointerValue); - bool hasTensorPointer = llvm::any_of(boundaryValues, isTensorPointerValue); - // An opaque scalar-pointer loop has no legacy BlockData schema. Leave it - // untouched so the conversion driver reports an ordinary unsupported - // boundary instead of entering the descriptor parser with a non-memref - // source. Tensor-pointer loops continue through their established path. - if (hasScalarPointer && !hasTensorPointer) - return false; - - // Inspect the original SCF boundary, before the dialect converter remaps a - // Triton pointer to a reinterpret-cast memref. Such loops still belong to the - // legacy pointer conversion even though their converted init may look like - // an ordinary memref later in rewriteLoopOp(). - if (hasPointerValue(loopOp.getInits()) || - hasPointerValue(loopOp.getRegionIterArgs()) || - llvm::any_of(loopOp->getResultTypes(), containsLegacyTritonPointer)) - return true; - if (auto whileOp = dyn_cast(loopOp.getOperation())) { - if (hasPointerValue(whileOp.getAfterArguments())) - return true; - } - - // Preserve the pre-existing tensor-offset/mask path. These pointer-free - // integer tensors are nevertheless expanded by BlockData when they feed the - // address operands recognized by the legacy analysis. - for (auto [index, init] : llvm::enumerate(loopOp.getInits())) { - auto tensorType = dyn_cast(init.getType()); - if (!tensorType) - continue; - auto integerType = dyn_cast(tensorType.getElementType()); - if (!integerType || integerType.getWidth() == 1) - continue; - if (isLoopCarriedAddressValueUsed(loopOp, index, [](OpOperand *use) { - Operation *user = use->getOwner(); - return isa(user) || - (isa(user) && use->getOperandNumber() == 1) || - (isa(user) && use->getOperandNumber() == 2); - })) - return true; - } - return false; -} - -// This function is util function for rewriteLoopOp that create value from data. -// Assume data is structured, and from regionIterArg from LoopLikeOpInterface. -// -// For example, -// -// %7 = scf.for %arg2 = %c0_i32 to %c3_i32 step %c1_i32 iter_args(%arg3 = %4) -> -// (tensor<128xi32>) : i32 { -// %8 = tt.addptr %5, %arg3 : tensor<128x!tt.ptr>, tensor<128xi32> -// ... -// } -// -// is converted to -// -// %7 = scf.for %arg2 = %c0_i32 to %c3_i32 step %c1_i32 iter_args(%arg3 = %4, -// %arg4 = %5, %arg5 = %6) -> (tensor<128xi32>) : i32 { -// %scalarOffset = arith.index_cast %arg4 : index to i32 -// %scalarStride = arith.index_cast %arg5 : index to i32 -// ... -// %newRes = arith.addi %offset, %stride : tensor<128xi32> -// %8 = tt.addptr %5, %newRes : tensor<128x!tt.ptr>, tensor<128xi32> -// } -Value createFromData(RankedTensorType resType, const BlockData &data, - const Location &loc, OpBuilder &builder, - bool isMaskIterArg) { - auto resShape = resType.getShape(); - Value newRes = nullptr; - for (size_t i = 0; i < resShape.size(); i++) { - auto axisType = - RankedTensorType::get({resShape[i]}, resType.getElementType()); - auto axisI32Type = - RankedTensorType::get({resShape[i]}, builder.getIntegerType(32)); - Value axisValue = - builder.create(loc, axisI32Type, 0, resShape[i]); - if (axisType != axisI32Type) { - axisValue = builder.create(loc, axisType, axisValue); - } - Value offset = cast(data.getOffset(i)); - Value offsetValue = builder.create( - loc, resType.getElementType(), offset); - offsetValue = builder.create(loc, axisType, offsetValue); - Value stride = cast(data.getStride(i)); - if (!isMaskIterArg) { - Value strideValue = builder.create( - loc, resType.getElementType(), stride); - strideValue = builder.create(loc, axisType, strideValue); - axisValue = builder.create(loc, axisValue, strideValue); - } - axisValue = builder.create(loc, axisValue, offsetValue); - - for (size_t j = 0; j < resShape.size(); j++) { - if (i != j) - axisValue = builder.create(loc, axisValue, j); - } - axisValue = builder.create(loc, resType, axisValue); - if (newRes) { - newRes = builder.create(loc, newRes, axisValue); - } else { - newRes = axisValue; - } - } - return newRes; -} - -LogicalResult -BlockDataParser::rewriteLoopOp(LoopLikeOpInterface op, - ConversionPatternRewriter &rewriter, - llvm::SmallDenseMap &known, - ArrayRef onlyIndexTensorSlots) { - SmallVector newInitArgs; - SmallVector iterArgIdxMap; - SmallVector maskIterArgs; - int64_t argCnt = 0; - - SmallVector, 5> initArgIndexIfBlockData; - SmallVector, 5> knownPtrsTmp; - llvm::SmallDenseSet blockArgIdxSet; - - // Create a new list of init args - for (auto [i, arg] : llvm::enumerate(op.getInits())) { - auto mappedV = rewriter.getRemappedValue(arg); - memref::ReinterpretCastOp reintCastOp; - maskIterArgs.push_back(false); - - // If this init arg is supposed to be remapped, use the remapped - // value instead. - // In addition, if this init arg is a memref created by a reinterpret_cast - // or a tensor of index, there is a chance that it will be used in addptr. - // Create BlockData for each such init arg. - if (mappedV) { - // TODO: - // Passing a block argument pointer directly into a for loop not - // supported. - assert(!(isa(mappedV) && - isa(mappedV.getType())) && - "cannot take pointer block argument as init arg for for loop"); - if (auto reinterpretCastOp = - mappedV.getDefiningOp()) { - // Record memref::ReinterpretCastOp - reintCastOp = reinterpretCastOp; - newInitArgs.push_back(mappedV); - iterArgIdxMap.push_back(argCnt++); - } else { - newInitArgs.push_back(mappedV); - iterArgIdxMap.push_back(argCnt++); - } - } else { - newInitArgs.push_back(arg); - iterArgIdxMap.push_back(argCnt++); - } - - auto indexTensor = - isa(arg.getType()) && - isa(cast(arg.getType()).getElementType()) && - cast(cast(arg.getType()).getElementType()) - .getWidth() != 1 && - isLoopCarriedValueUsedWithCondition(op, i, [](OpOperand *use) { - auto *user = use->getOwner(); - return isa(user) || - (isa(user) && use->getOperandNumber() == 1) || - (isa(user) && use->getOperandNumber() == 2); - }); - - if (!onlyIndexTensorSlots.empty() && - !llvm::is_contained(onlyIndexTensorSlots, static_cast(i))) - indexTensor = false; - - // Handle memref::ReinterpretCastOp and tensor specially - if (!reintCastOp && !indexTensor) - continue; - - BlockData data; - if (reintCastOp) { - parseReinterpretCast(reintCastOp, data, op.getLoc(), rewriter, - llvm::SmallDenseMap(0)); - } else { - if (failed(parse(arg, data, op.getLoc(), rewriter, - llvm::SmallDenseMap(0)))) - return failure(); - } - - maskIterArgs[i] = - indexTensor && - isLoopCarriedValueUsedWithCondition(op, i, [](OpOperand *use) { - auto *user = use->getOwner(); - return (isa(user) && use->getOperandNumber() == 1) || - (isa(user) && use->getOperandNumber() == 2); - }); - - if (indexTensor) { - newInitArgs.back() = nullptr; - iterArgIdxMap.back() = -1; - argCnt--; - } - - // Record the BlockData for later processing - initArgIndexIfBlockData.push_back(std::make_pair(i, data)); - } - - // Validate every structural backedge before creating constants or a new - // loop. This prevents a partially expanded signature when an opaque memref - // source cannot provide offsets and strides. - auto preflightBackedge = [&](ValueRange values, - StringRef edgeName) -> LogicalResult { - for (auto [index, data] : initArgIndexIfBlockData) { - (void)data; - if (index < 0 || static_cast(index) >= values.size()) { - op->emitError("legacy BlockData preflight found a missing ") - << edgeName << " value for carried slot " << index; - return failure(); - } - Value value = values[index]; - if (!canAnalyzeLegacyBlockDataBackedge(op, index, value)) { - InFlightDiagnostic diagnostic = op->emitError( - "legacy BlockData preflight cannot analyze carried slot "); - diagnostic << index << " on the " << edgeName << " edge produced by "; - if (Operation *producer = value.getDefiningOp()) - diagnostic << producer->getName(); - else - diagnostic << "an unknown source"; - return failure(); - } - } - return success(); - }; - if (auto forOp = dyn_cast(op.getOperation())) { - if (failed(preflightBackedge(forOp.getYieldedValues(), "yield"))) - return failure(); - } else if (auto whileOp = dyn_cast(op.getOperation())) { - if (failed(preflightBackedge(whileOp.getConditionOp().getArgs(), - "condition")) || - failed(preflightBackedge(whileOp.getYieldOp().getOperands(), "yield"))) - return failure(); - } - - // Set insertion point to be before the for loop for new variables passed - // into the new loop. - auto origIp = rewriter.saveInsertionPoint(); - rewriter.setInsertionPoint(op); - - // For each of the BlockData recorded in the last step, insert new - // instructions to describe offset and stride for each dimension and append - // them to init args - for (auto [i, data] : initArgIndexIfBlockData) { - // For each dimension, if the corresponding offset and stride is an - // integer attribute, create a constant value and append them at the - // end of init arg list, which is prepared for calculate layout info with - // loop interation index - for (auto &dataOffset : data.getOffsetsRef()) { - if (isa(dataOffset)) { - auto constDataOffset = cast(dataOffset); - assert(isa(constDataOffset)); - auto constOp = rewriter.create( - op.getLoc(), rewriter.getIndexAttr( - dyn_cast(constDataOffset).getInt())); - newInitArgs.push_back(constOp.getResult()); - dataOffset = constOp.getResult(); - } else { - assert(isa(cast(dataOffset).getType())); - newInitArgs.push_back(cast(dataOffset)); - } - } - - for (auto &dataStride : data.getStridesRef()) { - if (isa(dataStride)) { - auto constDataStride = cast(dataStride); - assert(isa(constDataStride)); - auto constOp = rewriter.create( - op.getLoc(), rewriter.getIndexAttr( - dyn_cast(constDataStride).getInt())); - newInitArgs.push_back(constOp.getResult()); - dataStride = constOp.getResult(); - } else { - assert(isa(cast(dataStride).getType())); - newInitArgs.push_back(cast(dataStride)); - } - } - - // Note that we want the knownPtrs to be indexed by block arg, but we - // only have index for now. Also, the blockdata we record is the init - // arg, but want to to use newly created block arg. These block args - // are not created yet. We will translate this mapping later. - knownPtrsTmp.push_back(std::make_pair(i, data)); - blockArgIdxSet.insert(i); - - // If the original init arg is a memref produced by reinterpret_cast, - // create a new memref using new strides and offsets created above. - // This produces a canonicalized memref, which will match what the - // for loop generates if it modifies the memref. E.g., original - // reinterpret_cast can produce a memref with const stride: - // - memref<4x256xbf16, affine_map<(d0, d1)[s0, s1] -> (d0 * 256 + - // s0 + d1 - // * s1)>> - // The new reinterpret_cast will always have dynamic stride and - // offset: - // - memref<4x256xbf16, affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s1 - // + s0 + d1 * s2)>> - if (newInitArgs[i] && - newInitArgs[i].getDefiningOp()) { - SmallVector resultShape; - for (auto size : data.getSizesRef()) { - auto constSize = getConstantIntValue(size); - assert(constSize && "expected constant size"); - resultShape.push_back(constSize.value()); - } - - // In current block data layout info, strides and offsets must be dynamic - // value - FailureOr castOp = - data.createCastOp(resultShape, op.getLoc(), rewriter); - if (failed(castOp)) - return op.emitOpError( - "could not materialize a loop-carried memref from the source type"); - if (resultShape.size() > 1) { - auto originalOffset = dyn_cast(data.getOffsetsRef()[0]); - for (auto &offsets : newInitArgs) { - if (offsets == originalOffset) { - offsets = (*castOp).getOffsets()[0]; - break; - } - } - data.getOffsetsRef()[0] = (*castOp).getOffsets()[0]; - } - - LLVM_DEBUG({ - llvm::dbgs() << "new reinterpret_cast with dynamic sizes " - "and offsets:"; - (*castOp).print(llvm::dbgs(), OpPrintingFlags().printGenericOpForm()); - llvm::dbgs() << "\n"; - }); - - newInitArgs[i] = (*castOp).getResult(); - } - } - - rewriter.restoreInsertionPoint(origIp); - IRMapping mapping; - - // Create a new LoopOp that uses updated init args and same loop body - LoopLikeOpInterface newOp; - auto newInits = to_vector( - make_filter_range(newInitArgs, [](Value v) { return v != nullptr; })); - auto commonBodyBuilder = [&](OpBuilder &b, Location loc, bool useInit, - ValueRange newRegionArgs, Region ®ion, - Block::BlockArgListType regionArgs, - ArrayRef isUsedForRegionArgs, - ArrayRef maskIterArgs) { - auto newArgIter = newRegionArgs.begin(); - for (const auto &[regionArg, isUsedForRegionArg] : - llvm::zip(regionArgs, isUsedForRegionArgs)) { - if (isUsedForRegionArg) { - mapping.map(regionArg, *newArgIter); - ++newArgIter; - } - } - - // Convert the book-keeping data structure to use the correct key and value. - // Key is converted from init arg index to newly created block arg, and - // Value's BlockData fields are converted from init arg to newly created - // block arg - - // TODO: remove (useInit = true) logic after supporting make_tensor_ptr - if (useInit) { - for (auto [i, data] : knownPtrsTmp) { - for (auto &offset : data.getOffsetsRef()) { - offset = *newArgIter; - ++newArgIter; - } - - for (auto &stride : data.getStridesRef()) { - stride = *newArgIter; - ++newArgIter; - } - - auto regionArg = regionArgs[i]; - auto key = mapping.lookupOrNull(regionArg); - if (!key) { - // Create IndexTensor regionArg from computed offset and stride data - key = createFromData(cast(regionArg.getType()), - data, op.getLoc(), rewriter, maskIterArgs[i]); - mapping.map(regionArg, key); - } - known.insert(std::make_pair(key, data)); - } - } else { - for (auto [i, isUsedForRegionArg] : - llvm::enumerate(isUsedForRegionArgs)) { - if (!isUsedForRegionArg) { - BlockData data; - auto regionArg = regionArgs[i]; - auto regionArgType = cast(regionArg.getType()); - data.getOffsetsRef().resize(regionArgType.getRank()); - data.getStridesRef().resize(regionArgType.getRank()); - for (auto &offset : data.getOffsetsRef()) { - offset = *newArgIter; - ++newArgIter; - } - for (auto &dim : regionArgType.getShape()) { - data.getSizesRef().push_back(rewriter.getIndexAttr(dim)); - } - for (auto &stride : data.getStridesRef()) { - stride = *newArgIter; - ++newArgIter; - } - - auto key = mapping.lookupOrNull(regionArg); - if (!key) { - // Create IndexTensor regionArg from computed offset and stride data - key = createFromData(regionArgType, data, op.getLoc(), rewriter, - maskIterArgs[i]); - mapping.map(regionArg, key); - } - known.insert(std::make_pair(key, data)); - } - } - } - - for (auto &bodyOp : region.getOps()) - b.clone(bodyOp, mapping); - }; - for (const auto &[initArg, newInitArg] : - llvm::zip(op.getInits(), newInitArgs)) { - if (newInitArg) { - mapping.map(initArg, newInitArg); - } - } - SmallVector newResults; - SmallVector markerSlotMap; - if (auto forOp = dyn_cast(op.getOperation())) { - SmallVector usedForRegionArgs; - for (auto newInitArg : newInitArgs) { - usedForRegionArgs.push_back(newInitArg ? true : false); - } - newOp = rewriter.create( - forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(), - forOp.getStep(), newInits, - [&](OpBuilder &b, Location loc, Value iv, ValueRange args) { - mapping.map(forOp.getInductionVar(), iv); - commonBodyBuilder(b, loc, true, args, forOp.getRegion(), - op.getRegionIterArgs(), usedForRegionArgs, - maskIterArgs); - }); - - // Replace only the results that correspond to the original scf.for - auto newResultIter = newOp->result_begin(); - rewriter.setInsertionPointAfter(newOp); - for (const auto &[res, regionArg, newIterArgIdx, mask] : - llvm::zip_equal(op->getResults(), op.getRegionIterArgs(), - iterArgIdxMap, maskIterArgs)) { - if (newIterArgIdx != -1) { - newResults.push_back(*newResultIter); - ++newResultIter; - } else { - auto key = mapping.lookup(regionArg); - auto data = known.at(key); - for (auto &offset : data.getOffsetsRef()) - offset = - newOp.getTiedLoopResult(cast(cast(offset))); - for (auto &stride : data.getStridesRef()) - stride = - newOp.getTiedLoopResult(cast(cast(stride))); - auto newRes = - createFromData(cast(regionArg.getType()), data, - op.getLoc(), rewriter, mask); - newResults.push_back(newRes); - } - } - markerSlotMap = iterArgIdxMap; - } else if (auto whileOp = dyn_cast(op.getOperation())) { - SmallVector resultTypes; - SmallVector usedForBeforeRegionArgs; - SmallVector usedForAfterRegionArgs; - llvm::SmallDenseSet blockArgIdxSetForAfter; - SmallVector iterArgIdxMapForAfter; - SmallVector maskIterArgsForAfter(whileOp->getNumResults()); - - int64_t indexCnt = 0; - - for (auto newInitArg : newInitArgs) { - usedForBeforeRegionArgs.push_back(newInitArg ? true : false); - } - for (size_t i = 0; i < whileOp->getNumResults(); i++) { - auto resType = whileOp->getResultTypes()[i]; - auto indexTensor = - isa(resType) && - isa(cast(resType).getElementType()) && - isLoopCarriedValueUsedWithCondition(whileOp, i, [](OpOperand *use) { - auto *user = use->getOwner(); - return isa(user) || - (isa(user) && - use->getOperandNumber() == 1) || - (isa(user) && use->getOperandNumber() == 2); - }); - if (indexTensor) { - indexCnt += 2 * cast(resType).getRank(); - usedForAfterRegionArgs.push_back(false); - iterArgIdxMapForAfter.push_back(-1); - maskIterArgsForAfter[i] = - isLoopCarriedValueUsedWithCondition(whileOp, i, [](OpOperand *use) { - auto *user = use->getOwner(); - return (isa(user) && - use->getOperandNumber() == 1) || - (isa(user) && - use->getOperandNumber() == 2); - }); - blockArgIdxSetForAfter.insert(i); - } else { - resultTypes.push_back(resType); - usedForAfterRegionArgs.push_back(true); - iterArgIdxMapForAfter.push_back(argCnt++); - } - } - resultTypes.append(indexCnt, rewriter.getIndexType()); - newOp = rewriter.create( - whileOp.getLoc(), resultTypes, newInits, - [&](OpBuilder &b, Location loc, ValueRange args) { - commonBodyBuilder(b, loc, true, args, whileOp.getBefore(), - whileOp.getBeforeArguments(), - usedForBeforeRegionArgs, maskIterArgs); - }, - [&](OpBuilder &b, Location loc, ValueRange args) { - commonBodyBuilder(b, loc, false, args, whileOp.getAfter(), - whileOp.getAfterArguments(), usedForAfterRegionArgs, - maskIterArgsForAfter); - }); - - auto newResultIter = newOp->result_begin(); - rewriter.setInsertionPointAfter(newOp); - for (const auto &[res, regionArg, newIterArgIdx, mask] : - llvm::zip_equal(op->getResults(), whileOp.getAfterArguments(), - iterArgIdxMapForAfter, maskIterArgsForAfter)) { - if (newIterArgIdx != -1) { - newResults.push_back(*newResultIter); - ++newResultIter; - } else { - auto key = mapping.lookup(regionArg); - auto data = known.at(key); - for (auto &offset : data.getOffsetsRef()) - offset = newOp->getResult( - cast(cast(offset)).getArgNumber()); - for (auto &stride : data.getStridesRef()) - stride = newOp->getResult( - cast(cast(stride)).getArgNumber()); - auto newRes = - createFromData(cast(regionArg.getType()), data, - op.getLoc(), rewriter, mask); - newResults.push_back(newRes); - } - } - - auto conditionOp = - cast(newOp.getOperation()).getConditionOp(); - if (failed(rewriteTerminator(conditionOp, rewriter, blockArgIdxSetForAfter, - iterArgIdxMapForAfter, known))) - return failure(); - markerSlotMap = iterArgIdxMapForAfter; - } - - if (!newOp || newResults.size() != op->getNumResults()) - return op->emitError( - "loop rewrite produced a result list with incompatible arity"); - - // Copy all attributes from op to newOp - newOp->setAttrs(op->getAttrs()); - if (!onlyIndexTensorSlots.empty()) { - if (Attribute marker = - op->getAttr(controlflow::kPointerDescriptorBoundaryAttr)) { - auto descriptorSlots = dyn_cast(marker); - if (!descriptorSlots) { - rewriter.eraseOp(newOp.getOperation()); - return op->emitError("invalid pointer descriptor boundary marker"); - } - - SmallVector remappedSlots; - remappedSlots.reserve(descriptorSlots.size()); - for (int32_t slot : descriptorSlots.asArrayRef()) { - if (slot < 0 || static_cast(slot) >= markerSlotMap.size() || - markerSlotMap[slot] < 0 || - markerSlotMap[slot] > std::numeric_limits::max()) { - rewriter.eraseOp(newOp.getOperation()); - return op->emitError( - "pointer descriptor slot cannot be remapped after range rewrite"); - } - remappedSlots.push_back(static_cast(markerSlotMap[slot])); - } - newOp->setAttr(controlflow::kPointerDescriptorBoundaryAttr, - DenseI32ArrayAttr::get(op->getContext(), remappedSlots)); - } - } - rewriter.replaceOp(op, newResults); - - // Update the loop body. Manually invoke the rewrite logic on addptr and yield - // in the loop body, so we can take advantage of the states we built up - for (auto *region : newOp.getLoopRegions()) { - for (auto &bodyOp : region->getOps()) { - if (isDistributedTypeCustomOp(&bodyOp)) { - rewriteStructuredCustomOp(&bodyOp, rewriter); - } else if (auto addptrOp = dyn_cast(bodyOp)) { - // FIXME: Constructed adaptor here does not hold the transformed op - // info. - auto adaptor = triton::AddPtrOp::Adaptor(addptrOp); - if (failed(rewriteAddPtr(addptrOp, adaptor, rewriter, known))) - return failure(); - } else if (auto advanceOp = dyn_cast(bodyOp)) { - if (failed(rewriteAdvanceOp(advanceOp, rewriter, known))) - return failure(); - } else if (auto makeTensorPtrOp = - dyn_cast(bodyOp)) { - ConversionPatternRewriter::InsertionGuard guard(rewriter); - rewriter.setInsertionPoint(makeTensorPtrOp); - if (failed(rewriteMakeTensorPtrOp( - makeTensorPtrOp, - rewriter.getRemappedValue(makeTensorPtrOp.getBase()), rewriter, - known))) - return failure(); - } else if (auto loopOp = dyn_cast(bodyOp); - loopOp && !loopOp->hasAttr("ExtractedLoadOrStore")) { - ConversionPatternRewriter::InsertionGuard guard(rewriter); - rewriter.setInsertionPoint(loopOp); - // Remove UnhandledLoopOp attr before process - rewriter.modifyOpInPlace( - loopOp, [&]() { loopOp->removeAttr("UnhandledLoopOp"); }); - if (failed(rewriteLoopOp(loopOp, rewriter, known))) - return failure(); - } - } - } - - if (!op.getRegionIterArgs().empty()) { - auto yieldOp = cast( - newOp.getLoopRegions().back()->back().getTerminator()); - if (failed(rewriteTerminator(yieldOp, rewriter, blockArgIdxSet, - iterArgIdxMap, known))) - return failure(); - } - - LLVM_DEBUG({ - llvm::dbgs() << "new loop\n"; - newOp.getOperation()->print(llvm::dbgs(), - OpPrintingFlags().printGenericOpForm()); - llvm::dbgs() << "\n"; - }); - return success(); -} - -/// @brief Rewrite the triton::AddPtrOp to handle unstructured memory access. -/// @param op The triton::AddPtrOp to be rewritten. -/// @param adaptor The adaptor of the triton::AddPtrOp, used to get operands. -/// @param rewriter The pattern rewriter used to modify the IR. -/// @param data The BlockData containing information about the memory access. -LogicalResult BlockDataParser::rewriteAddPtrToUnstrucMemAcc( - triton::AddPtrOp op, triton::AddPtrOp::Adaptor &adaptor, - ConversionPatternRewriter &rewriter, BlockData &data) { - auto loc = op.getLoc(); - auto &offsets = data.getOffsetsRef(); - auto &blockSizes = data.getSizesRef(); - auto &strides = data.getStridesRef(); - Value ptrOffset = adaptor.getOffset(); - Value zeroIdx = - rewriter.create(loc, rewriter.getIndexAttr(0)); - Value oneIdx = - rewriter.create(loc, rewriter.getIndexAttr(1)); - auto addptrRes = op.getResult(); - assert(addptrRes.hasOneUse() && "Invalid: tt.addptr has multiple users"); - auto loadOp = *(addptrRes.user_begin()); - - // Prepare empty tensor for loop based scalar load - // FIXME: We use cast here because addptr must return tensor>. - // True? - auto resTy = cast(addptrRes.getType()); - auto resEPtrTy = resTy.getElementType(); - auto resETy = cast(resEPtrTy).getPointeeType(); - Value loaded = rewriter.create(loc, blockSizes, resETy); - SmallVector initArgs; - initArgs.push_back(loaded); - - SmallVector forLBs; - SmallVector forUBs; - SmallVector forSteps; - for (auto &s : offsets) { - forLBs.push_back(zeroIdx); - } - for (auto &s : blockSizes) { - forUBs.push_back(getValueOrCreateConstantIndexOp(rewriter, loc, s)); - } - for (auto &s : strides) { - forSteps.push_back(oneIdx); - } - SmallVector ivs; - bool castFailed = false; - OpBuilder builder(op); - auto loop = createNestedLoops( - builder, loc, 0, blockSizes.size(), forLBs, forUBs, forSteps, ivs, - initArgs, - [&](OpBuilder &bB, Location bLoc, SmallVector &allIVs, - ValueRange iterArgs) { - OpBuilder::InsertionGuard g(bB); - bB.setInsertionPointToStart(bB.getBlock()); - - Value scalarOffsetRaw = - bB.create(bLoc, ptrOffset, allIVs); - Value scalarOffset = bB.create( - bLoc, bB.getIndexType(), scalarOffsetRaw); - OpFoldResult baseOffset = bB.getIndexAttr(0); - for (auto ofr : data.getOffsetsRef()) { - baseOffset = - addOpFoldResult(baseOffset, ofr, bLoc, bB, bB.getIndexType()); - } - Value baseVal = getValueOrCreateConstantIndexOp(bB, bLoc, baseOffset); - Value combinedOffset = - bB.create(bLoc, baseVal, scalarOffset); - // Replace offset & size. Only single element. - data.getOffsetsRef().clear(); - data.getOffsetsRef().push_back(combinedOffset); - data.getSizesRef().clear(); - data.getSizesRef().push_back(bB.getIndexAttr(1)); - data.getStridesRef().clear(); - data.getStridesRef().push_back(bB.getIndexAttr(1)); - FailureOr castOp = - data.createCastOp({1}, bLoc, bB); - if (failed(castOp)) { - castFailed = true; - return; - } - rewriter.replaceOp(op, (*castOp).getResult()); - // Move tt.load using this tt.addptr into this block - loadOp->moveAfter((*castOp).getOperation()); - loadOp->setAttr("IndirectLoad", UnitAttr::get(op.getContext())); - bB.create(bLoc, iterArgs); - }); - if (castFailed) - return op.emitOpError("could not materialize the indirect pointer source"); - return success(); -} - -} // namespace triton -} // namespace mlir diff --git a/third_party/ascend/lib/TritonToLinalg/CMakeLists.txt b/third_party/ascend/lib/TritonToLinalg/CMakeLists.txt deleted file mode 100644 index 9d01a991dd..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/CMakeLists.txt +++ /dev/null @@ -1,40 +0,0 @@ -add_triton_library(TritonToLinalg - TritonToLinalgPass.cpp - LoadStoreConverter.cpp - FunctionConverter.cpp - ArgMinMaxConverter.cpp - TritonOpConverter.cpp - HoistBroadcast.cpp - BlockPtrAnalysis.cpp - UseAnalysis.cpp - ImplicitPermute.cpp - DescriptorConverter.cpp - MarkTensorKindPass.cpp - DevicePrintOffsetRewrite.cpp - DeduplicateDebugNopsPass.cpp - CanonicalizeDebugLocationsPass.cpp - - DEPENDS - TritonToLinalgConversionPassIncGen - - LINK_LIBS PUBLIC - MLIRArithDialect - MLIRDialectUtils - MLIRIR - MLIRLLVMDialect - MLIRMathDialect - MLIRPass - MLIRTensorDialect - MLIRTransforms - MLIRSupport - BiShengIRHIVMDialect - BiShengIRScopeDialect - TritonIR - TritonTransforms - TritonAnalysis - TritonMemoryAccessAnalysis - TritonToGraph - MLIRTritonNPUUtils - MLIRSCFTransforms - MLIRLinalgTransforms -) diff --git a/third_party/ascend/lib/TritonToLinalg/CanonicalizeDebugLocationsPass.cpp b/third_party/ascend/lib/TritonToLinalg/CanonicalizeDebugLocationsPass.cpp deleted file mode 100644 index f767d1c947..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/CanonicalizeDebugLocationsPass.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -//===----------------------------------------------------------------------===// -// CanonicalizeDebugLocationsPass -// -// Collapses call-site locations whose callee is an inlined Triton stdlib -// helper (under site-packages) down to their caller (user-file) frame, so the -// emitted DWARF references the user's source line instead of the library. -// Fixes phantom line-table rows such as standard.py:306 produced by an inlined -// tl.sum reduction. -// -// Run BEFORE DeduplicateDebugNopsPass so NOP anchors are deduped against -// already-canonicalized (user-frame) locations. -// -// Gated by env var LLVM_EXTRACT_DI_LOCAL_VARIABLES=1. -//===----------------------------------------------------------------------===// - -#include "TritonToLinalg/CanonicalizeDebugLocationsPass.h" -#include "Utils/DebugUtils.h" - -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/Location.h" -#include "mlir/Pass/Pass.h" - -#include "llvm/Support/Debug.h" - -#define DEBUG_TYPE "canonicalize-debug-locations" - -using namespace mlir; -using namespace mlir::triton; - -namespace { - -struct CanonicalizeDebugLocationsPass - : public PassWrapper> { - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(CanonicalizeDebugLocationsPass) - - StringRef getArgument() const final { return "canonicalize-debug-locations"; } - - StringRef getDescription() const final { - return "Collapse inlined stdlib call-site locations to the user frame."; - } - - void runOnOperation() override { - if (!mlir::triton::debug::isDebugNopEnabled()) - return; - - [[maybe_unused]] unsigned rewritten = 0; - getOperation().walk([&](Operation *op) { - Location oldLoc = op->getLoc(); - Location newLoc = mlir::triton::debug::collapseForeignCallsites(oldLoc); - if (newLoc != oldLoc) { - op->setLoc(newLoc); - ++rewritten; - } - }); - - LLVM_DEBUG(llvm::dbgs() << "[canon-debug-locs] rewrote " << rewritten - << " foreign call-site locations\n"); - } -}; - -} // namespace - -namespace mlir { -namespace triton { - -std::unique_ptr createCanonicalizeDebugLocationsPass() { - return std::make_unique(); -} - -} // namespace triton -} // namespace mlir diff --git a/third_party/ascend/lib/TritonToLinalg/DeduplicateDebugNopsPass.cpp b/third_party/ascend/lib/TritonToLinalg/DeduplicateDebugNopsPass.cpp deleted file mode 100644 index 47ac32aab4..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/DeduplicateDebugNopsPass.cpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -//===----------------------------------------------------------------------===// -// DeduplicateDebugNopsPass -// -// Walks each function and removes redundant `llvm.inline_asm "nop"` ops that -// were inserted as DWARF anchors by the triton-to-linalg converters. Keeps -// the first NOP per unique source line; erases the rest. -// -// Source lines are resolved with the shared user-frame resolver -// (debug::unwrapToUserFileLineCol), so a NOP carrying an inlined call-site loc -// keys on the user's line rather than the inlined callee. Run this AFTER -// CanonicalizeDebugLocationsPass. -// -// Runtime-gated via the LLVM_EXTRACT_DI_LOCAL_VARIABLES environment variable. -//===----------------------------------------------------------------------===// - -#include "TritonToLinalg/DeduplicateDebugNopsPass.h" -#include "Utils/DebugUtils.h" - -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/Location.h" -#include "mlir/Pass/Pass.h" - -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/Support/Debug.h" - -#define DEBUG_TYPE "dedup-debug-nops" - -using namespace mlir; -using namespace mlir::triton; - -namespace { - -struct DeduplicateDebugNopsPass - : public PassWrapper> { - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(DeduplicateDebugNopsPass) - - StringRef getArgument() const final { return "deduplicate-debug-nops"; } - - StringRef getDescription() const final { - return "Deduplicate llvm.inline_asm 'nop' debug-anchor ops by source line."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - - void runOnOperation() override { - // Opt-in via env var; do nothing in production builds. - if (!mlir::triton::debug::isDebugNopEnabled()) - return; - - ModuleOp moduleOp = getOperation(); - // Only read inside LLVM_DEBUG, which is a no-op under NDEBUG. - [[maybe_unused]] unsigned totalDropped = 0; - [[maybe_unused]] unsigned totalKept = 0; - - moduleOp.walk([&](func::FuncOp func) { - // Per-function dedup: key = (filename, line). Column intentionally - // ignored -- for debugger stepping behaviour, two NOPs on the same line - // at different columns are equivalent. The StringRef points into a - // StringAttr owned by the MLIRContext, which outlives this pass. - llvm::DenseSet> seen; - llvm::SmallVector toErase; - - func.walk([&](Operation *op) { - if (!mlir::triton::debug::isDebugNop(op)) - return; - - // Resolve to the user's source line (caller frame for call sites). - FileLineColLoc flc = - mlir::triton::debug::unwrapToUserFileLineCol(op->getLoc()); - if (!flc) { - // No resolvable source location; leave the NOP alone. - LLVM_DEBUG(llvm::dbgs() - << "[dedup-nops] NOP with no FileLineColLoc, keeping: " - << *op << "\n"); - ++totalKept; - return; - } - - auto key = std::make_pair(flc.getFilename().getValue(), flc.getLine()); - if (!seen.insert(key).second) { - toErase.push_back(op); - } - }); - - totalKept += seen.size(); - totalDropped += toErase.size(); - - for (Operation *op : toErase) - op->erase(); - }); - - LLVM_DEBUG(llvm::dbgs() - << "[dedup-nops] dropped " << totalDropped - << " duplicate NOPs, kept " << totalKept << " unique anchors\n"); - } -}; - -} // namespace - -namespace mlir { -namespace triton { - -std::unique_ptr createDeduplicateDebugNopsPass() { - return std::make_unique(); -} - -} // namespace triton -} // namespace mlir diff --git a/third_party/ascend/lib/TritonToLinalg/DescriptorConverter.cpp b/third_party/ascend/lib/TritonToLinalg/DescriptorConverter.cpp deleted file mode 100644 index d204d0ea3d..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/DescriptorConverter.cpp +++ /dev/null @@ -1,566 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "TritonToLinalg/DescriptorConverter.h" -#include "TritonToLinalg/BlockPtrAnalysis.h" -#include "TritonToLinalg/MaskAnalysis.h" -#include "TritonToLinalg/TritonOpConverter.h" -#include "TritonToLinalg/TritonToLinalgPass.h" -#include "Utils/Utils.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/LogicalResult.h" -#include "llvm/Support/raw_ostream.h" -#include - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/ValueRange.h" -#include "mlir/Transforms/DialectConversion.h" - -namespace DescriptorConverter { -using namespace mlir; -using namespace triton; - -bool hasATensorDescriptorType(mlir::TypeRange types) { - return llvm::any_of(types, [](mlir::Type t) { - return llvm::isa(t); - }); -} - -Descriptor unpackDescriptor(TensorDescType type, Value desc, - ConversionPatternRewriter &rewriter) { - auto makeDescOp = desc.getDefiningOp(); - assert(makeDescOp && "Descriptor must be defined by MakeTensorDescOp"); - - Descriptor res; - - res.base = makeDescOp.getBase(); - for (auto s : makeDescOp.getShape()) { - res.shape.push_back(rewriter.createOrFold( - makeDescOp.getLoc(), rewriter.getI64Type(), s)); - } - for (auto st : makeDescOp.getStrides()) { - res.strides.push_back(rewriter.createOrFold( - makeDescOp.getLoc(), rewriter.getI64Type(), st)); - } - res.padding = makeDescOp.getPaddingAttr(); - - return res; -} - -SmallVector computeOrder(ArrayRef shape) { - SmallVector order; - int rank = shape.size(); - order.reserve(rank); - // default by [dims - 1, ..., 0] - for (int i = rank - 1; i >= 0; --i) { - order.push_back(i); - } - return order; -} - -DenseI32ArrayAttr getFullBoundaryCheckAttr(ConversionPatternRewriter &rewriter, - ArrayRef shape) { - SmallVector boundaryCheck; - boundaryCheck.reserve(shape.size()); - for (int32_t dim = 0; dim < static_cast(shape.size()); ++dim) { - boundaryCheck.push_back(dim); - } - return rewriter.getDenseI32ArrayAttr(boundaryCheck); -} - -Value expandOffsets(OpBuilder &builder, Location loc, - ArrayRef blockShape, Value offsets, unsigned dim) { - Value expandedResult = offsets; - for (size_t j = 0; j < blockShape.size(); ++j) { - if (j == dim) { - continue; - } - expandedResult = - builder.create(loc, expandedResult, j); - } - - return expandedResult; -} - -Value getExpandedOffsetWithRange(OpBuilder &builder, const Location &loc, - ArrayRef blockShape, - Value offset, unsigned dim) { - // Add range - auto indexI32RowType = - RankedTensorType::get({blockShape[dim]}, builder.getI32Type()); - auto indexRowType = - RankedTensorType::get({blockShape[dim]}, builder.getI64Type()); - Value splatOffset = - builder.create(loc, indexRowType, offset); - Value range = builder.create(loc, indexI32RowType, 0, - blockShape[dim]); - Value i64Range = builder.create(loc, indexRowType, range); - - Value offsets = builder.create(loc, splatOffset, i64Range); - return expandOffsets(builder, loc, blockShape, offsets, dim); -} - -Value generatePtrFromOffsetRanges(OpBuilder &builder, Location loc, - ArrayRef blockShape, - Descriptor &desc, ValueRange offsets) { - assert(blockShape.size() == desc.shape.size()); - assert(blockShape.size() == offsets.size()); - auto indexTensorType = - RankedTensorType::get(blockShape, builder.getI64Type()); - auto ptrType = cast(desc.base.getType()); - auto ptrTensorType = RankedTensorType::get(blockShape, ptrType); - - // Generate offsets per dimension - Value ptr = builder.create(loc, ptrTensorType, desc.base); - for (unsigned i = 0; i < blockShape.size(); ++i) { - // We must splat strides into the expanded shape not a row for retaining - // the divisibility information given by strides - Value splatStride = builder.create( - loc, offsets[i].getType(), desc.strides[i]); - Value offsetWithStride = - builder.create(loc, offsets[i], splatStride); - Value broadcasted = builder.create( - loc, indexTensorType, offsetWithStride); - - // Add to the pointer - ptr = - builder.create(loc, ptrTensorType, ptr, broadcasted); - } - - return ptr; -} - -Value generatePtr(OpBuilder &builder, const Location &loc, - ArrayRef blockShape, Descriptor &desc, - ValueRange offsets) { - assert(blockShape.size() == desc.shape.size()); - assert(blockShape.size() == offsets.size()); - SmallVector offsetRanges; - for (unsigned i = 0; i < blockShape.size(); ++i) { - auto offsetWithRange = - getExpandedOffsetWithRange(builder, loc, blockShape, offsets[i], i); - offsetRanges.push_back(offsetWithRange); - } - - return generatePtrFromOffsetRanges(builder, loc, blockShape, desc, - offsetRanges); -} - -Value generateMaskFromOffsetRanges(OpBuilder &builder, const Location &loc, - ArrayRef blockShape, - Descriptor &desc, ValueRange offsetRanges) { - assert(blockShape.size() == desc.shape.size()); - assert(blockShape.size() == offsetRanges.size()); - - // Generate mask per dimension - auto maskTensorType = RankedTensorType::get(blockShape, builder.getI1Type()); - Value mask; - for (std::size_t i = 0; i < blockShape.size(); ++i) { - auto offsetWithRange = offsetRanges[i]; - - // Compare with lower bound - Value lowerBound = builder.create( - loc, builder.getI64Type(), 0); - Value splatLowerBound = builder.create( - loc, offsetWithRange.getType(), lowerBound); - Value cmpLower = builder.create( - loc, arith::CmpIPredicate::sge, offsetWithRange, splatLowerBound); - - // Compare with upper bound - Value splatUpperBound = builder.create( - loc, offsetWithRange.getType(), desc.shape[i]); - Value cmpUpper = builder.create( - loc, arith::CmpIPredicate::slt, offsetWithRange, splatUpperBound); - - // And and broadcast - Value andResult = builder.create(loc, cmpLower, cmpUpper); - Value broadcasted = - builder.create(loc, maskTensorType, andResult); - - // And up all results - if (!mask) { - mask = broadcasted; - } else { - mask = builder.create(loc, mask, broadcasted); - } - } - - return mask; -} - -Value generateMask(OpBuilder &builder, const Location &loc, - ArrayRef blockShape, Descriptor &desc, - ValueRange offsets) { - assert(blockShape.size() == desc.shape.size()); - assert(blockShape.size() == offsets.size()); - SmallVector offsetRanges; - for (unsigned i = 0; i < blockShape.size(); ++i) { - auto offsetWithRange = - getExpandedOffsetWithRange(builder, loc, blockShape, offsets[i], i); - offsetRanges.push_back(offsetWithRange); - } - - return generateMaskFromOffsetRanges(builder, loc, blockShape, desc, - offsetRanges); -} - -SmallVector castToI64(OpBuilder &builder, - mlir::ValueRange values) { - auto i64Type = builder.getI64Type(); - return llvm::map_to_vector(values, [&](mlir::Value v) { - return builder.createOrFold(v.getLoc(), i64Type, v); - }); -} - -LogicalResult DescriptorLoadConverter::matchAndRewrite( - triton::DescriptorLoadOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - const auto blockShape = op.getDesc().getType().getBlockType().getShape(); - auto descTy = op.getDesc().getType(); - auto indices = op.getIndices(); - - // 1. unpack descriptor - auto desc = unpackDescriptor(descTy, adaptor.getDesc(), rewriter); - - // 2. create make_tensor_ptr - SmallVector tensorShapeValues; - for (auto dim : blockShape) { - tensorShapeValues.push_back(static_cast(dim)); - } - Value tensorPtr = - rewriter.create(loc, - desc.base, // base - desc.shape, // shape - desc.strides, // strides - indices, // offset - tensorShapeValues, // tensorShape - computeOrder(blockShape) // order - ); - // 3. replace tt.load - auto boundaryCheck = getFullBoundaryCheckAttr(rewriter, blockShape); - triton::PaddingOptionAttr padding = desc.padding; - auto cache = triton::CacheModifierAttr::get(rewriter.getContext(), - triton::CacheModifier::NONE); - auto evict = triton::EvictionPolicyAttr::get(rewriter.getContext(), - triton::EvictionPolicy::NORMAL); - auto isVolatile = rewriter.getBoolAttr(false); - - if (auto a = op->getAttrOfType("cache")) - cache = a; - if (auto a = op->getAttrOfType("evict")) - evict = a; - if (auto a = op->getAttrOfType("isVolatile")) - isVolatile = a; - - auto newLoad = rewriter.create( - loc, descTy.getSignlessBlockType(), tensorPtr, - Value(), // mask - Value(), // other - boundaryCheck, padding, cache, evict, isVolatile); - - rewriter.replaceOp(op, newLoad.getResult()); - - return success(); -} - -LogicalResult DescriptorStoreConverter::matchAndRewrite( - triton::DescriptorStoreOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - const auto blockShape = op.getDesc().getType().getBlockType().getShape(); - auto descTy = op.getDesc().getType(); - auto indices = op.getIndices(); - - // 1. unpack descriptor - auto desc = unpackDescriptor(descTy, adaptor.getDesc(), rewriter); - - // 2. create make_tensor_ptr - SmallVector tensorShapeValues; - for (auto dim : blockShape) { - tensorShapeValues.push_back(static_cast(dim)); - } - Value tensorPtr = - rewriter.create(loc, - desc.base, // base - desc.shape, // shape - desc.strides, // strides - indices, // offset - tensorShapeValues, // tensorShape - computeOrder(blockShape) // order - ); - - // 3. replace tt.store - Value valueToStore = adaptor.getSrc(); - - auto maskType = RankedTensorType::get(blockShape, rewriter.getI1Type()); - rewriter.create(loc, - DenseElementsAttr::get(maskType, true)); - auto boundaryCheck = getFullBoundaryCheckAttr(rewriter, blockShape); - auto cacheModifier = triton::CacheModifierAttr::get( - rewriter.getContext(), triton::CacheModifier::NONE); - auto evictionPolicy = triton::EvictionPolicyAttr::get( - rewriter.getContext(), triton::EvictionPolicy::NORMAL); - - auto newStore = rewriter.create(loc, tensorPtr, valueToStore, - Value(), // mask - boundaryCheck, cacheModifier, - evictionPolicy); - - rewriter.eraseOp(op); - return success(); -} - -LogicalResult DescriptorScatterConverter::matchAndRewrite( - triton::DescriptorScatterOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto descTy = op.getDesc().getType(); - auto srcType = cast(op.getSrc().getType()); - const auto rowBlockShape = descTy.getSignlessBlockType().getShape(); - - auto desc = unpackDescriptor(descTy, adaptor.getDesc(), rewriter); - SmallVector tensorShapeValues; - tensorShapeValues.reserve(rowBlockShape.size()); - for (auto dim : rowBlockShape) { - tensorShapeValues.push_back(static_cast(dim)); - } - - auto zeroIndex = rewriter.create(loc, 0); - auto oneIndex = rewriter.create(loc, 1); - Value rowUpperBound; - if (srcType.isDynamicDim(0)) { - rowUpperBound = rewriter.create(loc, adaptor.getSrc(), 0); - } else { - rowUpperBound = - rewriter.create(loc, srcType.getShape()[0]); - } - auto rowBoundaryCheck = getFullBoundaryCheckAttr(rewriter, rowBlockShape); - auto cacheModifier = triton::CacheModifierAttr::get( - rewriter.getContext(), triton::CacheModifier::NONE); - auto evictionPolicy = triton::EvictionPolicyAttr::get( - rewriter.getContext(), triton::EvictionPolicy::NORMAL); - - auto loop = rewriter.create( - loc, zeroIndex, rowUpperBound, oneIndex, ValueRange{}, - [&](OpBuilder &nestedBuilder, Location nestedLoc, Value rowIv, - ValueRange) { - Value xOffset = nestedBuilder.create( - nestedLoc, adaptor.getXOffsets(), ValueRange{rowIv}); - Value tensorPtr = nestedBuilder.create( - nestedLoc, desc.base, desc.shape, desc.strides, - ValueRange{xOffset, adaptor.getYOffset()}, tensorShapeValues, - computeOrder(rowBlockShape)); - SmallVector extractOffsets{rowIv, - nestedBuilder.getIndexAttr(0)}; - SmallVector extractSizes{ - nestedBuilder.getIndexAttr(rowBlockShape[0]), - nestedBuilder.getIndexAttr(rowBlockShape[1])}; - SmallVector extractStrides{nestedBuilder.getIndexAttr(1), - nestedBuilder.getIndexAttr(1)}; - auto rowValue = nestedBuilder.create( - nestedLoc, adaptor.getSrc(), extractOffsets, extractSizes, - extractStrides); - auto rowStore = nestedBuilder.create( - nestedLoc, tensorPtr, rowValue.getResult(), Value(), - rowBoundaryCheck, cacheModifier, evictionPolicy); - rowStore->setAttr(ConverterUtils::discreteAttrName, - nestedBuilder.getUnitAttr()); - nestedBuilder.create(nestedLoc); - }); - loop->setAttr("ExtractedLoadOrStore", rewriter.getUnitAttr()); - loop->setAttr("hivm.parallel_loop", rewriter.getUnitAttr()); - - rewriter.eraseOp(op); - return success(); -} - -SmallVector filterSegmentSizes(ArrayRef attrs) { - SmallVector filteredAttrs; - llvm::copy_if(attrs, std::back_inserter(filteredAttrs), - [](const NamedAttribute &attr) { - return attr.getName().getValue() != "operandSegmentSizes"; - }); - return filteredAttrs; -} - -LogicalResult DescriptorGatherConverter::matchAndRewrite( - triton::DescriptorGatherOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto descTy = cast(op.getDesc().getType()); - auto resultType = cast(op.getResult().getType()); - const auto blockShape = resultType.getShape(); - const auto rowBlockShape = descTy.getSignlessBlockType().getShape(); - - auto desc = unpackDescriptor(descTy, adaptor.getDesc(), rewriter); - SmallVector tensorShapeValues; - tensorShapeValues.reserve(rowBlockShape.size()); - for (auto dim : rowBlockShape) { - tensorShapeValues.push_back(static_cast(dim)); - } - - auto zeroIndex = rewriter.create(loc, 0); - auto oneIndex = rewriter.create(loc, 1); - Value rowUpperBound = - rewriter.create(loc, adaptor.getXOffsets(), zeroIndex); - auto rowBoundaryCheck = getFullBoundaryCheckAttr(rewriter, rowBlockShape); - auto cache = triton::CacheModifierAttr::get(rewriter.getContext(), - triton::CacheModifier::NONE); - auto evict = triton::EvictionPolicyAttr::get(rewriter.getContext(), - triton::EvictionPolicy::NORMAL); - auto isVolatile = rewriter.getBoolAttr(false); - - if (auto attr = op->getAttrOfType("cache")) - cache = attr; - if (auto attr = op->getAttrOfType("evict")) - evict = attr; - if (auto attr = op->getAttrOfType("isVolatile")) - isVolatile = attr; - - SmallVector dynamicResultSizes; - dynamicResultSizes.reserve(resultType.getNumDynamicDims()); - for (const auto &[dim, size] : llvm::enumerate(blockShape)) { - if (!ShapedType::isDynamic(size)) - continue; - if (dim == 0) { - dynamicResultSizes.push_back(rowUpperBound); - continue; - } - dynamicResultSizes.push_back( - rewriter.create(loc, rowBlockShape[dim])); - } - - auto initialTensor = rewriter.create( - loc, blockShape, resultType.getElementType(), dynamicResultSizes); - auto loop = rewriter.create( - loc, zeroIndex, rowUpperBound, oneIndex, - ValueRange{initialTensor.getResult()}, - [&](OpBuilder &nestedBuilder, Location nestedLoc, Value rowIv, - ValueRange iterArgs) { - Value xOffset = nestedBuilder.create( - nestedLoc, adaptor.getXOffsets(), ValueRange{rowIv}); - Value tensorPtr = nestedBuilder.create( - nestedLoc, desc.base, desc.shape, desc.strides, - ValueRange{xOffset, adaptor.getYOffset()}, tensorShapeValues, - computeOrder(rowBlockShape)); - auto rowLoad = nestedBuilder.create( - nestedLoc, descTy.getSignlessBlockType(), tensorPtr, - Value(), // mask - Value(), // other - rowBoundaryCheck, desc.padding, cache, evict, isVolatile); - for (const auto &attr : filterSegmentSizes(op->getAttrs())) { - if (!rowLoad->hasAttr(attr.getName())) { - rowLoad->setAttr(attr.getName(), attr.getValue()); - } - } - rowLoad->setAttr(ConverterUtils::discreteAttrName, - nestedBuilder.getUnitAttr()); - - auto insertSlice = nestedBuilder.create( - nestedLoc, rowLoad.getResult(), iterArgs[0], - SmallVector{rowIv, nestedBuilder.getIndexAttr(0)}, - SmallVector{ - nestedBuilder.getIndexAttr(rowBlockShape[0]), - nestedBuilder.getIndexAttr(rowBlockShape[1])}, - SmallVector{nestedBuilder.getIndexAttr(1), - nestedBuilder.getIndexAttr(1)}); - insertSlice->setAttr(ConverterUtils::discreteAttrName, - nestedBuilder.getUnitAttr()); - nestedBuilder.create(nestedLoc, insertSlice.getResult()); - }); - loop->setAttr("ExtractedLoadOrStore", rewriter.getUnitAttr()); - - rewriter.replaceOp(op, loop.getResult(0)); - return success(); -} - -std::optional translateReduceKind(DescriptorReduceKind kind, - TensorDescType ty) { - auto scalarTy = ty.getBlockType().getElementType(); - switch (kind) { - case DescriptorReduceKind::ADD: - return scalarTy.isInteger() ? RMWOp::ADD : RMWOp::FADD; - case DescriptorReduceKind::MIN: - if (scalarTy.isUnsignedInteger()) { - return RMWOp::UMIN; - } else if (scalarTy.isSignedInteger()) { - return RMWOp::MIN; - } - return {}; - case DescriptorReduceKind::MAX: - if (scalarTy.isUnsignedInteger()) { - return RMWOp::UMAX; - } else if (scalarTy.isSignedInteger()) { - return RMWOp::MAX; - } - return {}; - case DescriptorReduceKind::AND: - return RMWOp::AND; - case DescriptorReduceKind::OR: - return RMWOp::OR; - case DescriptorReduceKind::XOR: - return RMWOp::XOR; - default: - break; - } - return {}; -} - -LogicalResult DescriptorReduceConverter::matchAndRewrite( - triton::DescriptorReduceOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto descTy = op.getDesc().getType(); - const auto blockShape = descTy.getBlockType().getShape(); - auto desc = unpackDescriptor(descTy, adaptor.getDesc(), rewriter); - auto offsets = castToI64(rewriter, op.getIndices()); - auto rmwOp = translateReduceKind(op.getKind(), descTy); - if (!rmwOp) { - std::string msgstring; - llvm::raw_string_ostream msg(msgstring); - msg << "Cannot fallback on descriptor atomic op, unsupported for type " - << descTy.getBlockType().getElementType(); - return op->emitError(msgstring); - } - - rewriter.create( - loc, descTy.getSignlessBlockType(), *rmwOp, - generatePtr(rewriter, loc, blockShape, desc, offsets), op.getSrc(), - generateMask(rewriter, loc, blockShape, desc, offsets), - MemSemantic::ACQUIRE_RELEASE, MemSyncScope::GPU); - op.erase(); - return success(); -} - -} // namespace DescriptorConverter diff --git a/third_party/ascend/lib/TritonToLinalg/DevicePrintOffsetRewrite.cpp b/third_party/ascend/lib/TritonToLinalg/DevicePrintOffsetRewrite.cpp deleted file mode 100644 index b23a01c8e6..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/DevicePrintOffsetRewrite.cpp +++ /dev/null @@ -1,591 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/DevicePrintOffsetRewrite.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Dominance.h" -#include "mlir/IR/OpDefinition.h" - -#include "llvm/ADT/APInt.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" - -#include - -namespace mlir { -namespace triton { -namespace { - -static bool hasOnlyPrintUsers(Value v) { - if (v.use_empty()) - return false; - for (Operation *u : v.getUsers()) { - auto callOp = dyn_cast(u); - if (!callOp) - return false; - if (!callOp.getCallee().starts_with("triton_print")) - return false; - } - return true; -} - -static Value materializeIndex(OpBuilder &bd, Location loc, OpFoldResult ofr) { - if (auto attr = ofr.dyn_cast()) { - int64_t v = cast(attr).getInt(); - return bd.create(loc, v); - } - auto val = llvm::cast(ofr); - Type ty = val.getType(); - if (ty.isIndex()) - return val; - if (isa(ty)) - return bd.create(loc, bd.getIndexType(), val); - return nullptr; -} - -static bool isStaticEqual(OpFoldResult ofr, int64_t target) { - if (auto attr = ofr.dyn_cast()) - return cast(attr).getInt() == target; - return false; -} - -class ScalarChainWalker { -public: - ScalarChainWalker(OpBuilder &bd, Location loc) : bd(bd), loc(loc) {} - - Value emit(Value v, ArrayRef indices) { - Operation *def = v.getDefiningOp(); - if (!def) - return nullptr; - - if (auto cstOp = dyn_cast(def)) - return scalarFromConstant(cstOp); - if (auto fillOp = dyn_cast(def)) - return scalarFromFill(fillOp); - if (auto generic = dyn_cast(def)) - return scalarFromMakeRange(generic, indices); - - if (auto bc = dyn_cast(def)) - return emitBroadcast(bc, indices); - if (auto exp = dyn_cast(def)) - return emitExpandShape(exp, indices); - if (auto col = dyn_cast(def)) - return emitCollapseShape(col, indices); - - if (def->getNumOperands() != 2) - return nullptr; - Value lhs = emit(def->getOperand(0), indices); - Value rhs = emit(def->getOperand(1), indices); - if (!lhs || !rhs) - return nullptr; - return emitBinaryArith(def, lhs, rhs); - } - -private: - Value scalarFromConstant(arith::ConstantOp op) { - auto i32Ty = bd.getI32Type(); - if (auto dense = dyn_cast(op.getValue())) - if (dense.isSplat()) - return bd.create( - loc, dense.getSplatValue().getSExtValue(), i32Ty.getWidth()); - if (auto intAttr = dyn_cast(op.getValue())) - return bd.create(loc, intAttr.getInt(), - i32Ty.getWidth()); - return nullptr; - } - - Value scalarFromFill(linalg::FillOp op) { - if (op.getInputs().empty()) - return nullptr; - Value sc = op.getInputs()[0]; - auto i32Ty = bd.getI32Type(); - auto scTy = sc.getType(); - - if (auto cstOp = sc.getDefiningOp()) { - if (auto intAttr = dyn_cast(cstOp.getValue())) - return bd.create(loc, intAttr.getInt(), - i32Ty.getWidth()); - return nullptr; - } - if (scTy.isInteger(32)) - return sc; - if (scTy.isIndex()) - return bd.create(loc, i32Ty, sc); - if (auto intTy = dyn_cast(scTy)) { - if (intTy.getWidth() < 32) - return bd.create(loc, i32Ty, sc); - if (intTy.getWidth() > 32) - return bd.create(loc, i32Ty, sc); - } - return nullptr; - } - - Value scalarFromMakeRange(linalg::GenericOp op, ArrayRef indices) { - auto i32Ty = bd.getI32Type(); - if (!op->hasAttr("tt.from_make_range")) - return nullptr; - if (indices.size() != 1) - return nullptr; - int64_t off = 0; - if (auto a = op->getAttrOfType("tt.make_range_offset")) - off = a.getInt(); - Value v32 = bd.create(loc, i32Ty, indices[0]); - if (off != 0) { - Value oc = bd.create(loc, off, i32Ty.getWidth()); - v32 = bd.create(loc, v32, oc); - } - return v32; - } - - Value emitBroadcast(linalg::BroadcastOp op, ArrayRef indices) { - auto bcDims = op.getDimensions(); - llvm::DenseSet dropSet(bcDims.begin(), bcDims.end()); - SmallVector inputIdx; - for (size_t k = 0; k < indices.size(); ++k) - if (!dropSet.count(k)) - inputIdx.push_back(indices[k]); - return emit(op.getInput(), inputIdx); - } - - Value emitExpandShape(tensor::ExpandShapeOp op, ArrayRef indices) { - auto resTy = cast(op.getResult().getType()); - SmallVector inputIdx; - for (auto group : op.getReassociationIndices()) { - Value combined; - for (size_t i = 0; i < group.size(); ++i) { - int64_t d = group[i]; - int64_t weight = 1; - for (size_t j = i + 1; j < group.size(); ++j) - weight *= resTy.getDimSize(group[j]); - Value term = indices[d]; - if (weight != 1) { - Value w = bd.create(loc, weight); - term = bd.create(loc, term, w); - } - combined = - !combined ? term : bd.create(loc, combined, term); - } - if (!combined) - combined = bd.create(loc, 0); - inputIdx.push_back(combined); - } - return emit(op.getSrc(), inputIdx); - } - - Value emitCollapseShape(tensor::CollapseShapeOp op, ArrayRef indices) { - auto inTy = cast(op.getSrc().getType()); - SmallVector inputIdx(inTy.getRank()); - Value c0 = bd.create(loc, 0); - - auto reassoc = op.getReassociationIndices(); - for (size_t outDim = 0; outDim < indices.size(); ++outDim) { - auto &group = reassoc[outDim]; - Value out = indices[outDim]; - - int64_t leftmostNonOne = -1; - for (size_t i = 0; i < group.size(); ++i) - if (inTy.getDimSize(group[i]) != 1) { - leftmostNonOne = (int64_t)i; - break; - } - - for (size_t i = 0; i < group.size(); ++i) { - int64_t d = group[i]; - int64_t size = inTy.getDimSize(d); - if (size == 1) { - inputIdx[d] = c0; - continue; - } - int64_t weight = 1; - for (size_t j = i + 1; j < group.size(); ++j) - weight *= inTy.getDimSize(group[j]); - Value v = out; - if (weight != 1) { - Value w = bd.create(loc, weight); - v = bd.create(loc, v, w); - } - if ((int64_t)i != leftmostNonOne) { - Value s = bd.create(loc, size); - v = bd.create(loc, v, s); - } - inputIdx[d] = v; - } - } - return emit(op.getSrc(), inputIdx); - } - - Value emitBinaryArith(Operation *def, Value lhs, Value rhs) { - if (isa(def)) - return bd.create(loc, lhs, rhs); - if (isa(def)) - return bd.create(loc, lhs, rhs); - if (isa(def)) - return bd.create(loc, lhs, rhs); - if (isa(def)) - return bd.create(loc, lhs, rhs); - if (isa(def)) - return bd.create(loc, lhs, rhs); - if (isa(def)) - return bd.create(loc, lhs, rhs); - if (isa(def)) - return bd.create(loc, lhs, rhs); - return nullptr; - } - - OpBuilder &bd; - Location loc; -}; - -static void emitLoopsAndPrint( - OpBuilder &builder, Location loc, func::FuncOp scalarPrintFn, - ArrayRef shape, - llvm::function_ref)> - scalarGen) { - auto i32Ty = builder.getI32Type(); - Value c0 = builder.create(loc, 0); - Value c1 = builder.create(loc, 1); - - SmallVector ivs(shape.size(), c0); - OpBuilder bd = builder; - for (size_t k = 0; k < shape.size(); ++k) { - if (shape[k] <= 1) - continue; - Value upper = bd.create(loc, shape[k]); - auto forOp = bd.create(loc, c0, upper, c1); - ivs[k] = forOp.getInductionVar(); - bd.setInsertionPoint(forOp.getBody()->getTerminator()); - } - - Value scalar = scalarGen(bd, loc, ivs); - if (!scalar) - return; - if (scalar.getType().isIndex()) - scalar = bd.create(loc, i32Ty, scalar); - else if (!scalar.getType().isInteger(32)) - return; - bd.create(loc, scalarPrintFn, ValueRange{scalar}); -} - -enum class ChainKind { - Unsupported, - Linear, - Linearized, -}; - -struct OffsetChainInfo { - ChainKind kind = ChainKind::Unsupported; - StringRef rejectReason; - SmallVector shape; - Value chainRoot; - SmallVector strides; - OpFoldResult baseOffset; -}; - -static void classifyChain(Value root, bool &canWalk, bool &hasDivMod) { - canWalk = true; - hasDivMod = false; - - llvm::DenseSet visited; - std::function walk = [&](Value v) { - if (!canWalk) - return; - auto ty = dyn_cast(v.getType()); - if (!ty) - return; - if (!ty.hasStaticShape()) { - canWalk = false; - return; - } - - Operation *def = v.getDefiningOp(); - if (!def) { - canWalk = false; - return; - } - if (!visited.insert(def).second) - return; - - if (auto fillOp = dyn_cast(def)) { - if (fillOp.getInputs().empty() || - !fillOp.getInputs()[0].getType().isIntOrIndex()) - canWalk = false; - return; - } - if (isa(def)) - return; - if (auto g = dyn_cast(def)) { - if (!g->hasAttr("tt.from_make_range")) - canWalk = false; - return; - } - if (auto bc = dyn_cast(def)) { - walk(bc.getInput()); - return; - } - if (auto exp = dyn_cast(def)) { - auto resTy = cast(exp.getResult().getType()); - for (auto group : exp.getReassociationIndices()) - for (int64_t d : group) - if (ShapedType::isDynamic(resTy.getDimSize(d))) { - canWalk = false; - return; - } - walk(exp.getSrc()); - return; - } - if (auto col = dyn_cast(def)) { - auto inTy = cast(col.getSrc().getType()); - for (auto group : col.getReassociationIndices()) - for (int64_t d : group) - if (ShapedType::isDynamic(inTy.getDimSize(d))) { - canWalk = false; - return; - } - walk(col.getSrc()); - return; - } - if (isa( - def)) { - hasDivMod = true; - for (Value operand : def->getOperands()) - walk(operand); - return; - } - if (isa(def)) { - for (Value operand : def->getOperands()) - walk(operand); - return; - } - canWalk = false; - }; - - walk(root); -} - -static memref::ReinterpretCastOp -findMatchingCast(func::FuncOp funcOp, ArrayRef targetShape) { - memref::ReinterpretCastOp best; - unsigned bestScore = 0; - funcOp.walk([&](memref::ReinterpretCastOp c) { - auto resTy = dyn_cast(c.getResult().getType()); - if (!resTy) - return; - if (resTy.getShape() != targetShape) - return; - unsigned score = 0; - if (auto blockArg = dyn_cast(c.getSource())) { - unsigned idx = blockArg.getArgNumber(); - if (auto argDict = funcOp.getArgAttrDict(idx)) - if (argDict.contains("tt.tensor_kind")) - score = 1; - } - if (score >= bestScore) { - bestScore = score; - best = c; - } - }); - return best; -} - -static OffsetChainInfo analyzeChain(Value arg, func::FuncOp funcOp, - Operation *anchorOp, - DominanceInfo &domInfo) { - OffsetChainInfo info; - - auto argTy = dyn_cast(arg.getType()); - if (!argTy) { - info.rejectReason = "arg not RankedTensorType"; - return info; - } - if (!argTy.hasStaticShape()) { - info.rejectReason = "arg has dynamic shape"; - return info; - } - if (!hasOnlyPrintUsers(arg)) { - info.rejectReason = "arg has non-print users"; - return info; - } - - info.shape.assign(argTy.getShape().begin(), argTy.getShape().end()); - info.chainRoot = arg; - - bool canWalk = false; - bool hasDivMod = false; - classifyChain(arg, canWalk, hasDivMod); - if (!canWalk) { - info.rejectReason = "chain contains unsupported op"; - return info; - } - - if (argTy.getRank() > 1 && !hasDivMod) { - auto castOp = findMatchingCast(funcOp, argTy.getShape()); - if (castOp) { - SmallVector mixedStrides = castOp.getMixedStrides(); - SmallVector mixedOffsets = castOp.getMixedOffsets(); - if (mixedOffsets.size() == 1 && - (int64_t)mixedStrides.size() == argTy.getRank()) { - auto domOk = [&](OpFoldResult ofr) -> bool { - if (auto val = ofr.dyn_cast()) - return domInfo.dominates(val, anchorOp); - return true; - }; - bool domAllOk = domOk(mixedOffsets[0]); - if (domAllOk) - for (auto s : mixedStrides) - if (!domOk(s)) { - domAllOk = false; - break; - } - if (domAllOk) { - info.kind = ChainKind::Linear; - info.strides = std::move(mixedStrides); - info.baseOffset = mixedOffsets[0]; - return info; - } - } - } - } - - info.kind = ChainKind::Linearized; - return info; -} - -struct RewriteCandidate { - func::CallOp callOp; - OffsetChainInfo info; -}; - -static void rewriteCandidate(RewriteCandidate &cand, ModuleOp moduleOp) { - auto i32Ty = IntegerType::get(moduleOp.getContext(), 32); - auto oldFn = dyn_cast_or_null( - SymbolTable::lookupSymbolIn(moduleOp, cand.callOp.getCalleeAttr())); - if (!oldFn) - return; - - auto newFnTy = FunctionType::get(moduleOp.getContext(), {i32Ty}, {}); - oldFn.setFunctionType(newFnTy); - - OpBuilder b(cand.callOp); - Location loc = cand.callOp.getLoc(); - OffsetChainInfo &info = cand.info; - - switch (info.kind) { - case ChainKind::Linear: { - auto gen = [&info](OpBuilder &bd, Location loc, - ArrayRef ivs) -> Value { - Value acc; - if (!isStaticEqual(info.baseOffset, 0)) { - acc = materializeIndex(bd, loc, info.baseOffset); - if (!acc) - return nullptr; - } - for (size_t k = 0; k < info.shape.size(); ++k) { - if (info.shape[k] <= 1) - continue; - Value idx = ivs[k]; - if (!isStaticEqual(info.strides[k], 1)) { - Value s = materializeIndex(bd, loc, info.strides[k]); - if (!s) - return nullptr; - idx = bd.create(loc, idx, s); - } - if (!acc) - acc = idx; - else - acc = bd.create(loc, acc, idx); - } - if (!acc) - acc = bd.create(loc, 0); - return acc; - }; - emitLoopsAndPrint(b, loc, oldFn, info.shape, gen); - break; - } - - case ChainKind::Linearized: { - Value root = info.chainRoot; - auto gen = [root](OpBuilder &bd, Location loc, - ArrayRef ivs) -> Value { - ScalarChainWalker walker(bd, loc); - return walker.emit(root, ivs); - }; - emitLoopsAndPrint(b, loc, oldFn, info.shape, gen); - break; - } - - case ChainKind::Unsupported: - return; - } - - cand.callOp.erase(); -} - -static void rewriteOneFunc(func::FuncOp funcOp) { - auto moduleOp = funcOp->getParentOfType(); - if (!moduleOp) - return; - - DominanceInfo domInfo(funcOp); - SmallVector candidates; - - funcOp.walk([&](func::CallOp callOp) { - if (!callOp.getCallee().starts_with("triton_print")) - return; - if (callOp.getNumOperands() != 1) - return; - - OffsetChainInfo info = analyzeChain(callOp.getOperand(0), funcOp, - callOp.getOperation(), domInfo); - if (info.kind == ChainKind::Unsupported) - return; - - candidates.push_back({callOp, std::move(info)}); - }); - - for (auto &cand : candidates) - rewriteCandidate(cand, moduleOp); -} - -} // namespace - -void rewriteDevicePrintOffsets(ModuleOp moduleOp) { - moduleOp.walk([&](func::FuncOp funcOp) { - if (funcOp.isPrivate()) - return; - rewriteOneFunc(funcOp); - }); -} - -} // namespace triton -} // namespace mlir diff --git a/third_party/ascend/lib/TritonToLinalg/FunctionConverter.cpp b/third_party/ascend/lib/TritonToLinalg/FunctionConverter.cpp deleted file mode 100644 index d0f425c1bd..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/FunctionConverter.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/FunctionConverter.h" -#include "ascend/include/Utils/DebugUtils.h" - -namespace FunctionConverter { -using namespace mlir; -using namespace triton; - -LogicalResult GetProgramIDConverter::matchAndRewrite( - triton::GetProgramIdOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto axis = (uint32_t)op.getAxis(); - assert(axis < GetProgramIDConverter::LAUNCH_GRID_RANK && - "Invalid axis for GetProgramIdOp"); - auto func = op->getParentOfType(); - auto numArgs = func.getNumArguments(); - auto id = func.getArgument(numArgs - GetProgramIDConverter::LAUNCH_GRID_RANK + - axis); - - Location pidLoc = op.getLoc(); - insertDebugNop(pidLoc, rewriter); - rewriter.replaceOp(op, id); - return success(); -} - -LogicalResult GetNumProgramsConverter::matchAndRewrite( - triton::GetNumProgramsOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto axis = (uint32_t)op.getAxis(); - assert(axis < GetNumProgramsConverter::LAUNCH_GRID_RANK && - "Invalid axis for GetNumProgramsOp"); - auto func = op->getParentOfType(); - auto numArgs = func.getNumArguments(); - auto id = func.getArgument( - numArgs - GetNumProgramsConverter::LAUNCH_GRID_RANK * 2 + axis); - - Location numProgsLoc = op.getLoc(); - insertDebugNop(numProgsLoc, rewriter); - rewriter.replaceOp(op, id); - return success(); -} -} // namespace FunctionConverter diff --git a/third_party/ascend/lib/TritonToLinalg/HoistBroadcast.cpp b/third_party/ascend/lib/TritonToLinalg/HoistBroadcast.cpp deleted file mode 100644 index d16262e169..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/HoistBroadcast.cpp +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/HoistBroadcast.h" -#include "ascend/include/TritonToLinalg/TritonToLinalgPass.h" -#include "ascend/include/Utils/Utils.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/LogicalResult.h" -#include "llvm/Support/raw_ostream.h" -#include - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/MemRef/Transforms/Passes.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/ValueRange.h" - -namespace HoistBroadcast { -using namespace mlir; -using namespace triton; - -LogicalResult -BroadcastConverter::matchAndRewrite(triton::BroadcastOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - assert(op->getNumResults() == 1 && "BroadcastOp assumes single result"); - - if (!isa(op.getType().getElementType())) { - return rewriter.notifyMatchFailure( - op, "only support hoist broadcast for tt.ptr tensor right now."); - } - auto loc = op.getLoc(); - BroadcastHoister hoister(op); - - if (!hoister.canBroadcast()) { - return failure(); - } - - if (hoister.parse(op.getSrc(), loc, rewriter).failed()) { - return failure(); - } - if (hoister.replaceBroadcastOp(op, rewriter).failed()) { - return failure(); - } - return success(); -} - -BroadcastHoister::BroadcastHoister(triton::BroadcastOp op) { - source = nullptr; - if (findSrc(op.getSrc()).failed()) { - LLVM_DEBUG({ llvm::dbgs() << "No legal source found for broadcast op\n"; }); - } - opToHoist = op; - auto resultType = dyn_cast(op.getType()); - for (size_t i = 0; i < resultType.getShape().size(); ++i) { - tensorSizes.push_back(resultType.getShape()[i]); - } -} - -LogicalResult BroadcastHoister::findSrc(Value operand) { - // ptr tensor can only be defined by AddPtrOp or SplatOp in this converter - // another broadcast to be complemented - if (auto op = operand.getDefiningOp()) { - return findSrc(op.getPtr()); - } else if (auto op = operand.getDefiningOp()) { - source = op.getSrc(); - return success(); - } else { - LLVM_DEBUG({ - llvm::dbgs() << "Unsupported operation in BroadcastHoister::findSrc: " - << *operand.getDefiningOp() << "\n"; - }); - return failure(); - } -} - -LogicalResult BroadcastHoister::parse(Value operand, const Location &loc, - ConversionPatternRewriter &rewriter) { - if (auto op = operand.getDefiningOp()) { - return parseAddptr(op, loc, rewriter); - } else if (auto op = operand.getDefiningOp()) { - return parseSplat(op, loc, rewriter); - } else if (auto op = operand.getDefiningOp()) { - return parseBroadcast(op, loc, rewriter); - } else { - // Handle other cases or throw an error - LLVM_DEBUG({ - llvm::dbgs() << "Unsupported operation in BroadcastHoister::parse: " - << *operand.getDefiningOp() << "\n"; - }); - return failure(); - } -} - -LogicalResult -BroadcastHoister::parseAddptr(triton::AddPtrOp addptrOp, const Location &loc, - ConversionPatternRewriter &rewriter) { - // Implementation for parsing AddptrOp - if (parse(addptrOp.getPtr(), loc, rewriter).failed()) { - return failure(); - } - auto broadcastedPtr = broadcastMap[addptrOp.getPtr()]; - - RankedTensorType offsetType = - dyn_cast(addptrOp.getOffset().getType()); - if (!offsetType || !offsetType.hasStaticShape()) { - LLVM_DEBUG({ - llvm::dbgs() << "Offset must be a ranked tensor with static shape.\n"; - }); - return failure(); - } - - auto elementType = offsetType.getElementType(); - auto broadcastType = RankedTensorType::get({tensorSizes}, elementType); - auto broadcastedOffset = rewriter.create( - loc, broadcastType, addptrOp.getOffset()); - - auto ptrType = dyn_cast(source.getType()); - auto ptrTensorType = RankedTensorType::get({tensorSizes}, ptrType); - - auto newAddPtrOp = rewriter.create( - loc, ptrTensorType, broadcastedPtr, broadcastedOffset); - - size_t hoistDim = -1; - for (size_t i = 0; i < offsetType.getShape().size(); ++i) { - if (offsetType.getShape()[i] == 1 && - offsetType.getShape()[i] != tensorSizes[i]) { - hoistDim = i; - break; - } - } - if (hoistDim == -1) { - LLVM_DEBUG({ - llvm::dbgs() << "No dimension to hoist found in AddPtrOp offset.\n"; - }); - } - newAddPtrOp->setAttr( - "hoist_dim", rewriter.getI64IntegerAttr(static_cast(hoistDim))); - broadcastMap[addptrOp.getResult()] = newAddPtrOp.getResult(); - return success(); -} - -LogicalResult -BroadcastHoister::parseSplat(triton::SplatOp splatOp, const Location &loc, - ConversionPatternRewriter &rewriter) { - // End of parse: splat for ptr - auto src = splatOp.getSrc(); - auto dst = splatOp.getResult(); - if (!isa(src.getType())) { - LLVM_DEBUG( - { llvm::dbgs() << "SplatOp source must be of pointer type.\n"; }); - return failure(); - } - - auto ptrType = dyn_cast(source.getType()); - auto ptrTensorType = RankedTensorType::get({tensorSizes}, ptrType); - auto newSplatOp = rewriter.create(loc, ptrTensorType, src); - broadcastMap[splatOp.getResult()] = newSplatOp.getResult(); - return success(); -} - -LogicalResult -BroadcastHoister::parseBroadcast(triton::BroadcastOp broadcastOp, - const Location &loc, - ConversionPatternRewriter &rewriter) { - // Another broadcast for ptr tensor - // To be fixed if needed in future - LLVM_DEBUG({ - llvm::dbgs() << "Now cannot handle multi broadcast for ptr tensor.\n"; - }); - return failure(); -} - -LogicalResult -BroadcastHoister::replaceBroadcastOp(triton::BroadcastOp op, - ConversionPatternRewriter &rewriter) { - auto newOp = broadcastMap[op.getSrc()]; - rewriter.replaceOp(op, newOp); - return success(); -} - -bool BroadcastHoister::canBroadcast() { - auto resultType = dyn_cast(opToHoist.getType()); - auto srcType = dyn_cast(opToHoist.getSrc().getType()); - int broadcastedDims = 0; - for (size_t i = 0; i < resultType.getShape().size(); ++i) { - if (srcType.getShape()[i] == 1 && resultType.getShape()[i] != 1) { - broadcastedDims++; - } - } - if (broadcastedDims != 1) { - LLVM_DEBUG({ - llvm::dbgs() << "Now cannot handle broadcast for ptr tensor with multi " - "broadcasted dimension.\n"; - }); - return false; - } - return source != nullptr && isa(source.getType()); -} -} // namespace HoistBroadcast diff --git a/third_party/ascend/lib/TritonToLinalg/ImplicitPermute.cpp b/third_party/ascend/lib/TritonToLinalg/ImplicitPermute.cpp deleted file mode 100644 index 354486c69c..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/ImplicitPermute.cpp +++ /dev/null @@ -1,636 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include - -#include "mlir/Dialect/Affine/IR/AffineOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Arith/Utils/Utils.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Linalg/Passes.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/BuiltinTypeInterfaces.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Location.h" -#include "mlir/IR/Matchers.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Value.h" -#include "triton/Dialect/Triton/IR/Dialect.h" -#include "triton/Dialect/Triton/IR/Types.h" - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/MathExtras.h" - -#include "llvm/Support/Debug.h" - -#include "TritonToLinalg/ImplicitPermute.h" -#include "TritonToLinalg/TritonToLinalgPass.h" -#include "TritonToStructured/CannonicalizerConverter.h" -#include "TritonToStructured/MaskAnalysis.h" -#include "TritonToStructured/PtrAnalysis.h" - -#include "Utils/InterleaveOptimization.h" -#include "Utils/Utils.h" -#include "bishengir/Dialect/Annotation/IR/Annotation.h" -#include "bishengir/Dialect/HIVM/Utils/Utils.h" - -#define DEBUG_TYPE "triton-to-linalg-implicit-permute" - -namespace ImplicitPermute { -using namespace mlir; -using namespace triton; -using namespace TritonToStructured; - -LogicalResult LoadConverter::matchAndRewrite(triton::LoadOp op, - PatternRewriter &rewriter) const { - // no need to analyze and rewrite - if (compileOn91095Flag && !existDotFlag) { - LLVM_DEBUG({ - llvm::dbgs() << "----------------------------------------------\n"; - llvm::dbgs() << "compileOn91095Flag :" << compileOn91095Flag << "\n"; - llvm::dbgs() << "!existDotFlag :" << !existDotFlag << "\n"; - llvm::dbgs() << "no need to analyze and rewrite Load" << "\n"; - llvm::dbgs() << "----------------------------------------------\n"; - }); - return success(); - } - - auto loc = op.getLoc(); - auto oldPtr = op.getPtr(); - auto oldMask = op.getMask(); - auto oldOther = op.getOther(); - - MemOpTransformer tf(MemOpTransformer::MemType::load); - - Value newPtr = nullptr; - if (oldPtr.getDefiningOp()) { - newPtr = tf.createNewTensorPtr(oldPtr, loc, rewriter); - } else if (oldPtr.getDefiningOp()) { - newPtr = tf.createNewAdvancePtr(oldPtr, loc, rewriter); - } else if (oldPtr.getDefiningOp()) { - newPtr = tf.createNewAddPtr(oldPtr, loc, rewriter); - } else { - InFlightDiagnostic diag = emitWarning(loc) - << "PtrAnalysis: only MakeTensorPtrOp, " - "AdvanceOp, and AddPtrOp are supported."; - return success(); - } - if (!tf.ptrState.isPermuted) { - // no need to rewrite - return success(); - } - - auto newMask = tf.createNewMask(oldMask, loc, rewriter); - auto newOther = tf.createNewOther(oldOther, loc, rewriter); - - if (!newPtr) { - InFlightDiagnostic diag = emitWarning(loc) - << "PtrAnalysis: failed to analyze load pointer."; - return failure(); - } - - if (oldMask && !newMask) { - InFlightDiagnostic diag = emitWarning(loc) - << "MaskAnalysis: failed to analyze load mask."; - return failure(); - } - - auto newBoundaryCheck = tf.getBoundaryCheck(op.getBoundaryCheck()); - - auto loadOp = rewriter.create( - loc, newPtr, newMask, newOther, newBoundaryCheck, op.getPadding(), - op.getCache(), op.getEvict(), op.getIsVolatile()); - loadOp->setAttr(ImplicitPermuteHandledTAG, - UnitAttr::get(rewriter.getContext())); - - auto permuteResult = - tf.materializeImplicitPermute(loadOp.getResult(), loc, rewriter); - - rewriter.replaceOp(op, permuteResult); - return success(); -} - -LogicalResult StoreConverter::matchAndRewrite(triton::StoreOp op, - PatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto oldPtr = op.getPtr(); - auto oldMask = op.getMask(); - auto oldValue = op.getValue(); - - MemOpTransformer tf(MemOpTransformer::MemType::store); - Value newPtr = nullptr; - if (oldPtr.getDefiningOp()) { - newPtr = tf.createNewTensorPtr(oldPtr, loc, rewriter); - } else if (oldPtr.getDefiningOp()) { - newPtr = tf.createNewAdvancePtr(oldPtr, loc, rewriter); - } else if (oldPtr.getDefiningOp()) { - newPtr = tf.createNewAddPtr(oldPtr, loc, rewriter); - } else { - InFlightDiagnostic diag = emitWarning(loc) - << "PtrAnalysis: only MakeTensorPtrOp, " - "AdvanceOp, and AddPtrOp are supported."; - return success(); - } - if (!tf.ptrState.isPermuted) { - // no need to rewrite - return success(); - } - auto newMask = tf.createNewMask(oldMask, loc, rewriter); - - if (!newPtr) { - InFlightDiagnostic diag = - emitWarning(loc) << "PtrAnalysis: failed to analyze store pointer."; - return failure(); - } - - if (oldMask && !newMask) { - InFlightDiagnostic diag = emitWarning(loc) - << "MaskAnalysis: failed to analyze store mask."; - return failure(); - } - - auto permuteResult = tf.materializeImplicitPermute(oldValue, loc, rewriter); - - auto newBoundaryCheck = tf.getBoundaryCheck(op.getBoundaryCheck()); - - auto storeOp = rewriter.create(loc, newPtr, permuteResult, - newMask, newBoundaryCheck, - op.getCache(), op.getEvict()); - storeOp->setAttr(ImplicitPermuteHandledTAG, - UnitAttr::get(rewriter.getContext())); - - rewriter.eraseOp(op); - return success(); -} - -LogicalResult -AtomicRMWConverter::matchAndRewrite(triton::AtomicRMWOp op, - PatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto oldPtr = op.getPtr(); - auto oldMask = op.getMask(); - auto oldVal = op.getVal(); - - MemOpTransformer tf(MemOpTransformer::MemType::store); - - Value newPtr = nullptr; - if (oldPtr.getDefiningOp()) { - newPtr = tf.createNewAddPtr(oldPtr, loc, rewriter); - } else { - emitWarning(loc) << "PtrAnalysis: AtomicRMW only support AddPtrOp."; - return success(); - } - - Value newMask = tf.createNewMask(oldMask, loc, rewriter); - - if (!tf.ptrState.isPermuted) { - return success(); - } - if (!newPtr) { - emitWarning(loc) << "PtrAnalysis: failed to analyze atomic_rmw pointer."; - return failure(); - } - if (oldMask && !newMask) { - emitWarning(loc) << "MaskAnalysis: failed to analyze atomic_rmw mask."; - return failure(); - } - - Value newVal = tf.materializeImplicitPermute(oldVal, loc, rewriter); - - Type newAtomicResTy = newVal.getType(); - - auto newAtomic = rewriter.create( - loc, newAtomicResTy, op.getAtomicRmwOp(), newPtr, newVal, newMask, - op.getSem(), op.getScope()); - newAtomic->setAttr(ImplicitPermuteHandledTAG, - UnitAttr::get(rewriter.getContext())); - - // The returned old value should be in OLD layout for users => permute back - // (load-side). - MemOpTransformer tfLoad(MemOpTransformer::MemType::load); - tfLoad.ptrState = tf.ptrState; - Value permutedRes = - tfLoad.materializeImplicitPermute(newAtomic.getResult(), loc, rewriter); - - rewriter.replaceOp(op, permutedRes); - return success(); -} - -LogicalResult -AtomicCASConverter::matchAndRewrite(triton::AtomicCASOp op, - PatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto oldPtr = op.getPtr(); - auto oldCmp = op.getCmp(); - auto oldVal = op.getVal(); - - MemOpTransformer tf(MemOpTransformer::MemType::store); - - Value newPtr = nullptr; - if (oldPtr.getDefiningOp()) { - newPtr = tf.createNewAddPtr(oldPtr, loc, rewriter); - } else { - emitWarning(loc) << "PtrAnalysis: AtomicRMW only support AddPtrOp."; - return success(); - } - - if (!tf.ptrState.isPermuted) { - return success(); - } - if (!newPtr) { - emitWarning(loc) << "PtrAnalysis: failed to analyze atomic_cas pointer."; - return failure(); - } - - Value newCmp = tf.materializeImplicitPermute(oldCmp, loc, rewriter); - Value newVal = tf.materializeImplicitPermute(oldVal, loc, rewriter); - - // CAS result (old value) must have same shape as cmp/val operands. - Type newAtomicResTy = newVal.getType(); - - auto newAtomic = rewriter.create( - loc, newAtomicResTy, newPtr, newCmp, newVal, op.getSem(), op.getScope()); - newAtomic->setAttr(ImplicitPermuteHandledTAG, - UnitAttr::get(rewriter.getContext())); - - MemOpTransformer tfLoad(MemOpTransformer::MemType::load); - tfLoad.ptrState = tf.ptrState; - Value permutedRes = - tfLoad.materializeImplicitPermute(newAtomic.getResult(), loc, rewriter); - - rewriter.replaceOp(op, permutedRes); - return success(); -} - -Value MemOpTransformer::materializeImplicitPermute(Value srcTensor, - const Location loc, - PatternRewriter &rewriter) { - auto inTy = dyn_cast(srcTensor.getType()); - if (!inTy || !ptrState.isPermuted) - return srcTensor; - - auto inShape = inTy.getShape(); - auto orderSize = ptrState.sizes.size(); - SmallVector permuteOrder(orderSize); - for (size_t i = 0; i < orderSize; ++i) { - if (currentType == MemType::load) { - if (ptrState.isBlockPtr()) { - permuteOrder[i] = orderSize - 1 - ptrState.order[i]; - } else { - permuteOrder[ptrState.permuteIds[i]] = i; - } - } else { - if (ptrState.isBlockPtr()) { - permuteOrder[i] = ptrState.order[orderSize - 1 - i]; - } else { - permuteOrder[i] = ptrState.permuteIds[i]; - } - } - } - SmallVector outShape(permuteOrder.size()); - if (inShape.size() != outShape.size()) { - InFlightDiagnostic diag = emitWarning(loc) - << "PtrAnalysis: incompatible shape for permute"; - return srcTensor; - } - - for (size_t i = 0; i < outShape.size(); ++i) { - outShape[i] = inShape[permuteOrder[i]]; - } - - auto outTy = RankedTensorType::get(outShape, inTy.getElementType()); - auto transOp = - rewriter.create(loc, outTy, srcTensor, permuteOrder); - return transOp.getResult(); -} - -Value MemOpTransformer::createNewAddPtr(Value oldPtr, const Location loc, - PatternRewriter &rewriter) { - TritonToStructured::PtrAnalysis ptrAnalysis; - - LLVM_DEBUG({ - llvm::dbgs() << "----------------------------------------------\n"; - llvm::dbgs() << "PtrAnalysis: analyzing load/store's ptr.\n"; - }); - - if (ptrAnalysis.visitOperand(oldPtr, ptrState, loc, rewriter).failed()) { - ptrState.shouldLinearize = false; - InFlightDiagnostic diag = - emitWarning(loc) << "PtranAlysis: failed to analyze load/store ptr."; - return oldPtr; - } - - // compute missing strides - // if stateinfo.shape is 1 and sizes[dimIndex] is 1, - // then the stride is the accumulated size of all dimensions on the right side - // ie. for shape [1, 128], sizes [1, 128], originally stride is [0, 1], - // after normalization, stride is [128, 1] - OpFoldResult maxStride = rewriter.getIndexAttr(1); - for (auto it = ptrState.stateInfo.rbegin(); it != ptrState.stateInfo.rend(); - ++it) { - if (TritonToStructured::isOne(it->shape) && isZero(it->stride)) { - it->stride = maxStride; - } - maxStride = maxOpFoldResult(maxStride, it->stride, loc, rewriter); - } - - ptrState.analyzePermute(); - return ptrState.createAddPtrOp(rewriter, loc); -} - -Value MemOpTransformer::createNewTensorPtr(Value oldPtr, const Location loc, - PatternRewriter &rewriter) { - TritonToStructured::PtrAnalysis ptrAnalysis; - auto makeTPtrOp = oldPtr.getDefiningOp(); - if (!makeTPtrOp) { - InFlightDiagnostic diag = emitWarning(loc) - << "PtrAnalysis: load pointer must originate " - "from 'make_tensor_ptr' operation"; - return oldPtr; - } - if (ptrAnalysis.visitOperandMakeTensorPtr(makeTPtrOp, ptrState, loc, rewriter) - .failed()) { - ptrState.isPermuted = false; - return oldPtr; - } - ptrState.analyzePermute(); - LLVM_DEBUG({ - llvm::dbgs() << "----------------------------------------------\n"; - llvm::dbgs() << "After ptrState.analyzePermute:\n"; - llvm::dbgs() << "compileOn91095Flag: " << compileOn91095Flag << "\n"; - ptrState.dump(); - llvm::dbgs() << "----------------------------------------------\n"; - }); - return ptrState.createMakeTensorPtrOp(rewriter, loc); -} - -Value MemOpTransformer::createNewAdvancePtr(Value oldPtr, const Location loc, - PatternRewriter &rewriter) { - TritonToStructured::PtrAnalysis ptrAnalysis; - auto advOp = oldPtr.getDefiningOp(); - if (!advOp) { - emitWarning(loc) - << "PtrAnalysis: pointer must originate from 'advance' operation"; - return oldPtr; - } - - Value basePtr = advOp.getPtr(); - if (!basePtr || !basePtr.getDefiningOp()) { - emitWarning(loc) << "PtrAnalysis: advance base ptr must originate from " - "'make_tensor_ptr' operation"; - return oldPtr; - } - auto newBasePtr = createNewTensorPtr(basePtr, loc, rewriter); - if (!newBasePtr) - return oldPtr; - - if (!ptrState.isPermuted) - return oldPtr; - // 2) Rewrite advance offsets according to the new make_tensor_ptr layout. - auto oldOffsets = advOp.getOffsets(); - SmallVector newOffsets; - size_t rank = ptrState.order.size(); - // iterate reversed safely: i = rank-1, ..., 0 - for (size_t i = rank; i-- > 0;) { - newOffsets.push_back(oldOffsets[ptrState.order[i]]); - } - return rewriter.create(loc, newBasePtr.getType(), - newBasePtr, newOffsets); -} - -Value MemOpTransformer::createNewMask(Value oldMask, const Location loc, - PatternRewriter &rewriter) { - if (!oldMask) - return nullptr; - - LLVM_DEBUG({ - llvm::dbgs() << "----------------------------------------------\n"; - llvm::dbgs() << "MaskAnalysis: analyzing load/store mask.\n"; - }); - - if (!oldMask || maskState.analysisMask(oldMask).failed()) { - LLVM_DEBUG({ - llvm::dbgs() << "----------------------------------------------\n"; - llvm::dbgs() << "MaskAnalysis: no mask or failed to analyze mask.\n"; - llvm::dbgs() << "oldMask:" << oldMask << "\n"; - maskState.dump(); - llvm::dbgs() << "----------------------------------------------\n"; - }); - InFlightDiagnostic diag = - emitWarning(loc) << "MaskAnalysis: failed to analyze load/store mask."; - return nullptr; - } - - SmallVector newMaskInfo; - auto itPtr = ptrState.stateInfo.begin(); - auto itMask = maskState.stateInfo.begin(); - - // match and create new mask info - while (itPtr != ptrState.stateInfo.end() && - itMask != maskState.stateInfo.end()) { - // ptr'shape must be multiple of mask'shape or vice versa - if (!isMultiple(itMask->shape, itPtr->shape)) { - InFlightDiagnostic diag = - emitWarning(loc) - << "MaskAnalysis: incompatible shapes between ptr and mask."; - LLVM_DEBUG({ - llvm::dbgs() << "----------------------------------------------\n"; - ptrState.dump(); - llvm::dbgs() << "oldMask:" << oldMask << "\n"; - maskState.dump(); - llvm::dbgs() << "----------------------------------------------\n"; - }); - return nullptr; - } - - auto newShape = minOpFoldResult(itMask->shape, itPtr->shape, loc, rewriter); - if (isLess(newShape, itMask->shape) && !itMask->hasBroadCast) { - InFlightDiagnostic diag = - emitWarning(loc) - << "MaskAnalysis: the mask shape is incompatible with ptr shape."; - return nullptr; - } - - TritonToStructured::dimInfo newInfo(itMask->offset, newShape, - itMask->dimIndex, itMask->hasBroadCast, - itMask->currentType, itMask->rhs); - - if (!isZero(itPtr->stride)) { - newMaskInfo.emplace_back(newInfo); - } - - ++itPtr; - if (isEqual(itMask->shape, newShape)) { - ++itMask; - } - } - - if (itPtr != ptrState.stateInfo.end() || - itMask != maskState.stateInfo.end()) { - LLVM_DEBUG({ - llvm::dbgs() << "----------------------------------------------\n"; - llvm::dbgs() << "MaskAnalysis: failed to apply permute on mask.\n"; - ptrState.dump(); - llvm::dbgs() << "oldMask:" << oldMask << "\n"; - maskState.dump(); - llvm::dbgs() << "----------------------------------------------\n"; - }); - InFlightDiagnostic diag = emitWarning(loc) - << "MaskAnalysis: incompatible number of " - "dimensions between ptr and mask."; - return nullptr; - } - - maskState.stateInfo = newMaskInfo; - - if (ptrState.isPermuted && !applyPermuteOnMask()) { - LLVM_DEBUG({ - llvm::dbgs() << "----------------------------------------------\n"; - llvm::dbgs() << "MaskAnalysis: failed to apply permute on mask.\n"; - ptrState.dump(); - llvm::dbgs() << "oldMask:" << oldMask << "\n"; - maskState.dump(); - llvm::dbgs() << "----------------------------------------------\n"; - }); - InFlightDiagnostic diag = - emitWarning(loc) << "MaskAnalysis: failed to apply permute on mask."; - return nullptr; - } - - LLVM_DEBUG({ - llvm::dbgs() << "After matching MaskState: \n"; - for (auto info : newMaskInfo) { - info.dump(); - } - llvm::dbgs() << "----------------------------------------------\n"; - }); - - auto newMask = maskState.createNewMask(loc, rewriter); - return newMask; -} - -Value MemOpTransformer::createNewOther(Value oldOther, const Location loc, - PatternRewriter &rewriter) { - if (!oldOther || !maskState.newMask) - return nullptr; - - auto ptrType = dyn_cast(ptrState.source.getType()); - if (!ptrType) { - InFlightDiagnostic diag = - emitWarning(loc) - << "PtrAnalysis: source of ptrState is not a pointer type."; - return nullptr; - } - Type elementType = ptrType.getPointeeType(); - - SmallVector targetShape; - for (auto info : maskState.stateInfo) { - auto staticShape = getIntAttr(info.shape); - if (!staticShape.has_value()) { - InFlightDiagnostic diag = - emitWarning(loc) - << "MaskAnalysis: dynamic shape is not supported in reshape\n"; - return oldOther; - } - targetShape.emplace_back(staticShape.value()); - } - auto targetShapeAttr = DenseIntElementsAttr::get( - RankedTensorType::get({static_cast(targetShape.size())}, - rewriter.getI64Type()), - targetShape); - auto targetShapeType = RankedTensorType::get(targetShape, elementType); - auto targetShapeValue = - rewriter.create(loc, targetShapeAttr); - - auto reshapeOp = rewriter.create( - loc, targetShapeType, oldOther, targetShapeValue); - - return reshapeOp.getResult(); -} - -// Remap boundary_check for block ptr implicit permute. -// Formula -// - newAxis = rank - 1 - position(oldAxis in ptrState.order) -// Rules implemented -// - ptrState.order records original axes in memory-priority order. -// - createMakeTensorPtrOp rebuilds the new block ptr from -// reverse(ptrState.order), -// so the rebuilt ptr is canonicalized to descending order [rank-1, ..., 0]. -// - So each old boundary_check axis must be translated into the axis index -// of the rewritten block ptr before creating the new load/store. -// Examples -// - order=[1,0], boundary_check=[0] => [0] -// - order=[0,1], boundary_check=[0,1] => [1,0] -// - order=[2,0,1], boundary_check=[0,2] => [1,2] -SmallVector -MemOpTransformer::getBoundaryCheck(ArrayRef oldBoundaryCheck) const { - SmallVector newBoundaryCheck(oldBoundaryCheck.begin(), - oldBoundaryCheck.end()); - if (!ptrState.isPermuted || !ptrState.isBlockPtr() || - newBoundaryCheck.empty()) { - return newBoundaryCheck; - } - - int32_t rank = static_cast(ptrState.order.size()); - for (auto &boundaryAxis : newBoundaryCheck) { - auto pos = llvm::find(ptrState.order, static_cast(boundaryAxis)); - if (pos == ptrState.order.end()) { - continue; - } - boundaryAxis = - rank - 1 - - static_cast(std::distance(ptrState.order.begin(), pos)); - } - return newBoundaryCheck; -} - -bool MemOpTransformer::applyPermuteOnMask() { - if (!ptrState.isPermuted || maskState.isEmpty()) { - return true; - } - if (ptrState.permuteIds.size() != maskState.stateInfo.size()) { - return false; - } - SmallVector newMaskInfo; - for (auto id : ptrState.permuteIds) { - newMaskInfo.push_back(maskState.stateInfo[id]); - } - maskState.stateInfo = newMaskInfo; - return true; -} - -} // namespace ImplicitPermute diff --git a/third_party/ascend/lib/TritonToLinalg/LoadStoreConverter.cpp b/third_party/ascend/lib/TritonToLinalg/LoadStoreConverter.cpp deleted file mode 100644 index 8367307039..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/LoadStoreConverter.cpp +++ /dev/null @@ -1,1650 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/LoadStoreConverter.h" -#include "ascend/include/TritonToLinalg/BlockPtrAnalysis.h" -#include "ascend/include/TritonToLinalg/MaskAnalysis.h" -#include "ascend/include/TritonToLinalg/TritonToLinalgPass.h" -#include "ascend/include/Utils/DebugUtils.h" -#include "ascend/include/Utils/InterleaveOptimization.h" -#include "ascend/include/Utils/Utils.h" - -#include "bishengir/Dialect/Annotation/IR/Annotation.h" -#include "bishengir/Dialect/HFusion/IR/HFusion.h" -#include "bishengir/Dialect/HIVM/IR/HIVM.h" - -#include "Utils/Utils.h" -#include "mlir/Dialect/Affine/IR/AffineOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Arith/Utils/Utils.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Linalg/Passes.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/BuiltinTypeInterfaces.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Location.h" -#include "mlir/IR/Matchers.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Value.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/MathExtras.h" - -#include "llvm/Support/Debug.h" - -#include -#include -#include - -#define DEBUG_TYPE "triton-load-store-converter" - -namespace LoadStoreConverter { -using namespace mlir; -using namespace triton; - -const std::string MayImplicitTransposeWithLastAxisTAG = - "MayImplicitTransposeWithLastAxis"; -static constexpr llvm::StringLiteral kAlreadySyncAttr = "already_sync"; - -// During dialect conversion the adaptor may expose the converted memref -// directly, while a failed rewrite later leaves a one-input, one-output UCC in -// the diagnostic IR. Accept both representations, but only for the canonical -// i1-to-i8 scalar pointer normalization. -static Value -unwrapCanonicalI8ScalarPointerMaterialization(Value originalPointer, - Value convertedPointer) { - auto bitcastOp = originalPointer.getDefiningOp(); - if (!bitcastOp) - return nullptr; - - auto sourceType = dyn_cast(bitcastOp.getSrc().getType()); - auto resultType = dyn_cast(originalPointer.getType()); - if (!sourceType || !resultType || !sourceType.getPointeeType().isInteger(1) || - !resultType.getPointeeType().isInteger(8)) - return nullptr; - - Value memref = convertedPointer; - if (auto castOp = memref.getDefiningOp()) { - if (castOp.getInputs().size() != 1 || castOp.getOutputs().size() != 1 || - castOp.getOutputs().front() != memref) - return nullptr; - memref = castOp.getInputs().front(); - } - - auto memrefType = dyn_cast(memref.getType()); - if (!memrefType || memrefType.getRank() != 1 || - !memrefType.getElementType().isInteger(8) || - !memrefType.getLayout().isIdentity() || memrefType.getMemorySpace() || - memrefType.getDimSize(0) != ShapedType::kDynamic) - return nullptr; - - return memref; -} - -static bool shouldMaterializeCustomPointer(Value ptr) { - auto result = dyn_cast(ptr); - if (!result) - return false; - - Operation *defOp = result.getOwner(); - if (!defOp || !isDistributedTypeCustomOp(defOp)) - return false; - - auto srcIndices = defOp->getAttrOfType( - ConverterUtils::customSrcPtrIndexAttrName); - if (!srcIndices) - return false; - - auto values = srcIndices.asArrayRef(); - unsigned resultIdx = result.getResultNumber(); - return resultIdx < values.size() && values[resultIdx] >= 0; -} - -static FailureOr -resolveMemoryPointer(Value originalPtr, Value convertedPtr, - ConversionPatternRewriter &rewriter) { - llvm::SmallDenseMap known; - - if (shouldMaterializeCustomPointer(originalPtr)) - return BlockDataParser::materializePointer(originalPtr, rewriter, known); - - if (isa(convertedPtr.getType())) - return convertedPtr; - - return BlockDataParser::materializePointer(originalPtr, rewriter, known); -} - -LogicalResult -AddPtrConverter::matchAndRewrite(triton::AddPtrOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Location loc = op.getLoc(); - insertDebugNop(loc, rewriter); - llvm::SmallDenseMap known; - return BlockDataParser::rewriteAddPtr(op, adaptor, rewriter, known); -} - -LogicalResult LoadConverter::toTensorAndReplace( - triton::LoadOp &op, RankedTensorType &tensorType, Value localMem, - bool mayImplicitTransposeWithLastAxis, const Location &loc, - ConversionPatternRewriter &rewriter) const { - Value loadedTensor = rewriter.create( - loc, tensorType, localMem, true, true); - propagateWasBoolToInt8Attr(op.getOperation(), loadedTensor.getDefiningOp(), - rewriter); - - if (mayImplicitTransposeWithLastAxis) { - auto markOp = rewriter.create(loc, loadedTensor); - markOp->setAttr(MayImplicitTransposeWithLastAxisTAG, - UnitAttr::get(rewriter.getContext())); - } - rewriter.replaceOp(op, loadedTensor); - return success(); -} - -/// @brief Check whether the triton::LoadOp has been modified to the specified -/// state by the AddPtrConverter. -/// @param op The triton::LoadOp operation to be checked. -/// @return Return success if the operation conforms to the specified state; -/// otherwise, return failure. -LogicalResult -LoadConverter::checkModifiedByAddPtrConverter(triton::LoadOp &op) const { - if (!isa(op->getParentOp())) { - return failure(); - } - if (!op->hasAttr("IndirectLoad")) { - return failure(); - } - auto ptrOp = op.getPtr().getDefiningOp(); - auto ptrBlock = ptrOp->getBlock(); - auto opBlock = op->getBlock(); - if (ptrBlock == opBlock) { - return failure(); - } - - return success(); -} - -void LoadConverter::propagateWasBoolToInt8Attr( - Operation *srcLoadOp, Operation *dstOp, PatternRewriter &rewriter) const { - const std::string WasBoolToInt8TAG = "was_bool_to_int8"; - if (!srcLoadOp || !dstOp) - return; - if (srcLoadOp->hasAttr(WasBoolToInt8TAG)) { - dstOp->setAttr(WasBoolToInt8TAG, rewriter.getBoolAttr(true)); - } -} - -/// @brief Continue to modify the triton::LoadOp from the state modified by the -/// AddPtrConverter. -/// @param op The triton::LoadOp operation to be processed. -/// @param adaptor The adaptor for the operation, used to obtain operands. -/// @param rewriter The pattern rewriter used to rewrite the operation. -/// @return Return success if the operation is successful; otherwise, return -/// failure. -LogicalResult LoadConverter::continueModifyFromAddPtrConverter( - triton::LoadOp &op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto forOp = op->getParentOfType(); - Operation *firstOp = &forOp.getBody()->front(); - auto extractOp = cast(firstOp); - auto ivs = extractOp.getIndices(); - // Single iterArg which is inserted by AddPtrConverter. - auto iterArg = forOp.getRegionIterArg(0); - auto ptr = adaptor.getPtr(); - - rewriter.setInsertionPointAfter(op); - Value castVal = ptr.getDefiningOp(); - Value idxZero = - rewriter.create(loc, rewriter.getIndexAttr(0)); - Value loadVal = - rewriter.create(loc, castVal, ValueRange{idxZero}); - propagateWasBoolToInt8Attr(op.getOperation(), loadVal.getDefiningOp(), - rewriter); - Value insertedVal = - rewriter.create(loc, loadVal, iterArg, ValueRange{ivs}); - // a yield op is already created by AddPtrConverter. - // so we need to replace it with a new yield op. - Operation *terminator = forOp.getBody()->getTerminator(); - scf::YieldOp oldYieldOp = cast(terminator); - auto yieldOp = rewriter.create(loc, ValueRange{insertedVal}); - rewriter.replaceOp(oldYieldOp, yieldOp); - // Now the scf.for is complete, we can replace tt.load with it. - auto rank = cast(op.getResult().getType()).getShape().size(); - Operation *rootForOp = op; - while (rank != 0) { - rank--; - rootForOp = rootForOp->getParentOfType(); - } - rewriter.replaceOp(op, rootForOp); - LLVM_DEBUG({ llvm::dbgs() << *getModuleOpFromOperation(rootForOp) << "\n"; }); - return success(); -} - -void LoadConverter::fillTensorWithOtherForMaskScenario( - Value other, Value localMem, ArrayRef maskDim, - ConversionPatternRewriter &rewriter) const { - auto loc = localMem.getLoc(); - MemRefType originalType = cast(localMem.getType()); - assert(originalType.hasStaticShape() && "only support static shape"); - assert(originalType.getRank() == maskDim.size() && - "shape and mask must have same rank"); - - auto fillFlag = - rewriter.create(loc, rewriter.getBoolAttr(false)) - .getResult(); - - for (size_t i = 0; i < originalType.getShape().size(); ++i) { - // Use dynamic value to judge whether overstep boundary - auto shapeVal = rewriter.create( - loc, rewriter.getIndexAttr(originalType.getDimSize(i))); - - Value maskDimVal; - if (isa(maskDim[i])) - maskDimVal = rewriter.create( - loc, cast(cast(maskDim[i]))); - else - maskDimVal = cast(maskDim[i]); - - auto curCmp = rewriter.create(loc, arith::CmpIPredicate::slt, - maskDimVal, shapeVal); - - fillFlag = rewriter.create(loc, fillFlag, curCmp.getResult()) - .getResult(); - } - auto ifOp = rewriter.create(loc, fillFlag); - { - OpBuilder::InsertionGuard guard(rewriter); - rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front()); - rewriter.create(loc, ValueRange{other}, - ValueRange{localMem}); - } - ifOp->setAttr(rewriter.getStringAttr("hivm.unlikely_condition"), - UnitAttr::get(rewriter.getContext())); -} - -LoadConverter::LoadConverter(MLIRContext *context) - : OpConversionPattern(context) {} - -// Masked load whose `other` is a non-scalar tensor. -// --------------------------------------------------------------------------- -// Case 1: other = mask (Python: tl.load(ptr, mask=mask, other=mask)) -// -// DSL: -// mask = offs < N -// x = tl.load(in_ptr + offs, mask=mask, other=mask) -// tl.store(out_ptr + offs, x, mask=mask) -// -// TTIR (fp — uitofp; int — extui/extsi): -// %mask = arith.cmpi slt, %offs, %N : tensor -// %other = arith.uitofp %mask … // or arith.extui for i8 -// %x = tt.load %ptr, %mask, %other -// -// Lowering (linalg): -// %alloc = memref.alloc() -// memref.copy %src_view, %dst_view // active SRC only -// %loaded = bufferization.to_tensor %alloc -// %zeros = linalg.fill 0 // [OTHER] cast(mask)→0 on pad -// %x = arith.select %mask, %loaded, %zeros -// --------------------------------------------------------------------------- -// Case 2: other = prior tensor -// (Python: other = tl.load(fill_ptr+offs); x = tl.load(ptr, mask=mask, -// other=other); tl.store(out, x) # full store — pad must stay OTHER) -// -// TTIR: -// %other = tt.load %fill_ptr -// %x = tt.load %ptr, %mask, %other -// -// Lowering: -// %other_t = bufferization.to_tensor … // prior load -// %alloc = memref.alloc() -// memref.copy … // active SRC -// %loaded = bufferization.to_tensor %alloc -// %x = arith.select %mask, %loaded, %other_t -// --------------------------------------------------------------------------- -LogicalResult LoadConverter::replaceMaskedLoadWithTensorOther( - triton::LoadOp op, Value alloc, bool mayImplicitTransposeWithLastAxis, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto tensorType = cast(op.getResult().getType()); - Value mask = op.getMask(); - Value tensorOther = op.getOther(); - if (!mask || !tensorOther) { - return rewriter.notifyMatchFailure( - op, - "replaceMaskedLoadWithTensorOther requires non-null mask and other"); - } - - Value loadedTensor = rewriter.create( - loc, tensorType, alloc, true, true); - propagateWasBoolToInt8Attr(op.getOperation(), loadedTensor.getDefiningOp(), - rewriter); - if (mayImplicitTransposeWithLastAxis) { - auto markOp = rewriter.create(loc, loadedTensor); - markOp->setAttr(MayImplicitTransposeWithLastAxisTAG, - UnitAttr::get(rewriter.getContext())); - } - - // Case 1: cast(mask) → inactive is 0. Cover fp (uitofp/sitofp) and int - // (extui/extsi). Case 2: keep prior tensor as-is. - Value otherVal = tensorOther; - if (Value remappedOther = rewriter.getRemappedValue(tensorOther)) - otherVal = remappedOther; - - bool otherIsZeroOnInactive = false; - if (auto uitofp = tensorOther.getDefiningOp()) - otherIsZeroOnInactive = (uitofp.getIn() == mask); - else if (auto sitofp = tensorOther.getDefiningOp()) - otherIsZeroOnInactive = (sitofp.getIn() == mask); - else if (auto extui = tensorOther.getDefiningOp()) - otherIsZeroOnInactive = (extui.getIn() == mask); - else if (auto extsi = tensorOther.getDefiningOp()) - otherIsZeroOnInactive = (extsi.getIn() == mask); - - if (otherIsZeroOnInactive) { - auto empty = rewriter.create(loc, tensorType.getShape(), - tensorType.getElementType()); - Value zero = rewriter.create( - loc, rewriter.getZeroAttr(tensorType.getElementType())); - otherVal = - rewriter - .create(loc, ValueRange{zero}, ValueRange{empty}) - .getResult(0); - } - - if (Value remappedMask = rewriter.getRemappedValue(mask)) - mask = remappedMask; - - Value result = - rewriter.create(loc, mask, loadedTensor, otherVal); - rewriter.replaceOp(op, result); - return success(); -} - -static bool skipNonComplementaryStaticDeinterleave(triton::LoadOp lhs) { - if (lhs->hasAttr("skip_deinterleave")) - return true; - auto lhsAddPtr = lhs.getPtr().getDefiningOp(); - if (!lhsAddPtr) - return false; - auto lhsOffset = getConstantIntValue(lhsAddPtr.getOffset()); - if (!lhsOffset) - return false; - auto pairBase = [](int64_t offset) { - int64_t lane = offset % 2; - return offset - (lane < 0 ? lane + 2 : lane); - }; - - for (triton::LoadOp rhs : lhs->getBlock()->getOps()) { - if (lhs == rhs || rhs.getMask() || rhs.getOther() || - lhs.getResult().getType() != rhs.getResult().getType()) - continue; - auto rhsAddPtr = rhs.getPtr().getDefiningOp(); - if (!rhsAddPtr || lhsAddPtr.getPtr() != rhsAddPtr.getPtr()) - continue; - auto rhsOffset = getConstantIntValue(rhsAddPtr.getOffset()); - if (!rhsOffset) - continue; - if (lhsOffset.value() != rhsOffset.value() && - pairBase(lhsOffset.value()) == pairBase(rhsOffset.value())) - continue; - // The first load can be rewritten before its sibling; tag both now. - auto skipAttr = UnitAttr::get(lhs->getContext()); - lhs->setAttr("skip_deinterleave", skipAttr); - rhs->setAttr("skip_deinterleave", skipAttr); - return true; - } - return false; -} - -LogicalResult -LoadConverter::matchAndRewrite(triton::LoadOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - - // Check if tt.load is modified by AddPtrConverter to a specified state. - if (checkModifiedByAddPtrConverter(op).succeeded()) { - return continueModifyFromAddPtrConverter(op, adaptor, rewriter); - } - - auto ptr = adaptor.getPtr(); - auto mask = op.getMask(); - auto other = op.getOther(); - auto loc = op.getLoc(); - insertDebugNopForMask(mask, rewriter); - - bool isScalarLoad = !isa(op.getResult().getType()); - Value normalizedScalarMemref; - if (isScalarLoad && op.getResult().getType().isInteger(8)) - normalizedScalarMemref = - unwrapCanonicalI8ScalarPointerMaterialization(op.getPtr(), ptr); - - if (normalizedScalarMemref) { - ptr = normalizedScalarMemref; - } else { - FailureOr resolvedPtr = - resolveMemoryPointer(op.getPtr(), ptr, rewriter); - if (failed(resolvedPtr)) - return rewriter.notifyMatchFailure( - op, "unable to materialize the load pointer as a memref"); - ptr = *resolvedPtr; - } - // handling scalar - if (isScalarLoad) { - Value scalarMemref = normalizedScalarMemref; - if (!scalarMemref) { - FailureOr resolvedScalarMemref = - BlockDataParser::getScalarMemRef(op.getPtr(), ptr, loc, rewriter); - if (failed(resolvedScalarMemref)) - return rewriter.notifyMatchFailure( - op, "scalar pointer operand has not been converted to a memref"); - scalarMemref = *resolvedScalarMemref; - } - - auto resTy = op.getResult().getType(); - auto idxZero = - rewriter.create(loc, rewriter.getIndexAttr(0)); - auto createScalarLoad = [&]() -> Value { - auto load = rewriter.create(loc, resTy, scalarMemref, - idxZero.getResult()); - propagateWasBoolToInt8Attr(op.getOperation(), load.getOperation(), - rewriter); - return load.getResult(); - }; - if (!mask) { - rewriter.replaceOp(op, createScalarLoad()); - return success(); - } - - // Keep scalar results scalar, and guard the access itself: a select after - // an unconditional load would still read a masked-off, possibly invalid - // address. - auto ifOp = rewriter.create(loc, TypeRange{resTy}, - adaptor.getMask(), true); - { - OpBuilder::InsertionGuard guard(rewriter); - rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front()); - rewriter.create(loc, createScalarLoad()); - - rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front()); - Value inactiveValue = adaptor.getOther(); - if (!inactiveValue) { - // Without `other`, the inactive value is undefined. Choose zero - // without accessing memory or introducing a tensor temporary. - inactiveValue = rewriter.create( - loc, resTy, rewriter.getZeroAttr(resTy)); - } - rewriter.create(loc, inactiveValue); - } - rewriter.replaceOp(op, ifOp.getResults()); - return success(); - } - - int64_t lastStride = -1; - if (isa(ptr)) { - auto u = ptr; - while (auto blkArg = dyn_cast(u)) { - if (auto forOp = dyn_cast(blkArg.getOwner()->getParentOp())) { - auto prt = forOp->getOperand(3 + blkArg.getArgNumber() - 1); - u = prt; - } else { - u = nullptr; - break; - } - } - if (u && isa(u.getDefiningOp())) { - auto ret = mlir::ConverterUtils::getLastStrideOfReinterpretCastOp( - dyn_cast(u.getDefiningOp())); - if (ret.has_value()) - lastStride = *ret; - } - } - - // handling no mask - auto memRefType = dyn_cast(ptr.getType()); - if (!memRefType) { - return rewriter.notifyMatchFailure( - op, "LoadOp expects a memref, not a memref of pointers"); - } - if (!op->hasAttr(ConverterUtils::GeneratedByMakeTensorPtrTAG)) { - auto memrefOp = dyn_cast(ptr.getDefiningOp()); - auto ret = mlir::ConverterUtils::getLastStrideOfReinterpretCastOp(memrefOp); - if (ret.has_value()) - lastStride = *ret; - } - bool mayImplicitTransposeWithLastAxis = - (existDotFlag) && - (!op->hasAttr(ConverterUtils::GeneratedByMakeTensorPtrTAG)) && - (lastStride != 1 && - mlir::ConverterUtils::isaPermutedMemRefType(memRefType)); - auto memRefShape = memRefType.getShape(); - auto memRefElementType = memRefType.getElementType(); - - Value allocOp; - Value allocOpTmp; - auto parentLoop = dyn_cast(op->getParentOp()); - bool hasValidDiscreteLoop = op->hasAttr(ConverterUtils::discreteAttrName) && - parentLoop && - parentLoop->hasAttr("ExtractedLoadOrStore"); - if (hasValidDiscreteLoop) { - Operation *loop = op->getParentOp(); - int extractedLoopCount = 1; - for (auto parentOp = loop->getParentOp(); - parentOp->hasAttr("ExtractedLoadOrStore"); - parentOp = parentOp->getParentOp()) { - loop = parentOp; - extractedLoopCount++; - } - rewriter.setInsertionPoint(loop); - auto loopOp = cast(loop); - auto fullMemRefShape = - cast(loopOp.getInitArgs()[0].getType()).getShape(); - auto fullMemRefType = MemRefType::get(fullMemRefShape, memRefElementType); - bool isIndexSelectScenario = - (extractedLoopCount == 1) && (fullMemRefShape.size() > 1u); - if (isIndexSelectScenario) - loopOp->setAttr("hivm.parallel_loop", rewriter.getUnitAttr()); - allocOp = rewriter.create(loc, fullMemRefType); - allocOpTmp = allocOp; - rewriter.setInsertionPointAfter(loop); - auto toTensorOp = rewriter.create( - loc, RankedTensorType::get(fullMemRefShape, memRefElementType), allocOp, - true, true); - rewriter.replaceAllUsesWith(loopOp->getResult(0), toTensorOp->getResult(0)); - tensor::InsertSliceOp insertSliceOp = nullptr; - for (auto *user : op->getUsers()) { - if (auto targetOp = dyn_cast(user)) { - insertSliceOp = targetOp; - break; - } - } - auto offsets = insertSliceOp.getMixedOffsets(); - auto sizes = insertSliceOp.getMixedSizes(); - auto strides = insertSliceOp.getMixedStrides(); - auto allocType = memref::SubViewOp::inferResultType(fullMemRefType, offsets, - sizes, strides); - rewriter.setInsertionPoint(op); - allocOp = rewriter.create( - loc, cast(allocType), allocOp, offsets, sizes, strides); - rewriter.replaceAllUsesExcept(insertSliceOp.getResult(), - insertSliceOp.getDest(), insertSliceOp); - rewriter.eraseOp(insertSliceOp); - } else { - allocOp = rewriter.create( - loc, MemRefType::get(memRefShape, memRefElementType)); - } - - auto tensorType = RankedTensorType::get(memRefShape, memRefElementType); - // boundary check - auto boundaryCheck = op.getBoundaryCheck(); - if (!boundaryCheck.empty()) { - auto makeTensorPtrOp = op.getPtr().getDefiningOp(); - auto boundarySizes = mlir::ConverterUtils::getBoundarySizes( - boundaryCheck, /*remapped*/ ptr, loc, rewriter); - // handle the padding - auto padding = op.getPadding(); - SmallVector srcOffsets(boundarySizes.size(), - rewriter.getIndexAttr(0)); - SmallVector dstOffsets; - if (makeTensorPtrOp) { - auto zeroVal = rewriter.createOrFold( - loc, rewriter.getI32IntegerAttr(0)); - for (auto [idx, offVal] : llvm::enumerate(makeTensorPtrOp.getOffsets())) { - if (llvm::find(boundaryCheck, idx) == boundaryCheck.end()) { - dstOffsets.push_back(srcOffsets[idx]); - continue; - } - Value offset = - rewriter.createOrFold(loc, zeroVal, offVal); - Value size = - getValueOrCreateConstantIndexOp(rewriter, loc, boundarySizes[idx]); - offset = rewriter.createOrFold(loc, offset, zeroVal); - offset = rewriter.createOrFold( - loc, rewriter.getIndexType(), offset); - OpFoldResult ofr; - if (auto constOp = offset.getDefiningOp()) { - ofr = constOp.getValue(); - } else { - ofr = offset; - } - ofr = minOpFoldResult(ofr, size, loc, rewriter); - boundarySizes[idx] = subOpFoldResult(size, ofr, loc, rewriter); - dstOffsets.push_back(ofr); - } - } else { - dstOffsets = srcOffsets; - } - if (padding.has_value()) { - TypedAttr padAttr = rewriter.getZeroAttr(memRefElementType); - // triton already ensure only NAN and ZERO are passed in - if (padding.value() == triton::PaddingOption::PAD_NAN) { - // FIXME: Why NaN requires elemTy to be non-int or non-index? - assert(!memRefElementType.isIntOrIndex()); - auto apNaN = llvm::APFloat::getNaN( - cast(padAttr).getValue().getSemantics()); - padAttr = rewriter.getFloatAttr(memRefElementType, apNaN); - } - auto padVal = rewriter.create(loc, padAttr); - - fillTensorWithOtherForMaskScenario(padVal, allocOp, boundarySizes, - rewriter); - } - auto srcSubView = mlir::ConverterUtils::makeSubViewOp( - ptr, srcOffsets, boundarySizes, loc, rewriter); - auto dstSubview = mlir::ConverterUtils::makeSubViewOp( - allocOp, dstOffsets, boundarySizes, loc, rewriter); - auto copyOp = rewriter.create(loc, srcSubView, dstSubview); - propagateWasBoolToInt8Attr(op.getOperation(), copyOp.getOperation(), - rewriter); - if (mayImplicitTransposeWithLastAxis) { - auto markOp = rewriter.create(loc, dstSubview); - markOp->setAttr(MayImplicitTransposeWithLastAxisTAG, - UnitAttr::get(rewriter.getContext())); - } - return this->toTensorAndReplace(op, tensorType, allocOp, - mayImplicitTransposeWithLastAxis, loc, - rewriter); - } - - if (!mask) { - assert(!other && "can not input 'other' when 'mask' is not set"); - if (auto unrealizedCastOp = - ptr.getDefiningOp()) { - // TODO : not support handle associate with "module" - // hint : can be handled in Linearize - op->emitError("meeting unexpected UCC in LoadConverter!"); - return failure(); - } else { - // If last dimension stride equals 2, try deinterleave optimization. - auto [ptrStrides, ptrOffsets] = memRefType.getStridesAndOffset(); - if (ptrStrides.back() == 2 && (memRefShape.back() % 2 == 0) && - !skipNonComplementaryStaticDeinterleave(op) && - mlir::triton::DeinterleaveStatusOptimization(op, adaptor, rewriter) - .succeeded()) { - return success(); - } - auto copyOp = rewriter.create(loc, ptr, allocOp); - propagateWasBoolToInt8Attr(op.getOperation(), copyOp.getOperation(), - rewriter); - if (mayImplicitTransposeWithLastAxis && - allocOp.getDefiningOp()) { - auto markOp = rewriter.create(loc, allocOp); - markOp->setAttr(MayImplicitTransposeWithLastAxisTAG, - UnitAttr::get(rewriter.getContext())); - } else if (mayImplicitTransposeWithLastAxis && - allocOp.getDefiningOp()) { - auto markOp = rewriter.create(loc, allocOpTmp); - markOp->setAttr(MayImplicitTransposeWithLastAxisTAG, - UnitAttr::get(rewriter.getContext())); - } - } - - return this->toTensorAndReplace(op, tensorType, allocOp, - mayImplicitTransposeWithLastAxis, loc, - rewriter); - } - - MaskState mstate; - auto isContMask = mstate.parse(mask, loc, rewriter); - if (isContMask.failed()) { - return rewriter.notifyMatchFailure( - op, "can not lower uncontinuout masked loads"); - } - - Value tensorOtherBase; - if (other) { - auto scalarOther = - mlir::ConverterUtils::getScalarValue(other, loc, rewriter); - if (scalarOther) { - fillTensorWithOtherForMaskScenario(scalarOther, allocOp, mstate.dims, - rewriter); - } else if (isa(other.getType())) { - tensorOtherBase = other; - } else { - return rewriter.notifyMatchFailure( - op, "other value used in masked load produced by unsupported " - "instruction"); - } - } - - // To enable deinterleave optimization with mask load, mask state along last - // dimension couldn't be split, which means `dims.back()` must be equal to - // origin type last dimension constant size and `offsets.back()` must be 0. - // - // The basis is that last dimension range comparison would generate - // unaccepted discontinuous mask. - if (!tensorOtherBase && mstate.getRank() == memRefType.getRank() && - isConstantIntValue(mstate.offsets.back(), 0) && - isConstantIntValue(mstate.dims.back(), memRefType.getShape().back())) { - auto [ptrStrides, ptrOffsets] = memRefType.getStridesAndOffset(); - if (ptrStrides.back() == 2 && (memRefType.getShape().back() % 2 == 0) && - DeinterleaveStatusWithMaskOptimization(op, adaptor, rewriter, mstate, - allocOp) - .succeeded()) { - return success(); - } - } - - if (auto unrealizedCastOp = ptr.getDefiningOp()) { - // TODO : not support handle associate with "module" - // hint : can be handled in Linearize - op->emitError("meeting unexpected UCC in LoadConverter!"); - return failure(); - } else { - if (mstate.isMemrefSubviewValid(ptr, rewriter)) { - memref::SubViewOp srcSubView = mstate.getSubview(ptr, loc, rewriter); - memref::SubViewOp dstSubView = mstate.getSubview(allocOp, loc, rewriter); - MemRefType dstSubViewType = mlir::cast(dstSubView.getType()); - - auto [srcStrides, srcOffset] = dstSubViewType.getStridesAndOffset(); - MemRefType castType = MemRefType::get( - dstSubViewType.getShape(), dstSubViewType.getElementType(), - makeStridedLinearLayoutMap(srcStrides, srcOffset, - rewriter.getContext())); - auto castOp = rewriter.create(loc, castType, dstSubView); - auto copyOp = rewriter.create(loc, srcSubView, castOp); - propagateWasBoolToInt8Attr(op.getOperation(), copyOp.getOperation(), - rewriter); - } - - if (mayImplicitTransposeWithLastAxis && - allocOp.getDefiningOp()) { - auto markOp = rewriter.create(loc, allocOp); - markOp->setAttr(MayImplicitTransposeWithLastAxisTAG, - UnitAttr::get(rewriter.getContext())); - } else if (mayImplicitTransposeWithLastAxis && - allocOp.getDefiningOp()) { - auto markOp = rewriter.create(loc, allocOpTmp); - markOp->setAttr(MayImplicitTransposeWithLastAxisTAG, - UnitAttr::get(rewriter.getContext())); - } - } - - if (!tensorOtherBase) { - return this->toTensorAndReplace(op, tensorType, allocOp, - mayImplicitTransposeWithLastAxis, loc, - rewriter); - } - - return replaceMaskedLoadWithTensorOther( - op, allocOp, mayImplicitTransposeWithLastAxis, rewriter); -} - -AtomicRMWConverter::AtomicRMWConverter(MLIRContext *context) - : OpConversionPattern(context) {} - -// lowering tt.atomicRMW to linalg.generic -// If atomic op's return value is used by other op as it's the old value stored -// at the ptrwe will use tt.load to get it -// -// example: -// input: -// %return_value = tt.atomic_rmw fadd, acq_rel, gpu, -// %output_memref, %input_tensor, %mask : -// (tensor<256x!tt.ptr>, tensor<256xf32>, tensor<256xi1>) -// -> tensor<256xf32> -// -// output: -// memref.copy %output_memref, %ub_buf : memref to memref -// %17 = bufferization.to_tensor %alloc_3 restrict writable : memref<256xf32> -// linalg.generic -// {indexing_maps = [#map, #map, #map], iterator_types = ["parallel"]} -// ins(%output_memref, %masked_input_memref : memref, memref) -// outs(%subview_2 : memref) -// attrs = {GenericAtomicRMW = "fadd", MemSemantic = "acq_rel", -// MemSyncScope = "gpu"} { -// ^bb0(%in: f32, %in_9: f32, %out: f32): -// %25 = arith.addf %in, %in_9 : f32 -// linalg.yield %25 : f32 -// } -LogicalResult -AtomicRMWConverter::matchAndRewrite(triton::AtomicRMWOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto ptr = adaptor.getPtr(); - auto val = op.getVal(); - auto loc = op.getLoc(); - auto mask = op.getMask(); - auto rmwOp = op.getAtomicRmwOp(); - auto resType = dyn_cast(op.getResult().getType()); - insertDebugNop(loc, rewriter); - insertDebugNopForMask(mask, rewriter); - - FailureOr resolvedPtr = - resolveMemoryPointer(op.getPtr(), ptr, rewriter); - if (failed(resolvedPtr)) - return rewriter.notifyMatchFailure( - op, "unable to materialize the atomic RMW pointer as a memref"); - ptr = *resolvedPtr; - auto ptrType = dyn_cast(ptr.getType()); - if (!resType) - return rewriter.notifyMatchFailure( - op, "atomicRMWConverter: scalar will be handled by " - "ScalarAtomicRMWCanonicalizer"); - if (!ptrType) - return rewriter.notifyMatchFailure( - op, "AtomicRMWOp expects a memref, not a memref of pointers"); - - const std::map atomicKindMap = { - {RMWOp::ADD, hivm::AtomicKind::ADD}, - {RMWOp::FADD, hivm::AtomicKind::ADD}, - {RMWOp::OR, hivm::AtomicKind::OR}, - {RMWOp::XOR, hivm::AtomicKind::XOR}, - {RMWOp::AND, hivm::AtomicKind::AND}, - {RMWOp::MIN, hivm::AtomicKind::MIN}, - {RMWOp::UMIN, hivm::AtomicKind::UMIN}, - {RMWOp::MAX, hivm::AtomicKind::MAX}, - {RMWOp::UMAX, hivm::AtomicKind::UMAX}, - {RMWOp::XCHG, hivm::AtomicKind::XCHG}, - }; - const std::map hfusionAtomicKindMap = { - {RMWOp::ADD, hfusion::AtomicKind::ADD}, - {RMWOp::FADD, hfusion::AtomicKind::ADD}, - {RMWOp::OR, hfusion::AtomicKind::OR}, - {RMWOp::XOR, hfusion::AtomicKind::XOR}, - {RMWOp::AND, hfusion::AtomicKind::AND}, - {RMWOp::MIN, hfusion::AtomicKind::MIN}, - {RMWOp::UMIN, hfusion::AtomicKind::UMIN}, - {RMWOp::MAX, hfusion::AtomicKind::MAX}, - {RMWOp::UMAX, hfusion::AtomicKind::UMAX}, - {RMWOp::XCHG, hfusion::AtomicKind::XCHG}, - }; - assert(atomicKindMap.find(rmwOp) != atomicKindMap.end()); - auto atomicKind = - hivm::AtomicKindAttr::get(rewriter.getContext(), atomicKindMap.at(rmwOp)); - assert(hfusionAtomicKindMap.find(rmwOp) != hfusionAtomicKindMap.end()); - auto hfusionAtomicKind = hfusion::AtomicKindAttr::get( - rewriter.getContext(), hfusionAtomicKindMap.at(rmwOp)); - - auto dstMemref = ptr; - Value inputVal = val; - - // Lazily materialize a memref view only when we truly need buffer - // semantics (e.g., mask subview or XCHG lowering). Otherwise keep tensor - // inputs to avoid redundant to_memref conversions before hivm.store. - auto getInputMemref = [&]() -> Value { - if (isa(inputVal.getType())) - return inputVal; - return rewriter.create(loc, ptrType, inputVal); - }; - auto inputMemref = getInputMemref(); - auto inputMemrefType = cast(inputMemref.getType()); - auto elementType = inputMemrefType.getElementType(); - auto isHardwareSupported = - (rmwOp == RMWOp::ADD || rmwOp == RMWOp::FADD || rmwOp == RMWOp::MAX || - rmwOp == RMWOp::MIN) && - (elementType.isF16() || elementType.isBF16() || elementType.isF32() || - elementType.isInteger(8) || elementType.isInteger(16) || - elementType.isInteger(32)); - - bool isDiscreteMask = false; - bool hasContinuousMaskSubview = false; - bool hasUsedReturn = !op.getResult().use_empty(); - MaskState mstate; - if (mask) { - auto constantMask = mask.getDefiningOp(); - if (constantMask && !isConstantMaskTrue(mask)) { - rewriter.eraseOp(op); - return success(); - } - isDiscreteMask = mstate.parse(mask, loc, rewriter).failed(); - if (!constantMask && !isDiscreteMask) { - // For dstMemref (store output), use subview to maintain reference to - // original memref. For inputVal (store input), use tensor.extract_slice - // to keep tensor semantics. - dstMemref = mstate.getSubview(ptr, loc, rewriter); - hasContinuousMaskSubview = true; - if (isHardwareSupported) { - auto inputTensorType = RankedTensorType::get( - inputMemrefType.getShape(), inputMemrefType.getElementType()); - if (!isa(inputVal.getType())) - inputVal = rewriter.create( - loc, inputTensorType, inputMemref, true, true); - inputVal = mstate.getExtractSlice(inputVal, loc, rewriter); - } else { - inputMemref = mstate.getSubview(inputMemref, loc, rewriter); - } - } - } - - bool needsReturnValueLock = - hasUsedReturn && hasContinuousMaskSubview && isHardwareSupported; - Value returnValueLock; - if (hasUsedReturn) { - auto tensorType = - RankedTensorType::get(ptrType.getShape(), ptrType.getElementType()); - auto alloc = rewriter.create( - loc, MemRefType::get(ptrType.getShape(), ptrType.getElementType())); - Value copySrc = ptr; - Value copyDst = alloc; - if (hasContinuousMaskSubview) { - // Masked-off atomic results are undefined. Copy only the active region - // so the old-value read and the atomic store use the same address view. - copySrc = dstMemref; - copyDst = mstate.getSubview(alloc, loc, rewriter); - } - if (needsReturnValueLock) { - auto lockType = MemRefType::get({1}, rewriter.getI64Type()); - auto lockVar = - rewriter.create(loc, lockType, Value()); - returnValueLock = lockVar.getResult(); - rewriter.create(loc, returnValueLock); - } - rewriter.create(loc, copySrc, copyDst); - Value tensorToReplace = rewriter.create( - loc, tensorType, alloc, true /* restrict */, true /* writable */); - rewriter.replaceOp(op, tensorToReplace); - } - - if (isDiscreteMask) { - if (rmwOp != RMWOp::XCHG) { - return op.emitError( - "Discrete mask is only expected for XCHG; other atomics " - "should be lowered without discrete masks"); - } - Value memrefMask = mask; - if (auto maskTypeT = dyn_cast(mask.getType())) { - MemRefType maskTypeM = - MemRefType::get(maskTypeT.getShape(), maskTypeT.getElementType()); - memrefMask = - rewriter.create(loc, maskTypeM, mask); - } - rewriter.create(op.getLoc(), TypeRange(), - inputMemref, dstMemref, memrefMask); - } else { - if (isHardwareSupported) { - auto storeOp = rewriter.create( - op.getLoc(), TypeRange{}, inputVal, dstMemref, atomicKind); - if (needsReturnValueLock) { - // The return-value copy and the atomic update must be one critical - // section. Mark the store so HIVM does not add another lock. - storeOp->setAttr(kAlreadySyncAttr, rewriter.getUnitAttr()); - rewriter.create(loc, returnValueLock); - } - } else if (rmwOp == RMWOp::XCHG) - rewriter.create(op.getLoc(), TypeRange(), - inputMemref, dstMemref); - else { - if (rmwOp == RMWOp::OR || rmwOp == RMWOp::XOR || rmwOp == RMWOp::AND) { - if (!elementType.isSignlessIntOrIndex()) { - return op->emitOpError() - << "must be signless-integer-like, but got " << elementType; - } - } - // Currently, for atomic kind and element type that is not supported by - // the hardware, we use software to simulate the computation. However, - // decompose now happens in both HFusion and HIVM, and is not consistent - // for 910B and 91095. Therefore, we convert to different atomic/store ops - // for now. This should be unified and refactored later. - if (compileOn91095Flag) { - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{inputMemref}, - ValueRange{dstMemref}, hfusionAtomicKind, - ArrayRef{}); - } else { - rewriter.create(op.getLoc(), TypeRange(), - inputMemref, dstMemref, - hfusionAtomicKind); - } - } - } - - if (op.getResult().use_empty()) { - rewriter.eraseOp(op); - } - return success(); -} - -LogicalResult -AtomicCASConverter::matchAndRewrite(triton::AtomicCASOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - // If the result of AtomicCASOp is not used, we don't need to load the old - // data stored at the ptr - auto ptr = adaptor.getPtr(); - auto cmp = op.getCmp(); - auto val = op.getVal(); - auto loc = op.getLoc(); - - FailureOr resolvedPtr = - resolveMemoryPointer(op.getPtr(), ptr, rewriter); - if (failed(resolvedPtr)) - return rewriter.notifyMatchFailure( - op, "unable to materialize the atomic CAS pointer as a memref"); - ptr = *resolvedPtr; - auto resType = dyn_cast(op.getResult().getType()); - if (!resType) { - return rewriter.notifyMatchFailure( - op, "atomicCASConverter: scalar will be handled by " - "ScalarAtomicCASCanonicalizer"); - } - - // 1. Simple case where no mask is used. - auto type = dyn_cast(ptr.getType()); - if (!type) { - // Seen when implicit broadcasting is done late in a chain of - // operations. The workaround is to broadcast the pointers early in the - // address calculation. A proper fix is complicated, but at least we can - // provide a better error message. - return rewriter.notifyMatchFailure( - op, "AtomicCASOp expects a memref, not a memref of pointers"); - } - - auto dstMemref = ptr; - // Well, linalg structure op wouldn't support mixed tensor/buffer semantics - // any more in latest LLVM(triton LLVM dependency has involed this), so we - // need to convert tensor to buffer early. - auto dstOriType = cast(dstMemref.getType()); - MemRefType dstType = - MemRefType::get(dstOriType.getShape(), dstOriType.getElementType()); - Value inputMemref = - rewriter.create(loc, dstType, val); - - Value cmpMemref = - rewriter.create(loc, dstType, cmp); - - // create element-wise map - int64_t rank = type.getRank(); - SmallVector inputDims; - auto context = rewriter.getContext(); - - for (int i = 0; i < rank; i++) { - inputDims.push_back(getAffineDimExpr(i, context)); - } - - SmallVector indexingMaps; - // As mask has been erased for now - // the number of input must be 2 - // the input memref is also the output memref - // Thus, there are a total of four inputs and outputs. - // so here we have 4 map to create - for (int i = 0; i < 4; i++) { // 4: 3 input and 1 output - indexingMaps.push_back(AffineMap::get(rank, 0, inputDims, context)); - } - - if (!op.getResult().use_empty()) { - auto tensorType = - RankedTensorType::get(type.getShape(), type.getElementType()); - auto alloc = rewriter.create( - loc, MemRefType::get(type.getShape(), type.getElementType())); - - // For the return value, don't need to care about mask for now - // this op don't support other, so we best not fill it - rewriter.create(loc, ptr, alloc); - Value tensor = rewriter.create( - loc, tensorType, alloc, true /* restrict */, true /* writable */); - rewriter.replaceOp(op, tensor); - } - - auto linalgOp = rewriter.create( - loc, ValueRange{dstMemref, cmpMemref, inputMemref}, - mlir::ValueRange{dstMemref}, indexingMaps, - mlir::ConverterUtils::getNParallelLoopsAttrs(rank), - [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange blockArgs) { - Value lhs = blockArgs[0]; - Value rhs = blockArgs[1]; - Value setValue = blockArgs[2]; - Value cond; - if (mlir::isa(lhs.getType())) { - cond = nestedBuilder.create( - nestedLoc, arith::CmpFPredicate::UEQ, lhs, rhs); - } else { - cond = nestedBuilder.create( - nestedLoc, arith::CmpIPredicate::eq, lhs, rhs); - } - auto ifOp = nestedBuilder.create( - nestedLoc, TypeRange{setValue.getType()}, cond, true); - { - OpBuilder::InsertionGuard guard(nestedBuilder); - nestedBuilder.setInsertionPointToEnd(&ifOp.getThenRegion().front()); - nestedBuilder.create(nestedLoc, setValue); - } - { - OpBuilder::InsertionGuard guard(nestedBuilder); - nestedBuilder.setInsertionPointToEnd(&ifOp.getElseRegion().front()); - nestedBuilder.create(nestedLoc, lhs); - } - nestedBuilder.setInsertionPointToEnd(nestedBuilder.getBlock()); - nestedBuilder.create(nestedLoc, - ifOp.getResult(0)); - }); - - const StringRef genericAtomicRMW = "GenericAtomicRMW"; - const StringRef memSemantic = "MemSemantic"; - const StringRef memSyncScope = "MemSyncScope"; - auto attr = mlir::StringAttr::get(context, "cas"); - - linalgOp->setAttr(genericAtomicRMW, attr); - linalgOp->setAttr(memSemantic, - rewriter.getStringAttr(stringifyEnum(op.getSem()))); - linalgOp->setAttr(memSyncScope, - rewriter.getStringAttr(stringifyEnum(op.getScope()))); - - linalgOp->setAttr("Software", rewriter.getUnitAttr()); - - // tt.atomicRMW op has two part of feature - // 1. load the old data at the ptr - // 2. atomically store the data on ub to the ptr - // at the same time it perform the action it has been assigned - // So we lower this op to load + atomically store - // - // The first part is not necessary when the returned value of atomic op - // is not used, it will be deleted cause it's meaningless - // Here, we preemptively determine whether it will be used - // and decide whether it is necessary to create the load process based on - // this assessment. - // - // logic of handling is copied - if (op.getResult().use_empty()) { - rewriter.eraseOp(op); - } - return success(); -} - -LogicalResult -ScalarStoreCanonicalizer::matchAndRewrite(triton::StoreOp op, - PatternRewriter &rewriter) const { - if (!op.getValue().getType().isIntOrIndexOrFloat()) { - return rewriter.notifyMatchFailure( - op, "ScalarStoreCanonicalizer handles scalar store scene!"); - } - auto ptr = op.getPtr(); - auto mask = op.getMask(); - auto value = op.getValue(); - if (mask) { - rewriter.replaceOpWithNewOp( - op, mask, [&](OpBuilder &b, Location loc) { - b.create(loc, ptr, value, op.getCache(), - op.getEvict()); - b.create(loc); - }); - return success(); - } - - auto ptrTy = RankedTensorType::get({(int64_t)1}, ptr.getType()); - auto ptrSplat = rewriter.create(op.getLoc(), ptrTy, ptr); - auto valTy = RankedTensorType::get({(int64_t)1}, value.getType()); - auto valSplat = rewriter.create(op.getLoc(), valTy, value); - auto newStoreOp = rewriter.create( - op.getLoc(), ptrSplat, valSplat, op.getCache(), op.getEvict()); - rewriter.replaceOp(op, newStoreOp); - return success(); -} - -LogicalResult -ScalarAtomicRMWCanonicalizer::matchAndRewrite(triton::AtomicRMWOp op, - PatternRewriter &rewriter) const { - if (!op.getVal().getType().isIntOrIndexOrFloat()) { - return rewriter.notifyMatchFailure( - op, "ScalarAtomicRMWCanonicalizer handles scalar atomic rmw op scene!"); - } - - auto ptr = op.getPtr(); - auto ptrTy = RankedTensorType::get({(int64_t)1}, ptr.getType()); - auto ptrSplat = rewriter.create(op.getLoc(), ptrTy, ptr); - auto valTy = RankedTensorType::get({(int64_t)1}, op.getVal().getType()); - auto valSplat = - rewriter.create(op.getLoc(), valTy, op.getVal()); - auto maskTy = RankedTensorType::get({(int64_t)1}, op.getMask().getType()); - auto maskSplat = - rewriter.create(op.getLoc(), maskTy, op.getMask()); - - auto newAtomicOp = rewriter.create( - op.getLoc(), valTy, op.getAtomicRmwOp(), ptrSplat, valSplat, maskSplat, - op.getSem(), op.getScope()); - auto idxZero = - rewriter.create(op.getLoc(), rewriter.getIndexAttr(0)); - rewriter.replaceOpWithNewOp(op, newAtomicOp, - ValueRange({idxZero})); - return success(); -} - -LogicalResult -ScalarAtomicCASCanonicalizer::matchAndRewrite(triton::AtomicCASOp op, - PatternRewriter &rewriter) const { - if (!op.getVal().getType().isIntOrIndexOrFloat() && - !op.getCmp().getType().isIntOrIndexOrFloat()) { - return rewriter.notifyMatchFailure( - op, "ScalarAtomicCASCanonicalizer handles scalar atomic cas op scene!"); - } - - auto ptr = op.getPtr(); - auto ptrTy = RankedTensorType::get({(int64_t)1}, ptr.getType()); - auto ptrSplat = rewriter.create(op.getLoc(), ptrTy, ptr); - auto cmpTy = RankedTensorType::get({(int64_t)1}, op.getCmp().getType()); - auto cmpSplat = - rewriter.create(op.getLoc(), cmpTy, op.getCmp()); - auto valTy = RankedTensorType::get({(int64_t)1}, op.getVal().getType()); - auto valSplat = - rewriter.create(op.getLoc(), valTy, op.getVal()); - - auto newAtomicOp = rewriter.create( - op.getLoc(), valTy, ptrSplat, cmpSplat, valSplat, op.getSem(), - op.getScope()); - auto idxZero = - rewriter.create(op.getLoc(), rewriter.getIndexAttr(0)); - rewriter.replaceOpWithNewOp(op, newAtomicOp, - ValueRange({idxZero})); - return success(); -} - -// The atomic max op with float input will be devided into -// two atomic max ops with integer input -// One handles the part of the tensor greater than zero -// the other deals with the part less than zero -// It will lead to maskAnalysis failure -// So here we need to revert the procedures in semantics.py -// The triton IR is like -// -// %cst_0 = arith.constant dense<0.000000e+00> : tensor<1x256xf32> -// %1 = tt.bitcast %value : tensor<1x256xf32> -> tensor<1x256xi32> -// %2 = tt.bitcast %ptr : tensor<1x256x!tt.ptr> -> -// tensor<1x256x!tt.ptr> %3 = arith.cmpf oge, %1, %cst_0 %4 = arith.cmpf -// olt, %1, %cst_0 %5 = arith.andi %8, %3 %6 = tt.atomic_rmw max, acq_rel, gpu, -// %2, %1, %5 : -// (tensor<1x256x!tt.ptr>, tensor<1x256xi32>, tensor<1x256xi1>) -> -// tensor<1x256xi32> -// %7 = arith.andi %8, %4 -// %8 = tt.atomic_rmw umin, acq_rel, gpu, %2, %1, %7 : -// (tensor<1x256x!tt.ptr>, tensor<1x256xi32>, tensor<1x256xi1>) -> -// tensor<1x256xi32> -// -// it's hard to handle and meaningless complicated for our device -// so we revert it to -// %0 = tt.atomic_rmw max, acq_rel, gpu, %23, %21, %8 : -// (tensor<1x256x!tt.ptr>, tensor<1x256xf32>, tensor<1x256xi1>) -> -// tensor<1x256xf32> -LogicalResult -AtomicMaxMinCanonicalizer::matchAndRewrite(triton::AtomicRMWOp op, - PatternRewriter &rewriter) const { - // Revert the op to its original form - auto ptrBitcastOp = op.getPtr().getDefiningOp(); - triton::SplatOp ptrSplatOp; - if (!ptrBitcastOp) { - // For tensor atomics on a scalar base pointer, semantic.py emits - // splat(bitcast(ptr -> ptr)) - // instead of - // bitcast(tensor> -> tensor>). - // Accept both forms so the expanded integer atomics can be fused before - // discrete-mask conversion. - ptrSplatOp = op.getPtr().getDefiningOp(); - if (ptrSplatOp) - ptrBitcastOp = ptrSplatOp.getSrc().getDefiningOp(); - } - auto valueBitcastOp = op.getVal().getDefiningOp(); - if (!ptrBitcastOp || !valueBitcastOp) { - return failure(); - } - - auto eraseDeadPointerCasts = [&]() { - if (ptrSplatOp && ptrSplatOp->use_empty()) - rewriter.eraseOp(ptrSplatOp); - if (ptrBitcastOp->use_empty()) - rewriter.eraseOp(ptrBitcastOp); - }; - - // We only need to handle the op when the element type is float - auto elementType = getElementTypeOrSelf(valueBitcastOp.getSrc().getType()); - if (!isa(elementType)) { - return failure(); - } - - auto rmwOp = op.getAtomicRmwOp(); - // here we know that atomic UMAX/UMIN - // is created by special logic of triton right now - // so we can simply delete it - if (rmwOp == triton::RMWOp::UMAX || rmwOp == triton::RMWOp::UMIN) { - // if the return value of op is used, we can't simply erase it - if (op.getResult().use_empty()) { - rewriter.eraseOp(op); - eraseDeadPointerCasts(); - return success(); - } - return failure(); - } - - if (rmwOp != triton::RMWOp::MAX && rmwOp != triton::RMWOp::MIN) { - return failure(); - } - - // 1. Though semantic interpreter will generate full true tensor as original - // mask if atomicrmwOp don't have it, above float devision process will also - // generate positive and negative comparison mask, which will cause to fold - // true mask. - // 2. While if atomicrmwOp has original mask, there exists andiop between - // original mask and positive/negative comparison mask - // - // Here wanna extract original mask - Value originalMask = op.getMask(); - auto createAllTrueMask = [&]() -> Value { - Type maskType = op.getMask().getType(); - if (auto shapedMaskType = dyn_cast(maskType)) { - return rewriter.create( - op->getLoc(), DenseElementsAttr::get(shapedMaskType, true)); - } - if (maskType.isInteger(1)) { - return rewriter.create(op->getLoc(), - rewriter.getBoolAttr(true)); - } - return {}; - }; - - if (auto andOp = originalMask.getDefiningOp()) - // LHS is convention in semantic interpreter - originalMask = andOp.getLhs(); - else if (auto xorOp = originalMask.getDefiningOp()) { - // Current f32 atomic_min uses !signbit as the positive mask: - // shrui(value_bits, 31) -> cmpi ne 0 -> xori true. - if ((rmwOp != triton::RMWOp::MIN && rmwOp != triton::RMWOp::MAX) || - (!elementType.isF32() && !elementType.isF64()) || - !matchPattern(xorOp.getRhs(), m_One())) { - return failure(); - } - - auto cmpOp = xorOp.getLhs().getDefiningOp(); - if (!cmpOp || cmpOp.getPredicate() != arith::CmpIPredicate::ne || - !matchPattern(cmpOp.getRhs(), m_Zero())) - return op->emitError("Illegal mask for atomicrmwOp of float type"); - - auto shiftOp = cmpOp.getLhs().getDefiningOp(); - if (!shiftOp || shiftOp.getLhs() != valueBitcastOp.getResult()) - return op->emitError("Illegal mask for atomicrmwOp of float type"); - - // Restore the implicit all-true mask. - originalMask = createAllTrueMask(); - if (!originalMask) - return failure(); - } else if (auto cmpOp = originalMask.getDefiningOp()) { - // Scalar floating-point atomic max/min represents !signbit as: - // shrui(value_bits, 31/63) -> cmpi eq 0. - if (cmpOp.getPredicate() != arith::CmpIPredicate::eq || - !matchPattern(cmpOp.getRhs(), m_Zero())) - return failure(); - - auto shiftOp = cmpOp.getLhs().getDefiningOp(); - if (!shiftOp || shiftOp.getLhs() != valueBitcastOp.getResult()) - return failure(); - - originalMask = createAllTrueMask(); - if (!originalMask) - return failure(); - } else if (auto cmpOp = originalMask.getDefiningOp()) { - if (cmpOp.getPredicate() != mlir::arith::CmpFPredicate::OGE || - !matchPattern(cmpOp.getRhs(), - /*positive float zero matcher*/ m_PosZeroFloat())) - // Here recheck frontend interpreter generation in no manual mask state - return op->emitError("Illegal mask for atomicrmwOp of float type"); - // Restore original true mask - originalMask = createAllTrueMask(); - if (!originalMask) - return failure(); - } else - return op->emitError("Illegal mask for atomicrmwOp of float type"); - - Value originalPtr = ptrBitcastOp.getSrc(); - if (ptrSplatOp) { - auto ptrTensorType = dyn_cast(op.getPtr().getType()); - if (!ptrTensorType) - return failure(); - auto originalPtrType = RankedTensorType::get( - ptrTensorType.getShape(), ptrBitcastOp.getSrc().getType(), - ptrTensorType.getEncoding()); - originalPtr = rewriter.create(op.getLoc(), originalPtrType, - ptrBitcastOp.getSrc()); - } - - auto originAtomicOp = rewriter.create( - op.getLoc(), valueBitcastOp.getSrc().getType(), op.getAtomicRmwOp(), - originalPtr, valueBitcastOp.getSrc(), originalMask, op.getSem(), - op.getScope()); - - // if the return value of op is used - // we need to handle its usage - // In semantic.py, if the atomic Max/Min with float input is used - // It will use select + bitcast to get float value - // so here we need to revert it too - // - // For example: - // %0 = tt.atomic_rmw max, acq_rel, gpu, %gm, %input, %mask1 : - // (tensor<32x!tt.ptr>... %1 = tt.atomic_rmw umin, acq_rel, gpu, %gm, - // %input, %mask2 : (tensor<32x!tt.ptr>... %2 = arith.select - // %devidedMask, %0, %1 : tensor<32xi1>, tensor<32xi32> %3 = tt.bitcast %2 : - // tensor<32xi32> -> tensor<32xf32> tt.store %outputMemref, %3 : - // tensor<32x!tt.ptr> - // - // will be revert to: - // %0 = tt.atomic_rmw max, acq_rel, gpu, %gm, %input, %mask : - // (tensor<32x!tt.ptr>... tt.store %outputMemref, %0 : - // tensor<32x!tt.ptr> - // - if (!op.getResult().use_empty()) { - for (OpOperand &use : op->getUses()) { - auto selectOp = dyn_cast(use.getOwner()); - if (!selectOp) - continue; - - for (OpOperand &selectUse : selectOp->getUses()) { - if (auto bitcastOp = - dyn_cast(selectUse.getOwner())) { - bitcastOp.getResult().replaceAllUsesWith(originAtomicOp); - } - } - } - rewriter.replaceOp(op, originAtomicOp); - } else { - rewriter.eraseOp(op); - } - - // The restored atomic uses the original floating-point GM pointer. Remove - // the expanded pointer splat/bitcast once the paired integer atomic is gone. - eraseDeadPointerCasts(); - - return success(); -} - -StoreConverter::StoreConverter(MLIRContext *context) - : OpConversionPattern(context) {} - -LogicalResult -StoreConverter::matchAndRewrite(triton::StoreOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - - // triton store op basic - auto mask = op.getMask(); - auto loc = op.getLoc(); - insertDebugNopForMask(mask, rewriter); - auto ptr = adaptor.getPtr(); - auto val = adaptor.getValue(); - - FailureOr resolvedPtr = - resolveMemoryPointer(op.getPtr(), ptr, rewriter); - if (failed(resolvedPtr)) - return rewriter.notifyMatchFailure( - op, "unable to materialize the store pointer as a memref"); - ptr = *resolvedPtr; - // 1. boundary size check - auto boundaryCheck = op.getBoundaryCheck(); - if (!boundaryCheck.empty()) { - auto makeTensorPtrOp = op.getPtr().getDefiningOp(); - auto boundarySizes = mlir::ConverterUtils::getBoundarySizes( - boundaryCheck, /*remapped*/ ptr, loc, rewriter); - SmallVector srcOffsets; - SmallVector dstOffsets(boundarySizes.size(), - rewriter.getIndexAttr(0)); - if (makeTensorPtrOp) { - auto zeroVal = rewriter.createOrFold( - loc, rewriter.getI32IntegerAttr(0)); - for (auto [idx, offVal] : llvm::enumerate(makeTensorPtrOp.getOffsets())) { - if (llvm::find(boundaryCheck, idx) == boundaryCheck.end()) { - srcOffsets.push_back(dstOffsets[idx]); - continue; - } - Value offset = - rewriter.createOrFold(loc, zeroVal, offVal); - Value size = - getValueOrCreateConstantIndexOp(rewriter, loc, boundarySizes[idx]); - offset = rewriter.createOrFold(loc, offset, zeroVal); - offset = rewriter.createOrFold( - loc, rewriter.getIndexType(), offset); - OpFoldResult ofr; - if (auto constOp = offset.getDefiningOp()) { - ofr = constOp.getValue(); - } else { - ofr = offset; - } - ofr = minOpFoldResult(ofr, size, loc, rewriter); - boundarySizes[idx] = subOpFoldResult(size, ofr, loc, rewriter); - srcOffsets.push_back(ofr); - } - } else { - srcOffsets = dstOffsets; - } - auto srcSlice = mlir::ConverterUtils::makeExtractSliceOp( - val, srcOffsets, boundarySizes, loc, rewriter); - auto dstSubview = mlir::ConverterUtils::makeSubViewOp( - ptr, dstOffsets, boundarySizes, loc, rewriter); - auto storeOp = rewriter.create( - loc, srcSlice, dstSubview); - storeOp.setWritable(true); - rewriter.eraseOp(op); - return success(); - } - - // 2. Simple load with no mask - if (!mask) { - auto storeOp = rewriter.create( - loc, val, ptr); - storeOp.setWritable(true); - rewriter.eraseOp(op); - return success(); - } - - // 3. Continuous masked stores. - // Analyze the mask operand to determine at runtime the size of the data we - // are moving. - MaskState mstate; - auto isContMask = mstate.parse(mask, loc, rewriter); - - if (isContMask.failed()) { - return failure(); - } - LLVM_DEBUG({ llvm::dbgs() << *getModuleOpFromOperation(op) << "\n"; }); - auto srcSlice = mstate.getExtractSlice(val, loc, rewriter); - auto dstSubview = mstate.getSubview(ptr, loc, rewriter); - auto storeOp = rewriter.create( - loc, srcSlice, dstSubview); - storeOp.setWritable(true); - rewriter.eraseOp(op); - return success(); -} - -bool ReinterpretCastStrideCanonicalizer::hasFixableZeroStride( - memref::ReinterpretCastOp op) { - auto staticSizes = op.getStaticSizes(); - auto staticStrides = op.getStaticStrides(); - auto dynamicStrides = op.getStrides(); - - if (staticSizes.size() != staticStrides.size()) - return false; - - // now handle: size all static - if (llvm::any_of(staticSizes, ShapedType::isDynamic)) - return false; - - unsigned dynStrideIdx = 0; - for (unsigned i = 0; i < staticStrides.size(); ++i) { - // now handle: dynamic stride 0 with static size 1 - if (!ShapedType::isDynamic(staticStrides[i])) - continue; - - if (dynStrideIdx >= dynamicStrides.size()) - return false; - - Value st = dynamicStrides[dynStrideIdx]; - dynStrideIdx++; - if (staticSizes[i] == 1 && mlir::isZero(OpFoldResult(st))) - return true; - } - return false; -} - -LogicalResult ReinterpretCastStrideCanonicalizer::matchAndRewrite( - memref::ReinterpretCastOp op, PatternRewriter &rewriter) const { - if (!hasFixableZeroStride(op)) - return failure(); - - auto staticSizes = op.getStaticSizes(); - auto staticStrides = op.getStaticStrides(); - auto dynamicStrides = op.getStrides(); - - SmallVector newDynamicStrides; - newDynamicStrides.reserve(dynamicStrides.size()); - - unsigned dynStrideIdx = 0; - bool changed = false; - Value c1 = - rewriter.create(op.getLoc(), rewriter.getIndexAttr(1)); - - for (unsigned i = 0, e = staticStrides.size(); i < e; ++i) { - if (!ShapedType::isDynamic(staticStrides[i])) - continue; - - if (dynStrideIdx >= dynamicStrides.size()) - return failure(); - - Value oldStride = dynamicStrides[dynStrideIdx]; - dynStrideIdx++; - if (staticSizes[i] == 1 && mlir::isZero(OpFoldResult(oldStride))) { - newDynamicStrides.push_back(c1); - changed = true; - } else { - newDynamicStrides.push_back(oldStride); - } - } - - // all dynStride should be visited, and at least one should be changed - if (dynStrideIdx != dynamicStrides.size()) - return failure(); - if (!changed) - return failure(); - - auto newReinterpretCast = rewriter.create( - op.getLoc(), cast(op.getResult().getType()), op.getSource(), - op.getOffsets(), op.getSizes(), newDynamicStrides, op.getStaticOffsets(), - op.getStaticSizes(), op.getStaticStrides()); - - rewriter.replaceOp(op, newReinterpretCast.getResult()); - - return success(); -} - -} // namespace LoadStoreConverter diff --git a/third_party/ascend/lib/TritonToLinalg/MarkTensorKindPass.cpp b/third_party/ascend/lib/TritonToLinalg/MarkTensorKindPass.cpp deleted file mode 100644 index 59858b80f2..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/MarkTensorKindPass.cpp +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/MarkTensorKindPass.h" -#include "ascend/include/Dialect/TritonAscend/IR/TritonAscendDialect.h" - -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/Support/Debug.h" - -#define DEBUG_TYPE "mark-tensor-kind" - -using namespace mlir; -using namespace triton; -const unsigned INT_BIT_WIDTH = 32; -const unsigned SET_INIT_SIZE = 16; - -template struct has_getPtr : std::false_type {}; -template -struct has_getPtr().getPtr())>> - : std::true_type {}; - -template struct has_getSrc : std::false_type {}; -template -struct has_getSrc().getSrc())>> - : std::true_type {}; - -template struct has_getDst : std::false_type {}; -template -struct has_getDst().getDst())>> - : std::true_type {}; - -template struct has_getBase : std::false_type {}; -template -struct has_getBase().getBase())>> - : std::true_type {}; - -template static Value extractPointer(OpTy op) { - if constexpr (has_getPtr::value) - return op.getPtr(); - else if constexpr (has_getDst::value) - return op.getDst(); - else if constexpr (has_getSrc::value) - return op.getSrc(); - else if constexpr (has_getBase::value) - return op.getBase(); - else { - Operation *raw = op.getOperation(); - if (!raw || raw->getNumOperands() == 0) - return Value(); - return raw->getOperand(0); - } -} - -static void setBlockArgumentAttr(BlockArgument blockArg, triton::FuncOp func, - TensorKind tensorKind) { - unsigned argIdx = blockArg.getArgNumber(); - auto existingAttr = - func.getArgAttrOfType(argIdx, "tt.tensor_kind"); - TensorKind oldVal = existingAttr - ? static_cast(existingAttr.getInt()) - : TensorKind::NONE; - - TensorKind finalVal = tensorKind; - if ((oldVal == TensorKind::INPUT && tensorKind == TensorKind::OUTPUT) || - (oldVal == TensorKind::OUTPUT && tensorKind == TensorKind::INPUT)) { - finalVal = TensorKind::INPUT_OUTPUT; - } else if (oldVal == TensorKind::INPUT_OUTPUT) { - finalVal = oldVal; - } - - LLVM_DEBUG(llvm::dbgs() << "Setting tensor_kind for argument " << argIdx - << ": " << finalVal << "\n";); - - func.setArgAttr( - argIdx, "tt.tensor_kind", - IntegerAttr::get(IntegerType::get(func.getContext(), INT_BIT_WIDTH), - static_cast(finalVal))); -} - -template -static void addTensorKindToArguments(OpTy op, TensorKind tensorKind) { - Value ptr = extractPointer(op); - if (!ptr) - return; - - LLVM_DEBUG(llvm::dbgs() << "Processing op: " << *op.getOperation() << "\n";); - - Value cur = ptr; - llvm::SmallPtrSet visited; - while (visited.insert(cur).second) { - if (auto blockArg = dyn_cast(cur)) { - if (auto func = dyn_cast_or_null( - blockArg.getOwner()->getParentOp())) { - if (blockArg.getOwner() == &func.getBody().front() && - isa(blockArg.getType())) { - setBlockArgumentAttr(blockArg, func, tensorKind); - break; - } - } - } - - Operation *defOp = cur.getDefiningOp(); - if (!defOp || defOp->getNumOperands() == 0) - break; - cur = defOp->getOperand(0); - } -} - -template -struct MarkTensorKindPattern : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(OpTy op, - PatternRewriter &rewriter) const override { - addTensorKindToArguments(op, Kind); - return success(); - } -}; - -void MarkTensorKindPass::runOnOperation() { - RewritePatternSet patterns(&getContext()); - - // INPUT tensors - patterns.add< - MarkTensorKindPattern, - MarkTensorKindPattern, - MarkTensorKindPattern, - MarkTensorKindPattern, - MarkTensorKindPattern>( - &getContext()); - - // OUTPUT tensors - patterns.add< - MarkTensorKindPattern, - MarkTensorKindPattern, - MarkTensorKindPattern, - MarkTensorKindPattern, - MarkTensorKindPattern>(&getContext()); - - // INPUT_OUTPUT tensors - patterns.add< - MarkTensorKindPattern, - MarkTensorKindPattern>( - &getContext()); - - (void)applyPatternsGreedily(getOperation(), std::move(patterns)); -} - -std::unique_ptr> triton::createMarkTensorKindPass() { - return std::make_unique(); -} diff --git a/third_party/ascend/lib/TritonToLinalg/TritonOpConverter.cpp b/third_party/ascend/lib/TritonToLinalg/TritonOpConverter.cpp deleted file mode 100644 index f1d7516e5b..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/TritonOpConverter.cpp +++ /dev/null @@ -1,4435 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/TritonOpConverter.h" -#include "ascend/include/TritonToLinalg/BlockPtrAnalysis.h" -#include "ascend/include/TritonToLinalg/MaskAnalysis.h" -#include "ascend/include/TritonToLinalg/TritonToLinalgPass.h" -#include "ascend/include/Utils/DebugUtils.h" -#include "ascend/include/Utils/Utils.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/SCF/IR/SCF.h" - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/LogicalResult.h" -#include "llvm/Support/raw_ostream.h" -#include -#include -#include - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Math/IR/Math.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/MemRef/Transforms/Passes.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/ValueRange.h" -#include "mlir/Interfaces/CallInterfaces.h" - -#include "bishengir/Dialect/Annotation/IR/Annotation.h" -#include "bishengir/Dialect/HFusion/IR/HFusion.h" -#include "bishengir/Dialect/HIVM/IR/HIVM.h" -#include "bishengir/Dialect/Scope/IR/Scope.h" - -namespace TTOpConverters { -using namespace mlir; -using namespace triton; - -/** - * Retrieves a boolean environment variable. - * @param envVar The name of the environment variable. - * @param defaultValue The default value to return if the variable is not set - * or cannot be parsed. - * @return true if the environment variable exists and its value is parsed as - * "true", otherwise returns defaultValue. Parsing rules (case-insensitive): - * "true" values: any non-empty string not equal to "0", "false", "no", "off" - * is considered true. "false" values: an empty string or a string equal to - * any of the false literals is considered false. - */ -bool getEnvBool(const char *envVar, bool defaultValue) { - const char *val = std::getenv(envVar); - if (val == nullptr) { - return defaultValue; // variable not set - } - - std::string s(val); - // Convert to lowercase for easier comparison - std::transform(s.begin(), s.end(), s.begin(), - [](unsigned char c) { return std::tolower(c); }); - - // Common false literals - if (s.empty() || s == "0" || s == "false" || s == "no" || s == "off") { - return false; - } - // All other cases (including "1", "true", "yes", "on", etc.) are considered - // true - return true; -} - -static llvm::SmallString -generateUniqueFuncName(ModuleOp moduleOp, llvm::StringRef funcNameBase) { - llvm::SmallString funcName = funcNameBase; - int uniqueId = 0; - while (SymbolTable::lookupSymbolIn(moduleOp, funcName)) { - funcName = funcNameBase; - funcName += ("_" + std::to_string(uniqueId++)); - } - return funcName; -} - -// Indirect load/store conversion normally consumes the already-converted -// memref adaptor directly. That is correct for ordinary pointer bases, but it -// skips BlockDataParser::materializePointer for an int_to_ptr base and loses -// the required one-element identity view. Only scalar pointer-preserving -// producers are followed here; tensor-of-pointers and arbitrary pointer -// arithmetic retain their established conversion path. -static bool isIntToPtrBasedScalarPointer(Value value) { - auto pointerType = dyn_cast(value.getType()); - if (!pointerType || isa(pointerType.getPointeeType())) - return false; - - SmallPtrSet visited; - while (value && visited.insert(value).second) { - if (value.getDefiningOp()) - return true; - - Operation *producer = value.getDefiningOp(); - if (!producer) - return false; - if (auto addPtr = dyn_cast(producer)) { - value = addPtr.getPtr(); - continue; - } - if (auto bitcast = dyn_cast(producer)) { - value = bitcast.getSrc(); - continue; - } - return false; - } - return false; -} - -// A volatile indirect load whose scalar source is obtained through a Triton -// pointer bitcast needs the same one-element identity descriptor as the -// scalar-pointer path. Restrict this provenance check to scalar pointers and -// a pointer-preserving bitcast/addptr chain so ordinary dynamic memrefs and -// tensor-of-pointers keep their established ABI. -static bool isVolatileBitcastScalarPointer(Value value) { - auto pointerType = dyn_cast(value.getType()); - if (!pointerType || isa(pointerType.getPointeeType())) - return false; - - SmallPtrSet visited; - bool sawBitcast = false; - while (value && visited.insert(value).second) { - if (auto bitcast = value.getDefiningOp()) { - sawBitcast = true; - value = bitcast.getSrc(); - continue; - } - if (auto addPtr = value.getDefiningOp()) { - value = addPtr.getPtr(); - continue; - } - break; - } - return sawBitcast; -} - -LogicalResult -BitcastConverter::matchAndRewrite(triton::BitcastOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Value result; - auto loc = op.getLoc(); - - if (auto dstPtrTy = dyn_cast(op.getType())) { - auto srcPtrTy = cast(op.getSrc().getType()); - auto resType = - MemRefType::get({ShapedType::kDynamic}, dstPtrTy.getPointeeType()); - - auto i1Ty = rewriter.getIntegerType(1); - auto i8Ty = rewriter.getIntegerType(8); - bool isI1toI8 = (srcPtrTy.getPointeeType() == i1Ty) && - (dstPtrTy.getPointeeType() == i8Ty); - // handling special case: ptr -> ptr, directly forward without - // arith.bitcast - if (isI1toI8) { - // TypeConverter has already converted i1 to i8 memref, - LLVM_DEBUG({ - llvm::dbgs() - << "[BitcastConverter] Special i1->i8 pointer bitcast. Forward " - "without arith.bitcast. srcConvertedTy=" - << adaptor.getSrc().getType() << "\n"; - }); - rewriter.replaceOp(op, adaptor.getSrc()); - return success(); - } - result = rewriter.create(loc, resType, adaptor.getSrc()); - } else { - // handling normal case: bitcast between tensors/memrefs - result = - rewriter.create(loc, op.getType(), adaptor.getSrc()); - } - rewriter.replaceOp(op, result); - return success(); -} - -LogicalResult -TransposeConverter::matchAndRewrite(triton::TransOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - insertDebugNop(op.getLoc(), rewriter); - auto src = adaptor.getSrc(); - auto res = ConverterUtils::getTransposedValue(src, op.getLoc(), rewriter, - op.getOrder()); - rewriter.replaceOp(op, res); - return success(); -} - -bool hasScalarPointerResult(scf::IfOp op) { - bool hasPointerResult = false; - for (Type resultType : op.getResultTypes()) { - auto pointerType = dyn_cast(resultType); - if (!pointerType) - continue; - if (isa(pointerType.getPointeeType())) - return false; - hasPointerResult = true; - } - return hasPointerResult; -} - -bool isScalarPointerSelect(arith::SelectOp op) { - auto pointerType = dyn_cast(op.getType()); - return pointerType && !isa(pointerType.getPointeeType()); -} - -// Convert a scalar Triton pointer type to the canonical memref descriptor used -// by TritonTypeConverter. It is reconstructed only after a complete byte -// address has crossed control flow, so it needs no dynamic layout. -static FailureOr -getScalarPointerCarrierType(Type originalType, - const TypeConverter &typeConverter) { - auto pointerType = dyn_cast(originalType); - if (!pointerType || isa(pointerType.getPointeeType())) - return failure(); - - Type convertedType = typeConverter.convertType(originalType); - if (!convertedType) - return failure(); - auto memrefType = dyn_cast(convertedType); - if (!memrefType) - return failure(); - - return memrefType; -} - -static Type getIfResultCarrierType(Type originalType) { - if (isa(originalType)) - return IntegerType::get(originalType.getContext(), 64); - - // This converter exists only to carry scalar pointers as integer addresses. - // Sibling results already have valid SSA types and must stay unchanged; using - // the global type converter here would turn a tensor mask into a memref even - // though its existing users still consume the tensor value. - return originalType; -} - -// Materialize a no-op-compatible memref cast to the common carrier. Returning -// failure for non-memref or incompatible values prevents the pointer transport -// pattern from silently changing element, shape, rank, or memory-space types. -static FailureOr -castToMemRefCarrier(Value value, MemRefType carrierType, Location loc, - ConversionPatternRewriter &rewriter) { - if (value.getType() == carrierType) - return value; - - auto sourceType = dyn_cast(value.getType()); - if (!sourceType || - !memref::CastOp::areCastCompatible(sourceType, carrierType)) - return failure(); - - return rewriter.create(loc, carrierType, value).getResult(); -} - -// Dialect conversion may adapt a lane-local pointer descriptor such as -// `memref<1xT, strided<[1], offset: ?>>` to the canonical scalar-pointer -// carrier `memref`. Address materialization must inspect the original -// descriptor: its dynamic layout offset is part of the represented address and -// cannot be recovered from the shape-only carrier type. -// -// Only unwrap a one-to-one memref materialization that preserves rank, element -// type, and memory space. Other unrealized casts may represent a real element -// or address-space conversion and must remain visible to their converters. -static Value unwrapPointerDescriptorMaterialization(Value value) { - auto materialization = value.getDefiningOp(); - if (!materialization || materialization.getInputs().size() != 1 || - materialization.getOutputs().size() != 1) - return value; - - Value source = materialization.getInputs().front(); - auto sourceType = dyn_cast(source.getType()); - auto targetType = dyn_cast(value.getType()); - if (!sourceType || !targetType || - sourceType.getRank() != targetType.getRank() || - sourceType.getElementType() != targetType.getElementType() || - sourceType.getMemorySpace() != targetType.getMemorySpace()) - return value; - - return source; -} - -// Converts a memref descriptor into the complete byte address represented by -// that descriptor. extract_aligned_pointer_as_index yields the aligned buffer -// pointer; the descriptor's element offset must therefore be converted to -// bytes and added explicitly before the address crosses control flow. -static FailureOr -materializePointerAddress(Value value, Location loc, - ConversionPatternRewriter &rewriter) { - value = unwrapPointerDescriptorMaterialization(value); - auto memrefType = dyn_cast(value.getType()); - if (!memrefType) - return failure(); - Type elementType = memrefType.getElementType(); - if (!elementType.isIntOrFloat()) - return failure(); - - // A canonical scalar PointerCast already stores the complete byte address - // that IntToPtr received. Extracting its aligned pointer immediately and - // casting it back to i64 is an identity round trip. In current failing - // kernels this chain also survives into InjectSync, so folding it keeps a - // redundant region-local form out of the backend. Only fold the canonical - // one-dimensional carrier; descriptors with a non-zero offset or a non-unit - // stride still require the general address materialization below. - if (auto pointerCast = value.getDefiningOp(); - pointerCast && pointerCast->hasAttr(kScalarPointerCarrierAttr)) { - auto [strides, offset] = memrefType.getStridesAndOffset(); - if (pointerCast.getAddrs().size() == 1 && memrefType.getRank() == 1 && - offset == 0 && strides.size() == 1 && strides.front() == 1) { - Value address = pointerCast.getAddrs().front(); - if (address.getType().isInteger(64)) - return address; - if (address.getType().isIndex()) - return rewriter - .create(loc, rewriter.getI64Type(), address) - .getResult(); - } - } - - Value address = - rewriter.create(loc, value); - int64_t staticOffset = memrefType.getStridesAndOffset().second; - if (staticOffset != 0) { - Value elementOffset; - if (ShapedType::isDynamic(staticOffset)) { - elementOffset = - rewriter.create(loc, value) - .getOffset(); - } else { - elementOffset = - rewriter.create(loc, staticOffset); - } - - int64_t elementBytes = (elementType.getIntOrFloatBitWidth() + 7) / 8; - if (elementBytes != 1) { - Value scale = rewriter.create(loc, elementBytes); - elementOffset = rewriter.create(loc, elementOffset, scale); - } - address = rewriter.create(loc, address, elementOffset); - } - return rewriter - .create(loc, rewriter.getI64Type(), address) - .getResult(); -} - -LogicalResult -IfConverter::matchAndRewrite(scf::IfOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - const TypeConverter *typeConverter = getTypeConverter(); - if (!typeConverter) - return rewriter.notifyMatchFailure(op, "requires a type converter"); - if (!hasScalarPointerResult(op)) - return failure(); - if (!op->hasAttr(kScalarPointerCarrierBoundaryAttr)) - return rewriter.notifyMatchFailure( - op, "scalar-pointer if is missing its carrier boundary marker"); - - SmallVector convertedResultTypes; - convertedResultTypes.reserve(op.getNumResults()); - for (Type resultType : op.getResultTypes()) { - convertedResultTypes.push_back(getIfResultCarrierType(resultType)); - } - - auto newIfOp = rewriter.create(op.getLoc(), convertedResultTypes, - adaptor.getCondition(), - /*withElseRegion=*/true); - newIfOp->setAttrs(op->getAttrs()); - newIfOp->setAttr(kScalarPointerCarrierBoundaryAttr, - UnitAttr::get(rewriter.getContext())); - - // Move the original regions instead of cloning them. Besides preserving - // side effects, this keeps nested operations in the conversion driver's - // worklist so their operands are remapped normally. - rewriter.eraseBlock(newIfOp.thenBlock()); - rewriter.eraseBlock(newIfOp.elseBlock()); - rewriter.inlineRegionBefore(op.getThenRegion(), newIfOp.getThenRegion(), - newIfOp.getThenRegion().end()); - rewriter.inlineRegionBefore(op.getElseRegion(), newIfOp.getElseRegion(), - newIfOp.getElseRegion().end()); - - SmallVector replacementResults; - replacementResults.reserve(op.getNumResults()); - rewriter.setInsertionPointAfter(newIfOp); - for (auto [originalResultType, newResult] : - llvm::zip(op.getResultTypes(), newIfOp.getResults())) { - if (!isa(originalResultType)) { - replacementResults.push_back(newResult); - continue; - } - FailureOr resultType = - getScalarPointerCarrierType(originalResultType, *typeConverter); - if (failed(resultType)) - return rewriter.notifyMatchFailure( - op, "could not reconstruct a scalar pointer result"); - replacementResults.push_back( - createScalarPointerCast(rewriter, op.getLoc(), *resultType, newResult) - .getResult()); - } - rewriter.replaceOp(op, replacementResults); - return success(); -} - -LogicalResult PointerSelectConverter::matchAndRewrite( - arith::SelectOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - if (!isScalarPointerSelect(op)) - return failure(); - - const TypeConverter *typeConverter = getTypeConverter(); - if (!typeConverter) - return rewriter.notifyMatchFailure(op, "requires a type converter"); - - FailureOr resultType = - getScalarPointerCarrierType(op.getType(), *typeConverter); - if (failed(resultType)) - return rewriter.notifyMatchFailure( - op, "could not build a scalar pointer memref carrier"); - - FailureOr trueAddress = - materializePointerAddress(adaptor.getTrueValue(), op.getLoc(), rewriter); - FailureOr falseAddress = - materializePointerAddress(adaptor.getFalseValue(), op.getLoc(), rewriter); - if (failed(trueAddress) || failed(falseAddress)) - return rewriter.notifyMatchFailure( - op, "selected pointer values have no complete integer address"); - - auto selectedAddress = rewriter.create( - op.getLoc(), rewriter.getI64Type(), adaptor.getCondition(), *trueAddress, - *falseAddress); - selectedAddress->setAttrs(op->getAttrs()); - auto pointerCast = createScalarPointerCast(rewriter, op.getLoc(), *resultType, - selectedAddress.getResult()); - rewriter.replaceOp(op, pointerCast.getResult()); - return success(); -} - -LogicalResult -YieldConverter::matchAndRewrite(scf::YieldOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto parentIf = dyn_cast(op->getParentOp()); - if (!parentIf || !parentIf->hasAttr(kScalarPointerCarrierBoundaryAttr)) - return failure(); - - SmallVector convertedOperands(adaptor.getOperands()); - - if (parentIf.getNumResults() != convertedOperands.size()) - return rewriter.notifyMatchFailure(op, "yield/result arity does not match"); - - for (auto [index, targetType] : llvm::enumerate(parentIf.getResultTypes())) { - Value &operand = convertedOperands[index]; - if (operand.getType() == targetType) - continue; - - if (targetType.isInteger(64) && isa(operand.getType())) { - FailureOr address = - materializePointerAddress(operand, op.getLoc(), rewriter); - if (failed(address)) - return rewriter.notifyMatchFailure( - op, "could not materialize a yielded pointer address"); - operand = *address; - continue; - } - auto targetMemrefType = dyn_cast(targetType); - if (!targetMemrefType) - return rewriter.notifyMatchFailure( - op, "converted yield operand is incompatible with if result"); - - FailureOr casted = - castToMemRefCarrier(operand, targetMemrefType, op.getLoc(), rewriter); - if (failed(casted)) - return rewriter.notifyMatchFailure( - op, "converted yield operand is incompatible with if result"); - operand = *casted; - } - - rewriter.replaceOpWithNewOp(op, convertedOperands); - return success(); -} - -LogicalResult -AdvanceConverter::matchAndRewrite(triton::AdvanceOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - llvm::SmallDenseMap known; - return BlockDataParser::rewriteAdvanceOp(op, rewriter, known); -} - -// ToDo: -// 1. Refactor MakeTensorPtrConverter and AdvanceConverter with -// memref::ReinterpretCastOp and memref::SubViewOp. -// Use recast to describe full shape of tensor, and use subview to represent -// current block tensor. -LogicalResult MakeTensorPtrConverter::matchAndRewrite( - triton::MakeTensorPtrOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - llvm::SmallDenseMap known; - return BlockDataParser::rewriteMakeTensorPtrOp(op, adaptor.getBase(), - rewriter, known); -} - -LogicalResult PreciseDivConverter::matchAndRewrite( - triton::PreciseDivFOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Value opa = adaptor.getX(); - Value opb = adaptor.getY(); - auto loc = op.getLoc(); - - if (opa.getType() != opb.getType()) - return rewriter.notifyMatchFailure(op, "operands must have the same type"); - - // Let DivFOp infer its result type from converted operands. PreciseDivFOp - // is valid for both scalar and tensor floating-point values; casting the - // original result to RankedTensorType made scalar divisions produce a null - // result type during partial conversion. - auto divOp = rewriter.create(loc, opa, opb); - rewriter.replaceOp(op, divOp.getResult()); - return success(); -} - -LogicalResult -SelectCanonicalizer::matchAndRewrite(arith::SelectOp op, - PatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - // 0. Shortcut for scalars and bool type - auto type = dyn_cast(op.getResult().getType()); - if (!type) { - // do nothing non-tensor select - return failure(); - } - auto elementType = type.getElementType(); - if (elementType.isInteger(1)) { - // do nothing with bool type - return failure(); - } - auto tensorShape = type.getShape(); - auto mask = op.getCondition(); - if (!isa(mask.getType())) { - // do nothing for scalar mask - return failure(); - } - - // 1. Check for continuous masked loads. - // Analyze the mask operand to determine at runtime the size of the data we - // are moving. - MaskState mstate; - auto isContMask = mstate.parse(mask, loc, rewriter); - - if (isContMask.failed()) { - mstate.eraseInsertedOps(op, rewriter); - return rewriter.notifyMatchFailure( - op, "Cannot lower continuous masked selects"); - } - - // 2. Get mask position - MaskPosition maskPos = mstate.getMaskPosition(tensorShape); - LLVM_DEBUG({ - llvm::dbgs() << "[SelectAnalysis] MaskPosition detected: " - << (maskPos == MaskPosition::Head ? "Head" - : maskPos == MaskPosition::Tail ? "Tail" - : maskPos == MaskPosition::Middle ? "Middle" - : "Unknown") - << "\n"; - }); - - if (maskPos == MaskPosition::Unknown) { - mstate.eraseInsertedOps(op, rewriter); - return failure(); - } - auto trueTensor = op.getTrueValue(); - auto falseTensor = op.getFalseValue(); - - // 3. Slice and insert out the masked part - if (maskPos == MaskPosition::Head) { - // Slice out the masked part of true tensor - auto extractSliceOp = mstate.getExtractSlice(trueTensor, loc, rewriter); - - // Insert out the sliced true tensor into false tensor - auto insertSliceOp = - mstate.getInsertSlice(extractSliceOp, falseTensor, loc, rewriter); - - LLVM_DEBUG({ - llvm::dbgs() << " -> Created ExtractSlice: " - << *extractSliceOp.getOperation() << "\n" - << " -> Created InsertSlice: " - << *insertSliceOp.getOperation() << "\n"; - }); - rewriter.replaceOp(op, insertSliceOp); - return success(); - } - - // For Tail or Middle positions, we need to compute inverted dimensions - // to handle the masking logic - SmallVector invertOffsets; - SmallVector invertFalseDims; - SmallVector invertTrueDims; - OpFoldResult falseDimOp; - OpFoldResult trueDimOp; - int valDim = -1; - for (int i = 0; i < mstate.getRank(); ++i) { - const auto &offVal = mstate.offsets[i]; - const auto &dimVal = mstate.dims[i]; - auto constOffVal = getConstantIntValue(offVal); - invertOffsets.push_back(rewriter.getIndexAttr(0)); - if (constOffVal.has_value() && constOffVal.value() == 0) { - invertFalseDims.push_back(dimVal); - invertTrueDims.push_back(dimVal); - } else { - assert(valDim == -1 && - "The offset in only one dimension can be not zero."); - if (!constOffVal.has_value()) { - valDim = i; - falseDimOp = offVal; - } - - invertFalseDims.push_back(offVal); - trueDimOp = addOpFoldResult(offVal, dimVal, loc, rewriter, - rewriter.getIndexType()); - invertTrueDims.push_back(trueDimOp); - } - } - - // Slice out the invert first masked part of false tensor - auto falseExtractSliceOp = mstate.getExtractSlice( - falseTensor, loc, rewriter, invertOffsets, invertFalseDims); - // Insert out the sliced false tensor into true tensor - auto trueInsertSliceOp = - mstate.getInsertSlice(falseExtractSliceOp, trueTensor, loc, rewriter, - invertOffsets, invertFalseDims); - // Slice out the invert first masked and masked part of inserted true tensor - auto extractSliceOp = mstate.getExtractSlice(trueInsertSliceOp, loc, rewriter, - invertOffsets, invertTrueDims); - // Insert out the sliced true tensor into false tensor - auto insertSliceOp = - mstate.getInsertSlice(extractSliceOp, falseTensor, loc, rewriter, - invertOffsets, invertTrueDims); - if (valDim != -1) { - rewriter.setInsertionPointAfter(trueInsertSliceOp); - assert(isa(falseDimOp) && - "Expected to be a runtime Value for dynamic dimension check."); - Value zeroIndex = rewriter.create(loc, 0); - Value isNegative = rewriter.create( - loc, arith::CmpIPredicate::slt, cast(falseDimOp), zeroIndex); - - Value sizeIndex = - rewriter.create(loc, tensorShape[valDim]); - Value isOutOfRange = rewriter.create( - loc, arith::CmpIPredicate::sge, cast(falseDimOp), sizeIndex); - auto orOp = rewriter.create(loc, isNegative, isOutOfRange); - auto ifOp = rewriter.create(loc, TypeRange{type}, - orOp.getResult(), true, true); - - Block *thenBlock = &ifOp.getThenRegion().front(); - rewriter.setInsertionPointToStart(thenBlock); - rewriter.create(loc, ValueRange{falseTensor}); - - Block *elseBlock = &ifOp.getElseRegion().front(); - rewriter.setInsertionPointToStart(elseBlock); - falseExtractSliceOp->moveBefore(elseBlock, elseBlock->begin()); - trueInsertSliceOp->moveAfter(falseExtractSliceOp); - extractSliceOp->moveAfter(trueInsertSliceOp); - insertSliceOp->moveAfter(extractSliceOp); - - rewriter.setInsertionPointAfter(insertSliceOp); - rewriter.create(loc, ValueRange{insertSliceOp.getResult()}); - rewriter.replaceOp(op, ifOp); - } else { // static offsets - rewriter.replaceOp(op, insertSliceOp); - } - LLVM_DEBUG({ - llvm::dbgs() << " -> [invert] Created false tensor extractSlice: " - << *falseExtractSliceOp.getOperation() << "\n" - << " -> [invert] Created true tensor insertSlice: " - << *trueInsertSliceOp.getOperation() << "\n" - << " -> [invert] Created ExtractSlice: " - << *extractSliceOp.getOperation() << "\n" - << " -> [invert] Created InsertSlice: " - << *insertSliceOp.getOperation() << "\n"; - }); - return success(); -} - -/* - * Move tt.bitcast to a previous location if tt.bitcast is not directly - * applied on function arguments - */ -LogicalResult -BitcastCanonicalizer::matchAndRewrite(triton::BitcastOp bitcastOp, - PatternRewriter &rewriter) const { - Value castSrc = bitcastOp.getSrc(); - Value castRes = bitcastOp.getResult(); - Type castSrcTy = castSrc.getType(); - Type castSrcPtrTy = isa(castSrcTy) - ? cast(castSrcTy).getElementType() - : castSrcTy; - if (!isa(castSrcPtrTy)) - return failure(); - - auto origBitwidth = getPointeeBitWidth(castSrc.getType()); - auto castBitwidth = getPointeeBitWidth(castRes.getType()); - - if (origBitwidth == 1) - origBitwidth = 8; - if (castBitwidth == 1) - castBitwidth = 8; - if (origBitwidth != castBitwidth) { - bitcastOp.emitError() << "Casting pointers with unmatched bitwidth!\n"; - return failure(); - } - - Operation *beforeCastOp = castSrc.getDefiningOp(); - if (beforeCastOp == nullptr) { - return failure(); - } - - auto newRes = - TypeSwitch>(beforeCastOp) - // before: addptr - bitcast - load/store - // after: bitcast - addptr - load/store - .Case([&](triton::AddPtrOp addptrOp) { - auto newCastOp = rewriter.create( - bitcastOp.getLoc(), castRes.getType(), addptrOp.getPtr()); - return rewriter.create( - bitcastOp.getLoc(), castRes.getType(), newCastOp.getResult(), - addptrOp.getOffset()); - }) - .Case([&](triton::SplatOp splatOp) { - Type newCastSrcTy = - cast(castRes.getType()).getElementType(); - - Value splatSrc = splatOp.getSrc(); - Type splatSrcTy = splatSrc.getType(); - if (auto splatSrcTensorTy = dyn_cast(splatSrcTy)) - newCastSrcTy = - splatSrcTensorTy.cloneWith(std::nullopt, newCastSrcTy); - auto newCastOp = rewriter.create( - bitcastOp.getLoc(), newCastSrcTy, splatSrc); - return rewriter.create( - bitcastOp.getLoc(), castRes.getType(), newCastOp); - }) - // before: bitcast - bitcast - // after(fusion optimization): bitcast - .Case([&](triton::BitcastOp prevCastOp) { - return rewriter.create( - bitcastOp.getLoc(), castRes.getType(), prevCastOp.getSrc()); - }) - .Default([&](Operation *op) { - return rewriter.notifyMatchFailure(bitcastOp, - "Unknown bitcast pattern"); - }); - if (succeeded(newRes)) { - // Preserve debug location in case beforeCastOp is erased below. - insertDebugNop(beforeCastOp->getLoc(), rewriter); - rewriter.replaceOp(bitcastOp, newRes.value()); - if (beforeCastOp->use_empty()) { - rewriter.eraseOp(beforeCastOp); - } - return success(); - } - return failure(); -} - -LogicalResult -FpToFpCanonicalizer::matchAndRewrite(triton::FpToFpOp op, - PatternRewriter &rewriter) const { - auto loc = op.getLoc(); - Value input = op.getSrc(); - auto resultType = op.getResult().getType(); - - // Check if rounding mode is specified - auto roundingMode = op.getRounding(); - if (roundingMode.has_value() && - roundingMode.value() != triton::RoundingMode::RTNE) { - // Non-RTNE rounding modes (e.g., RTZ) should be handled by - // TritonToHFusion pass Return failure here so this pattern doesn't match - return failure(); - } - - // Handle RTNE (default) rounding mode with arith.truncf/extf. This can be - // either a scalar conversion or a ranked tensor conversion. - auto srcElemType = getElementTypeOrSelf(input.getType()); - auto dstElemType = getElementTypeOrSelf(resultType); - if (!isa(srcElemType) || !isa(dstElemType)) { - return op.emitError("FpToFp expects floating point types"); - } - - unsigned srcBitwidth = srcElemType.getIntOrFloatBitWidth(); - unsigned dstBitwidth = dstElemType.getIntOrFloatBitWidth(); - - // Create round_mode attribute (RINT for RTNE) - auto roundModeAttr = hfusion::RoundModeAttr::get(rewriter.getContext(), - hfusion::RoundMode::RINT); - - // A no-op conversion is valid only when the complete MLIR type is identical. - // Equal element bitwidth alone is insufficient: FP8 formats such as - // f8E4M3FN and f8E5M2 have different numerical semantics. - if (input.getType() == resultType) { - rewriter.replaceOp(op, input); - return success(); - } - - if (srcBitwidth == dstBitwidth) { - if (srcElemType == dstElemType) { - return op.emitError( - "fp_to_fp with identical element types has incompatible " - "container or layout types"); - } - - // arith.extf/truncf require a strict bitwidth change. Materialize an f32 - // intermediate so the conversion remains a numerical cast in TTAdapter - // IR and can follow the normal Bisheng lowering path. - Type f32Type; - if (auto tensorType = dyn_cast(input.getType())) { - f32Type = - RankedTensorType::get(tensorType.getShape(), rewriter.getF32Type(), - tensorType.getEncoding()); - } else if (isa(input.getType())) { - f32Type = rewriter.getF32Type(); - } else { - return op.emitError("FpToFp expects a scalar or ranked tensor type"); - } - - auto extOp = rewriter.create(loc, f32Type, input); - extOp->setAttr("round_mode", roundModeAttr); - auto truncOp = - rewriter.create(loc, resultType, extOp.getResult()); - truncOp->setAttr("round_mode", roundModeAttr); - rewriter.replaceOp(op, truncOp.getResult()); - } else if (srcBitwidth > dstBitwidth) { - // Downcast: use arith.truncf with round_mode=rint - auto truncOp = rewriter.create(loc, resultType, input); - truncOp->setAttr("round_mode", roundModeAttr); - rewriter.replaceOp(op, truncOp.getResult()); - } else { - // Upcast: use arith.extf with round_mode=rint - auto extOp = rewriter.create(loc, resultType, input); - extOp->setAttr("round_mode", roundModeAttr); - rewriter.replaceOp(op, extOp.getResult()); - } - - return success(); -} - -void rewriteUserWithNewOrder( - mlir::OpOperand *use, PatternRewriter &rewriter, - llvm::SmallVector &blkShapeI64, // 8: container size - mlir::Location &loc, llvm::ArrayRef &order, size_t &orderSize) { - Operation *user = use->getOwner(); - rewriter.setInsertionPointAfter(user); - if (auto loadOp = dyn_cast(user)) { - auto loadResTy = loadOp.getResult().getType(); - auto loadResShapedTy = cast(loadResTy); - auto newLoadTy = loadResShapedTy.cloneWith( - blkShapeI64, loadResShapedTy.getElementType()); - auto newLoadOp = rewriter.create( - loc, newLoadTy, loadOp->getOperands(), loadOp->getAttrs()); - newLoadOp->setAttr(ConverterUtils::GeneratedByMakeTensorPtrTAG, - UnitAttr::get(rewriter.getContext())); - rewriter.replaceOp(loadOp, newLoadOp); - // load contiguous data then permute. thus the permute order is as - // follows. - SmallVector permuteOrder; // 8: container size - for (auto [i, v] : llvm::enumerate(order)) { - permuteOrder.push_back(orderSize - 1 - order[i]); - } - auto permuteOp = rewriter.create( - loc, newLoadOp.getResult(), - DenseI32ArrayAttr::get(loadOp.getContext(), permuteOrder)); - newLoadOp.getResult().replaceAllUsesExcept(permuteOp.getResult(), - permuteOp); - } else if (auto storeOp = dyn_cast(user)) { - // permute to contiguous then store. thus the permute order is as follows. - SmallVector permuteOrder; // 8: container size - for (auto [i, v] : llvm::enumerate(order)) { - permuteOrder.push_back(order[orderSize - 1 - i]); - } - auto permuteOp = rewriter.create( - loc, storeOp.getValue(), - DenseI32ArrayAttr::get(storeOp.getContext(), permuteOrder)); - storeOp.getValue().replaceAllUsesExcept(permuteOp.getResult(), permuteOp); - auto newStoreOp = rewriter.create( - loc, storeOp.getPtr(), storeOp.getValue(), storeOp.getMask(), - storeOp.getBoundaryCheck(), storeOp.getCache(), storeOp.getEvict()); - rewriter.replaceOp(storeOp, newStoreOp); - } else if (auto advanceOp = dyn_cast(user)) { - auto advanceResPtrTy = - cast(advanceOp.getResult().getType()); - auto advanceResShapedTy = - cast(advanceResPtrTy.getPointeeType()); - auto newAdvanceResShapedTy = advanceResShapedTy.cloneWith( - blkShapeI64, advanceResShapedTy.getElementType()); - auto newAdvanceResPtrTy = triton::PointerType::get( - newAdvanceResShapedTy, advanceResPtrTy.getAddressSpace()); - auto advanceOffsets = advanceOp.getOffsets(); - llvm::SmallVector newAdvanceOffsets; // 8: container size - for (int i = orderSize - 1; i >= 0; i--) { - newAdvanceOffsets.push_back(advanceOffsets[order[i]]); - } - SmallVector resUses; - for (auto &use : advanceOp->getUses()) - resUses.push_back(&use); - auto newAdvanceOp = rewriter.create( - loc, newAdvanceResPtrTy, advanceOp.getPtr(), newAdvanceOffsets); - rewriter.replaceOp(advanceOp, newAdvanceOp); - for (auto resUse : resUses) - rewriteUserWithNewOrder(resUse, rewriter, blkShapeI64, loc, order, - orderSize); - } else if (auto loopOp = dyn_cast(user)) { - auto initArg = use->get(); - auto iterArg = loopOp.getTiedLoopRegionIterArg(use); - auto resultValue = loopOp.getTiedLoopResult(use); - iterArg.setType(initArg.getType()); - resultValue.setType(initArg.getType()); - for (auto &argUse : iterArg.getUses()) - rewriteUserWithNewOrder(&argUse, rewriter, blkShapeI64, loc, order, - orderSize); - for (auto &resUse : resultValue.getUses()) - rewriteUserWithNewOrder(&resUse, rewriter, blkShapeI64, loc, order, - orderSize); - } else if (isa(user)) { - return; - } else { - llvm_unreachable( - "[MakeTensorPtrCanonicalizer] tt.make_tensor_ptr's result is " - "not used by load/store/advance op"); - } -} - -void markLoadUsers(mlir::OpOperand *use, PatternRewriter &rewriter) { - Operation *user = use->getOwner(); - if (auto loadOp = dyn_cast(user)) { - loadOp->setAttr(ConverterUtils::GeneratedByMakeTensorPtrTAG, - UnitAttr::get(rewriter.getContext())); - } else if (auto storeOp = dyn_cast(user)) { - return; - } else if (auto advanceOp = dyn_cast(user)) { - SmallVector resUses; - for (auto &use : advanceOp->getUses()) - resUses.push_back(&use); - for (auto resUse : resUses) - markLoadUsers(resUse, rewriter); - } else if (auto loopOp = dyn_cast(user)) { - auto initArg = use->get(); - auto iterArg = loopOp.getTiedLoopRegionIterArg(use); - auto resultValue = loopOp.getTiedLoopResult(use); - iterArg.setType(initArg.getType()); - resultValue.setType(initArg.getType()); - for (auto &argUse : iterArg.getUses()) - markLoadUsers(&argUse, rewriter); - for (auto &resUse : resultValue.getUses()) - markLoadUsers(&resUse, rewriter); - } else if (isa(user)) { - return; - } else { - llvm_unreachable( - "[MakeTensorPtrCanonicalizer] tt.make_tensor_ptr's result is " - "not used by load/store/advance op"); - } -} - -LogicalResult -MakeTensorPtrCanonicalizer::matchAndRewrite(triton::MakeTensorPtrOp op, - PatternRewriter &rewriter) const { - auto order = op.getOrder(); - auto orderSize = order.size(); - if (orderSize == 1) { - return rewriter.notifyMatchFailure( - op, "make_tensor_ptr's order has single value."); - } - - bool isPermuted = false; - for (auto [first, second] : llvm::zip(order.slice(0, orderSize - 1), - order.slice(1, orderSize - 1))) { - if (first != second + 1) { - isPermuted = true; - break; - } - } - - auto loc = op.getLoc(); - auto base = op.getBase(); - auto shape = op.getShape(); - auto strides = op.getStrides(); - auto offsets = op.getOffsets(); - auto result = op.getResult(); - SmallVector opUses; - - for (auto &use : result.getUses()) - opUses.push_back(&use); - for (auto use : opUses) - markLoadUsers(use, rewriter); - - if (!isPermuted) { - return rewriter.notifyMatchFailure( - op, "make_tensor_ptr's order is contiguous."); - } - - llvm::SmallVector blkShapeI32; - llvm::SmallVector blkShapeI64; - auto resPtrType = cast(result.getType()); - if (auto resShapedTy = dyn_cast(resPtrType.getPointeeType())) { - auto resBlkShape = resShapedTy.getShape(); - for (auto [i, v] : llvm::enumerate(resBlkShape)) { - auto reverseI = orderSize - 1 - i; - blkShapeI32.push_back(resBlkShape[order[reverseI]]); - blkShapeI64.push_back(resBlkShape[order[reverseI]]); - } - } - - llvm::SmallVector newShape; - llvm::SmallVector newStrides; - llvm::SmallVector newOffsets; - for (int i = orderSize - 1; i >= 0; i--) { - newShape.push_back(shape[order[i]]); - newStrides.push_back(strides[order[i]]); - newOffsets.push_back(offsets[order[i]]); - } - - llvm::SmallVector contiguousOrder; - for (int i = orderSize - 1; i >= 0; i--) - contiguousOrder.push_back(i); - - rewriter.setInsertionPoint(op); - auto newMakeTensorPtrOp = rewriter.create( - loc, base, ValueRange(newShape), ValueRange(newStrides), - ValueRange(newOffsets), blkShapeI32, contiguousOrder); - rewriter.replaceOp(op, newMakeTensorPtrOp); - for (auto use : opUses) - rewriteUserWithNewOrder(use, rewriter, blkShapeI64, loc, order, orderSize); - return success(); -} - -LogicalResult -ReduceSingleCanonicalizer::matchAndRewrite(triton::ReduceOp reduceOp, - PatternRewriter &rewriter) const { - // This canonicalization only handles value reductions and value/index - // reductions. Multi-input reductions, such as Welford's - // (mean, count, m2) reduction, must fall through to ReduceConverter's - // extended lowering instead of terminating the compiler here. - if (reduceOp.getSrcs().size() > 2) - return rewriter.notifyMatchFailure( - reduceOp, "only canonicalizes value and value/index reductions"); - auto src = reduceOp.getSrcs()[0]; - auto srcType = cast(src.getType()); - auto srcShape = srcType.getShape(); - if (llvm::any_of(srcShape, [](auto s) { return s != 1; })) - return rewriter.notifyMatchFailure( - reduceOp, "reduce's srcs are not all with single element"); - auto loc = reduceOp->getLoc(); - - // Handle Reduce Value - auto res = reduceOp.getResult()[0]; - Value extracted; - if (srcType.getRank() == 1) { - auto zero = - rewriter.create(loc, rewriter.getIndexAttr(0)); - extracted = rewriter.create(loc, src, zero.getResult()) - .getResult(); - } else { - auto resShape = cast(res.getType()).getShape(); - auto collapseReassociationIndicesOptional = - getReassociationIndicesForCollapse(srcShape, resShape); - if (!collapseReassociationIndicesOptional.has_value()) { - return rewriter.notifyMatchFailure( - reduceOp, "Failure with getReassociationIndicesForCollapse call"); - } - auto collapseReassociationIndices = - collapseReassociationIndicesOptional.value(); - extracted = rewriter - .create( - loc, src, collapseReassociationIndices) - .getResult(); - } - res.replaceAllUsesWith(extracted); - - // Handle Reduce Index - if (reduceOp.getSrcs().size() == 1) - return success(); - - auto resIdx = reduceOp.getResult()[1]; - auto zeroI32 = - rewriter.create(loc, rewriter.getI32IntegerAttr(0)); - if (srcType.getRank() == 1) { - resIdx.replaceAllUsesWith(zeroI32); - } else { - auto resIdxShape = cast(resIdx.getType()).getShape(); - auto initTensor = rewriter.create(loc, resIdxShape, - rewriter.getI32Type()); - auto fillOp = rewriter.create(loc, ValueRange{zeroI32}, - ValueRange{initTensor}); - resIdx.replaceAllUsesWith(fillOp.getResult(0)); - } - - return success(); -} - -LogicalResult DenseConstantConverter::matchAndRewrite( - arith::ConstantOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto denseAttr = cast(op.getValue()); - auto loc = op.getLoc(); - auto constSplatOp = arith::ConstantOp::materialize( - rewriter, denseAttr.getSplatValue(), - denseAttr.getElementType(), loc); - auto emptyOp = rewriter.create( - loc, cast(op.getResult().getType()).getShape(), - denseAttr.getElementType()); - - rewriter.replaceOpWithNewOp(op, ValueRange{constSplatOp}, - ValueRange{emptyOp}); - - return success(); -} - -LogicalResult -MakeRangeConverter::matchAndRewrite(triton::MakeRangeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - insertDebugNop(loc, rewriter); - auto type = cast(op.getResult().getType()); - auto shape = type.getShape(); - auto elementType = type.getElementType(); - auto context = op.getContext(); - - assert(type.getShape().size() == 1 && - isa(type.getElementType()) && - type.getElementType().getIntOrFloatBitWidth() == 32 && - "make range can only return 1D int32 tensor"); - - SmallVector indexingMaps{AffineMap::get( - /* dimCount */ 1, /* symbolCount */ 0, - {mlir::getAffineDimExpr(0, context)}, context)}; - - auto init = rewriter.create(loc, shape, elementType); - - auto nestedBody = [&](OpBuilder &nestedBuilder, Location nestedLoc, - ValueRange blockArgs) { - Value index = nestedBuilder.create(loc, 0); - Value res = - nestedBuilder.create(loc, elementType, index); - nestedBuilder.create(loc, res); - }; - - auto linalgOp = rewriter.create( - loc, op->getResultTypes(), /* operands */ ValueRange{}, ValueRange{init}, - indexingMaps, ConverterUtils::getNParallelLoopsAttrs(1), nestedBody); - - linalgOp->setAttr("tt.from_make_range", mlir::UnitAttr::get(context)); - linalgOp->setAttr("tt.make_range_offset", - mlir::IntegerAttr::get(mlir::IndexType::get(context), 0)); - linalgOp->setAttr( - "tt.make_range_size", - mlir::IntegerAttr::get(mlir::IndexType::get(context), shape[0])); - - int32_t startVal = op.getStartAttr().getInt(); - if (startVal == 0) { - rewriter.replaceOp(op, linalgOp->getResults()); - return success(); - } - - // Apply start offset - Value startScaler = rewriter.create( - loc, rewriter.getI32IntegerAttr(static_cast(startVal))); - auto startInit = rewriter.create(loc, shape, elementType); - Value startTensor = rewriter - .create(loc, ValueRange{startScaler}, - ValueRange{startInit}) - .getResult(0); - auto addOp = - rewriter.create(loc, linalgOp->getResult(0), startTensor); - rewriter.replaceOp(op, addOp); - return success(); -} - -LogicalResult -SplatConverter::matchAndRewrite(triton::SplatOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - // These NOPs are transient debug anchors: DeduplicateDebugNopsPass collapses - // them to one per source line before codegen. - insertDebugNopForAllLines(loc, rewriter); - auto shape = op.getType().getShape(); - auto init = rewriter.create(loc, shape, - op.getType().getElementType()); - if (llvm::all_of(shape, [](int64_t dim) { return dim == 1; })) { - SmallVector idx(shape.size(), rewriter.create( - loc, rewriter.getIndexAttr(0))); - rewriter.replaceOpWithNewOp(op, adaptor.getSrc(), init, - idx); - } else { - rewriter.replaceOpWithNewOp( - op, ValueRange{adaptor.getSrc()}, ValueRange{init}); - } - return success(); -} - -LogicalResult -UnsplatConverter::matchAndRewrite(triton::UnsplatOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto src = adaptor.getSrc(); - auto srcType = cast(src.getType()); - auto shape = srcType.getShape(); - - // Create index constants for all dimensions (all zeros since we're - // extracting the single element) - SmallVector indices; - for (int64_t dim : shape) { - indices.push_back( - rewriter.create(loc, rewriter.getIndexAttr(0))); - } - - // Extract the scalar element from the tensor - auto elementType = srcType.getElementType(); - auto extractOp = - rewriter.create(loc, elementType, src, indices); - rewriter.replaceOp(op, extractOp.getResult()); - return success(); -} - -LogicalResult -ReshapeConverter::matchAndRewrite(triton::ReshapeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - insertDebugNop(loc, rewriter); - auto src = op.getSrc(); - auto dst = op.getResult(); - Value shape = rewriter.create( - loc, - rewriter.getI64TensorAttr(cast(dst.getType()).getShape())); - auto reshapeOp = - rewriter.create(loc, dst.getType(), src, shape); - rewriter.replaceOp(op, reshapeOp.getResult()); - return success(); -} - -LogicalResult ExpandDimsConverter::matchAndRewrite( - triton::ExpandDimsOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - insertDebugNop(loc, rewriter); - auto src = op.getSrc(); - auto resShape = cast(op.getResult().getType()).getShape(); - auto axis = op.getAxis(); - - SmallVector reassociation; - - auto src_last_dim = resShape.size() - 2; - auto map_func = [&](unsigned i) -> ReassociationIndices { - if (i < axis) { - return i == src_last_dim ? ReassociationIndices{i, i + 1} - : ReassociationIndices{i}; - } - return i == axis ? ReassociationIndices{i, i + 1} - : ReassociationIndices{i + 1}; - }; - - reassociation = llvm::to_vector( - llvm::map_range(llvm::seq(0, src_last_dim + 1), map_func)); - - auto expandShapeOp = rewriter.create( - op.getLoc(), op.getResult().getType(), src, reassociation); - rewriter.replaceOp(op, expandShapeOp.getResult()); - return success(); -} - -LogicalResult -ClampFConverter::matchAndRewrite(triton::ClampFOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto input = adaptor.getX(); - auto min_para = adaptor.getMin(); - auto max_para = adaptor.getMax(); - auto propagateNan_para = adaptor.getPropagateNan(); - - if (auto input_type = dyn_cast(input.getType())) { - if (isa(min_para.getType())) { - auto minEmptyTensor = rewriter.create( - loc, input_type.getShape(), input_type.getElementType()); - min_para = rewriter - .create(loc, ValueRange{min_para}, - ValueRange{minEmptyTensor}) - .result(); - } - if (isa(max_para.getType())) { - auto maxEmptyTensor = rewriter.create( - loc, input_type.getShape(), input_type.getElementType()); - max_para = rewriter - .create(loc, ValueRange{max_para}, - ValueRange{maxEmptyTensor}) - .result(); - } - } - - if (propagateNan_para == PropagateNan::NONE) { - auto minOp = rewriter.create(loc, input, max_para); - auto maxOp = rewriter.create(loc, min_para, minOp); - rewriter.replaceOp(op, ValueRange{maxOp}); - } else if (propagateNan_para == PropagateNan::ALL) { - auto minOp = rewriter.create(loc, input, max_para); - auto maxOp = rewriter.create(loc, min_para, minOp); - rewriter.replaceOp(op, ValueRange{maxOp}); - } else { - return failure(); - } - - return success(); -} - -// Here convert tt.broadcast to linalg.broadcast -// -// before -// %out = tt.broadcast %in : tensor<1x4x8xf32> -> tensor<128x4x8xf32> -// -// after -// %collpased = tensor.collapse_shape %in [[0, 1], [2]] : -// tensor<1x4x8xf32> into tensor<4x8xf32> -// %out = linalg.broadcast ins(%collpased : tensor<4x8xf32>) -// outs(%empty : tensor<128x4x8xf32>) dimensions = [0] -LogicalResult -BroadcastConverter::matchAndRewrite(triton::BroadcastOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - assert(op->getNumResults() == 1 && "BroadcastOp assumes single result"); - - RankedTensorType sourceType = - cast(adaptor.getSrc().getType()); - RankedTensorType resultType = cast(op.getType()); - auto elementType = resultType.getElementType(); - auto loc = op.getLoc(); - insertDebugNopForAllLines(loc, rewriter); - auto initEmpty = - rewriter.create(loc, resultType.getShape(), elementType); - - SmallVector broadcastDims = - ConverterUtils::getBroadcastDims(sourceType, resultType); - SmallVector unbroadcastDims = - ConverterUtils::getUnbroadcastDims(sourceType, resultType); - - SmallVector collapseReassociationIndices; - auto collapseReassociationIndicesOptional = - getReassociationIndicesForCollapse(sourceType.getShape(), - unbroadcastDims); - if (!collapseReassociationIndicesOptional.has_value()) { - return rewriter.notifyMatchFailure( - op, "Failure with getReassociationIndicesForCollapse call"); - } - collapseReassociationIndices = collapseReassociationIndicesOptional.value(); - - RankedTensorType collapseResultType = - RankedTensorType::get(unbroadcastDims, sourceType.getElementType()); - - auto collpasedOp = rewriter.create( - loc, collapseResultType, adaptor.getSrc(), collapseReassociationIndices); - - auto broadcastOp = rewriter.create( - loc, collpasedOp, initEmpty, - rewriter.getDenseI64ArrayAttr(broadcastDims)); - - rewriter.replaceOp(op, broadcastOp.getResults()); - return success(); -} - -// Reduce Converter -bool ReduceConverter::isReductionOpSupported(Operation *redOp) const { - return isa(redOp); -} - -bool ReduceConverter::isMultiReductionOpSupported(Operation *redOp) { - return isa( - redOp); -} - -Value ReduceConverter::cloneReduceOps(OpBuilder &builder, Value in, Value out, - Value opIns, Value opOuts, - triton::ReduceOp op) const { - auto ® = op->getRegion(0); - assert(reg.getBlocks().size() == 1); - auto &body = reg.getBlocks().front(); - auto numArguments = 2; - assert(body.getNumArguments() == numArguments); - - Value ttIn = body.getArgument(0); - Value ttOut = body.getArgument(1); - - IRMapping mapping; - mapping.map(ttIn, in); - mapping.map(ttOut, out); - - for (auto &op : body.without_terminator()) { - builder.clone(op, mapping); - } - auto yield = cast(body.getTerminator()); - return mapping.lookup(yield->getOperand(0)); -} - -void ReduceConverter::checkIsNotCallOp( - const llvm::SmallVector &reductionOps) const { - llvm::for_each(reductionOps, [](Operation *op) { - assert(!isa(op) && "tt.call ops expected to be inlined in " - "tt.reduce body in ttir building stage"); - }); -} - -bool ReduceConverter::isSCFOpReduce( - const llvm::SmallVector &reductionOps) const { - return (reductionOps.size() == 1 && - reductionOps.front()->getDialect()->getNamespace() == - scf::SCFDialect::getDialectNamespace()); -} - -bool ReduceConverter::isMultiOpReduce( - const llvm::SmallVector &reductionOps) const { - this->checkIsNotCallOp(reductionOps); - - return (reductionOps.size() > 1) || - (reductionOps.size() == 1 && - this->isMultiReductionOpSupported(reductionOps.front())) || - this->isSCFOpReduce(reductionOps); -} - -Value ReduceConverter::computeReduceResultWithCompileFlag( - OpBuilder &opBuilder, Location loc, Value lhs, Value rhs, Value source, - Value initTensor, triton::ReduceOp op, bool compileOn91095Flag) const { - // Original operation list (including all operations) - auto originalReductionOps = this->getReductionOps(op); - auto realReductionOps = this->getRealReductionOps(op); - - // If the size of the original operation list is greater than 1, - // there are additional operations such as type conversion, and these - // operations must be cloned. - bool needClone = compileOn91095Flag || originalReductionOps.size() > 1; - if (needClone) { - return this->cloneReduceOps(opBuilder, lhs, rhs, source, initTensor, op); - } else { - assert(realReductionOps.size() == 1); - auto rop = realReductionOps.front(); - return this->getReductionElement(lhs, rhs, loc, rop, opBuilder, false); - } -} - -LogicalResult -ReduceConverter::convertToTargetOp(triton::ReduceOp op, - typename triton::ReduceOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto source = adaptor.getOperands().front(); - auto sourceType = cast(source.getType()); - auto elemType = sourceType.getElementType(); - auto resType = op.getResult().front().getType(); - auto loc = op.getLoc(); - - // Actual operation list (filtering type conversion operations, leaving only - // actual reduce operations) - auto realReductionOps = this->getRealReductionOps(op); - - bool multiOpReduce = this->isMultiOpReduce(realReductionOps); - // Reduction of arbitrary operations isn't supported because using the first - // element across the reduction dimension requires us to iterate over a - // subview that skips over each first element. - if (!multiOpReduce && - !this->isReductionOpSupported(realReductionOps.front())) { - if (compileOn91095Flag) { - llvm_unreachable("All reduction cases expected to be covered"); - } - return rewriter.notifyMatchFailure( - op, "Only support lowering reduction with single op and limited types " - "of reduction"); - } - - auto rop = realReductionOps.front(); - auto ropLoc = rop->getLoc(); - auto axis = op.getAxis(); - auto isVectorReduce = sourceType.getRank() == 1; - - auto constantType = elemType; - - auto accBaseConstOp = - multiOpReduce - ? this->getMultiOpReductionBaseConstOp(rewriter, op, ropLoc, - constantType) - : this->getReductionBaseConstOp(rewriter, rop, constantType); - - Value initTensor; - if (isVectorReduce) { - auto holder = rewriter.create( - loc, RankedTensorType::get({}, constantType), ValueRange{}); - initTensor = rewriter - .create(loc, accBaseConstOp.getResult(), - holder.getResult()) - .getResult(0); - } else { - Value init = rewriter.create( - loc, cast(resType).getShape(), constantType); - initTensor = - rewriter.create(loc, accBaseConstOp.getResult(), init) - .getResult(0); - } - - Value finalResult = - rewriter - .create( - loc, ValueRange{source}, ValueRange{initTensor}, - SmallVector{axis}, - [&](OpBuilder &opBuilder, Location loc, ValueRange inputs) { - assert(inputs.size() == 2); - Value result = this->computeReduceResultWithCompileFlag( - opBuilder, loc, inputs[0], inputs[1], source, initTensor, - op, compileOn91095Flag); - opBuilder.create(loc, result); - }) - .getResult(0); - - if (sourceType.getRank() == 1) { - finalResult = - rewriter.create(loc, constantType, finalResult); - } - insertDebugNop(loc, rewriter); - rewriter.replaceOp(op, finalResult); - return success(); -} - -LogicalResult ReduceConverter::convertToTargetOpExtended( - triton::ReduceOp op, typename triton::ReduceOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - auto operands = adaptor.getOperands(); - - // BiShengIR's VReduce lowering only supports a single value input (or a - // value/index pair). A multi-input `tt.reduce`, such as Welford's - // (mean, count, m2) reduction, cannot be represented by that VReduceOp. - // Keep the current reduction body, but lower the static rank-1 scalar case - // to an explicit scalar loop so it never reaches the variadic VReduce path. - if (operands.size() > 2) { - auto inputType = dyn_cast(operands.front().getType()); - if (!inputType || inputType.getRank() != 1 || adaptor.getAxis() != 0 || - ShapedType::isDynamic(inputType.getShape()[0]) || - inputType.getShape()[0] < 1) { - return rewriter.notifyMatchFailure( - op, "multi-input reduce fallback requires static rank-1 axis-0 " - "inputs"); - } - if (op.getResult().size() != operands.size()) { - return rewriter.notifyMatchFailure( - op, "multi-input reduce results do not match input count"); - } - - for (auto [i, operand] : llvm::enumerate(operands)) { - auto operandType = dyn_cast(operand.getType()); - if (!operandType || operandType.getShape() != inputType.getShape() || - op.getResult()[i].getType() != operandType.getElementType()) { - return rewriter.notifyMatchFailure( - op, "multi-input reduce fallback requires matching scalar " - "results"); - } - } - - auto reduceBlock = op.getBody(); - if (reduceBlock->getNumArguments() != 2 * operands.size() || - reduceBlock->getTerminator()->getNumOperands() != operands.size()) { - return rewriter.notifyMatchFailure( - op, "unexpected multi-input reduce combine region"); - } - - Value zero = rewriter.create(loc, 0); - Value one = rewriter.create(loc, 1); - Value upper = - rewriter.create(loc, inputType.getShape()[0]); - SmallVector initialValues; - initialValues.reserve(operands.size()); - for (Value operand : operands) - initialValues.push_back( - rewriter.create(loc, operand, zero)); - - auto loop = rewriter.create( - loc, one, upper, one, initialValues, - [&](OpBuilder &builder, Location loopLoc, Value inductionVar, - ValueRange iterArgs) { - IRMapping mapping; - for (auto [i, operand] : llvm::enumerate(operands)) { - Value current = builder.create(loopLoc, operand, - inductionVar); - mapping.map(reduceBlock->getArgument(i), current); - mapping.map(reduceBlock->getArgument(i + operands.size()), - iterArgs[i]); - } - for (Operation &innerOp : reduceBlock->without_terminator()) - builder.clone(innerOp, mapping); - - SmallVector yielded; - yielded.reserve(operands.size()); - for (Value value : reduceBlock->getTerminator()->getOperands()) - yielded.push_back(mapping.lookup(value)); - builder.create(loopLoc, yielded); - }); - rewriter.replaceOp(op, loop.getResults()); - return success(); - } - - auto elemTypes = op.getElementTypes(); - - auto valueResultType = dyn_cast(op.getType(0)); - const auto isScalarReduce = valueResultType == nullptr; - - SmallVector outputs; - for (auto i = 0; i < op.getResult().size() && i < elemTypes.size(); i++) { - auto result = dyn_cast(op.getType(i)); - SmallVector resultShape{ - isScalarReduce ? SmallVector{} - : SmallVector(result.getShape())}; - outputs.push_back( - rewriter.create(loc, resultShape, elemTypes[i])); - } - - auto linalgOp = rewriter.create( - loc, adaptor.getOperands(), outputs, - SmallVector{adaptor.getAxis()}, - [&](OpBuilder &b, Location loc, ValueRange inputs) { - auto tritonReduceBlock = op.getBody(); - IRMapping mapping; - mapping.map(tritonReduceBlock->getArguments(), inputs); - - for (auto &op : tritonReduceBlock->without_terminator()) { - b.clone(op, mapping); - } - - auto tritonYield = tritonReduceBlock->getTerminator(); - auto results = - llvm::map_to_vector(tritonYield->getOperands(), - [&](Value val) { return mapping.lookup(val); }); - b.create(loc, results); - }); - - auto params = getReduceWithIndexParams(op); - if (failed(params)) { - return rewriter.notifyMatchFailure(op, "meaningless reduce operation"); - } else if (params->withIndexType != ReduceWithIndexType::None) { - addReduceWithIndexAttr(*params, rewriter, linalgOp); - } - - insertDebugNop(loc, rewriter); - if (isScalarReduce) { - SmallVector reduceResults; - for (auto i = 0; i < linalgOp.getResults().size() && i < elemTypes.size(); - i++) { - reduceResults.push_back(rewriter.create( - loc, elemTypes[i], linalgOp.getResults()[i], ValueRange{})); - } - rewriter.replaceOp(op, reduceResults); - } else { - rewriter.replaceOp(op, linalgOp); - } - return success(); -} - -bool ScanConverter::isReductionOpSupported(Operation *reductionOp) const { - if (isa( - reductionOp)) { - return true; - } - if (compileOn91095Flag && - isa(reductionOp)) { - return true; - } - return false; -} - -LogicalResult -ScanConverter::convertToTargetOp(triton::ScanOp op, - typename triton::ScanOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto reductionOps = this->getRealReductionOps(op); - if (reductionOps.empty()) { - return rewriter.notifyMatchFailure(op, - "No reduction op found in scan body"); - } - - llvm::SmallString<64> funcName; - auto rop = reductionOps.front(); - if (this->isReductionOpSupported(reductionOps.front())) { - bool propagateNan = true; - bool isMinMax = false; - if (isa(rop)) { - funcName = "triton_cumsum"; - } else if (isa(rop)) { - funcName = "triton_cumprod"; - } else if (isa(rop)) { - funcName = "triton_cummax"; - propagateNan = isa(rop); - isMinMax = true; - } else if (isa(rop)) { - funcName = "triton_cummin"; - propagateNan = isa(rop); - isMinMax = true; - } - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - auto loc = op.getLoc(); - auto src = adaptor.getOperands().front(); - auto resTy = op.getResult().front().getType(); - // cummax/cummin take a trailing propagateNan flag; cumsum/cumprod do not. - SmallVector argTypes{src.getType(), rewriter.getI32Type(), - rewriter.getI1Type()}; - if (isMinMax) { - argTypes.push_back(rewriter.getI1Type()); - } - auto libFnType = rewriter.getFunctionType(argTypes, {resTy}); - // Unique the name before creating the op: building a SymbolTable over a - // module that already holds a symbol of this name -- a kernel named - // "triton_cumsum", say -- trips SymbolTable's "uniquely named symbol - // operations" assertion before any renaming can happen. - auto uniqueName = generateUniqueFuncName(moduleOp, funcName); - auto funcOp = - rewriter.create(loc, uniqueName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - - rewriter.setInsertionPoint(op); - auto scanAxis = op.getAxis(); - auto scanReverse = op.getReverse(); - Value axis = rewriter.create(loc, scanAxis, 32); - Value reverseVal = - rewriter.create(loc, scanReverse, 1); - SmallVector callOperands{src, axis, reverseVal}; - if (isMinMax) { - callOperands.push_back( - rewriter.create(loc, propagateNan, 1)); - } - auto callOp = rewriter.create( - loc, funcOp.getSymNameAttr(), TypeRange({resTy}), callOperands); - - rewriter.replaceOp(op, callOp); - - return success(); - } else { - // This branch is the associative_scan op. - bool reverse = op.getReverse(); - - auto loc = op.getLoc(); - - Value scanInput = op.getOperand(0); - - auto srcType = mlir::dyn_cast(scanInput.getType()); - if (!srcType) { - return rewriter.notifyMatchFailure( - op, "Expected RankedTensorType input for associative_scan"); - } - - auto elementType = srcType.getElementType(); - auto shape = srcType.getShape(); - int rank = shape.size(); - int axis = op.getAxis(); - - if (axis < 0 || axis >= rank) { - return rewriter.notifyMatchFailure(op, "Invalid scan axis: " + - std::to_string(axis)); - } - - if (op->getNumRegions() < 1 || op->getRegion(0).empty()) { - return rewriter.notifyMatchFailure(op, "Missing combine region"); - } - - OpBuilder::InsertionGuard guard(rewriter); - - auto memrefType = MemRefType::get(shape, elementType); - Value inputMemRef = - rewriter.create(loc, memrefType, scanInput); - - // Wrap scan logic in a scope with UB address space for the output buffer. - auto tensorResultType = RankedTensorType::get(shape, elementType); - auto scopeOp = - rewriter.create(loc, TypeRange{tensorResultType}); - scopeOp.getBodyRegion().emplaceBlock(); - rewriter.setInsertionPointToEnd(&scopeOp.getBodyRegion().front()); - - auto ubMemRefType = MemRefType::get( - shape, elementType, nullptr, - rewriter.getAttr(hivm::AddressSpace::UB)); - Value outputMemRef = rewriter.create(loc, ubMemRefType); - - auto processDimension = [&](ArrayRef baseIdxsArray) { - auto startInd = rewriter.create(op.getLoc(), 0); - if (reverse) { - startInd = rewriter.create(op.getLoc(), - shape[axis] - 1); - } - llvm::SmallVector baseIdxs(baseIdxsArray.begin(), - baseIdxsArray.end()); - llvm::SmallVector firstIdx = baseIdxs; - if (axis <= firstIdx.size()) { - firstIdx.insert(firstIdx.begin() + axis, startInd); - } else { - firstIdx.push_back(startInd); - } - - Value firstVal = - rewriter.create(loc, inputMemRef, firstIdx); - rewriter.create(loc, firstVal, outputMemRef, firstIdx); - - Value axisSize = - rewriter.create(loc, inputMemRef, axis).getResult(); - Value one = rewriter.create(loc, 1); - - Value cmp = rewriter.create(loc, arith::CmpIPredicate::sgt, - axisSize, one); - auto ifOp = rewriter.create(loc, cmp, false); - - // Create a loop only when the axis size is greater than 1. - rewriter.setInsertionPointToStart(ifOp.thenBlock()); - - auto forOp = rewriter.create(loc, one, axisSize, one); - rewriter.setInsertionPointToStart(forOp.getBody()); - - Value k = forOp.getInductionVar(); - if (reverse) { - llvm::SmallVector fixInd; - fixInd.push_back( - rewriter - .create(op.getLoc(), shape[axis] - 1) - .getResult()); - fixInd.push_back(k); - auto fixIndVal = rewriter.create(op.getLoc(), fixInd); - k = fixIndVal.getResult(); - } - llvm::SmallVector currIdx = baseIdxs; - if (axis <= currIdx.size()) { - currIdx.insert(currIdx.begin() + axis, k); - } else { - currIdx.push_back(k); - } - - Value km1 = rewriter.create(loc, k, one); - if (reverse) { - km1 = rewriter.create(loc, k, one); - } - llvm::SmallVector prevIdx = baseIdxs; - if (axis <= prevIdx.size()) { - prevIdx.insert(prevIdx.begin() + axis, km1); - } else { - prevIdx.push_back(km1); - } - - Value currentVal = - rewriter.create(loc, inputMemRef, currIdx); - Value prevResult = - rewriter.create(loc, outputMemRef, prevIdx); - - Region &combineRegion = op->getRegion(0); - Block &combineBlock = combineRegion.front(); - IRMapping mapping; - mapping.map(combineBlock.getArgument(0), prevResult); - mapping.map(combineBlock.getArgument(1), currentVal); - - for (Operation &innerOp : combineBlock.without_terminator()) { - rewriter.clone(innerOp, mapping); - } - - Operation *yieldOp = combineBlock.getTerminator(); - Value resultVal = mapping.lookup(yieldOp->getOperand(0)); - - rewriter.create(loc, resultVal, outputMemRef, currIdx); - - rewriter.setInsertionPointAfter(ifOp); - }; - - // Constructing loops for non-scanning dimensions - llvm::SmallVector nonScanDims; - for (int i = 0; i < rank; ++i) { - if (i != axis) - nonScanDims.push_back(i); - } - - createSimpleNestedLoops(rewriter, loc, outputMemRef, nonScanDims, - processDimension); - - mlir::Type resultType = mlir::memref::getTensorTypeFromMemRefType( - dyn_cast(outputMemRef.getType())); - Value outputTensor = rewriter.create( - loc, resultType, outputMemRef, true); - rewriter.create(loc, ValueRange{outputTensor}); - - scopeOp->setAttr(hivm::TCoreTypeAttr::name, - hivm::TCoreTypeAttr::get(rewriter.getContext(), - hivm::TCoreType::VECTOR)); - - rewriter.replaceOp(op, scopeOp.getResult(0)); - return success(); - } -} - -LogicalResult ScanConverter::convertToTargetOpExtended( - triton::ScanOp op, typename triton::ScanOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - bool reverse = op.getReverse(); - - // 1. Extract all input tensors (supports multiple inputs) - auto operands = op->getOperands(); - if (operands.empty()) { - return rewriter.notifyMatchFailure(op, - "No input operands for extended scan"); - } - - // 2. Validate all inputs are of RankedTensorType - llvm::SmallVector inputTensTypes; - for (auto operand : operands) { - auto tensorTy = dyn_cast(operand.getType()); - if (!tensorTy) { - return rewriter.notifyMatchFailure(op, - "All inputs must be RankedTensorType"); - } - inputTensTypes.push_back(tensorTy); - } - - // 3. Validate all input tensors have the same shape (scan operation - // requires matching input dimensions) - auto baseShape = inputTensTypes[0].getShape(); - int rank = baseShape.size(); - int axis = op.getAxis(); - if (axis < 0 || axis >= rank) { - return rewriter.notifyMatchFailure(op, "Invalid scan axis: " + - std::to_string(axis)); - } - for (size_t i = 1; i < inputTensTypes.size(); ++i) { - if (inputTensTypes[i].getShape() != baseShape) { - return rewriter.notifyMatchFailure(op, - "All inputs must have the same shape"); - } - } - - // 4. Prepare MemRefs for multiple inputs/outputs - llvm::SmallVector inputMemRefs; - llvm::SmallVector outputMemRefs; - llvm::SmallVector memRefTypes; - for (size_t i = 0; i < inputTensTypes.size(); ++i) { - auto &tensorTy = inputTensTypes[i]; - auto memRefTy = - MemRefType::get(tensorTy.getShape(), tensorTy.getElementType()); - memRefTypes.push_back(memRefTy); - // Convert input tensors to MemRefs - inputMemRefs.push_back( - rewriter.create(loc, memRefTy, operands[i])); - // Allocate MemRefs for outputs - outputMemRefs.push_back(rewriter.create(loc, memRefTy)); - } - - // 5. Define scanning logic for multiple inputs/outputs - LogicalResult loopResult = success(); - auto processDimension = [&](ArrayRef baseIdxsArray) { - llvm::SmallVector baseIdxs(baseIdxsArray.begin(), - baseIdxsArray.end()); - - auto startInd = rewriter.create(op.getLoc(), 0); - if (reverse) { - startInd = rewriter.create(op.getLoc(), - baseShape[axis] - 1); - } - - llvm::SmallVector firstIdx = baseIdxs; - if (axis <= firstIdx.size()) { - firstIdx.insert(firstIdx.begin() + axis, startInd); - } else { - firstIdx.push_back(startInd); - } - - // 5.1 Process the first element: directly copy multiple inputs to - // multiple outputs (initialize cumulative results) - for (size_t i = 0; i < inputMemRefs.size(); ++i) { - Value firstVal = - rewriter.create(loc, inputMemRefs[i], firstIdx); - rewriter.create(loc, firstVal, outputMemRefs[i], - firstIdx); - } - - Value axisSize = - rewriter.create(loc, baseShape[axis]); - Value one = rewriter.create(loc, 1); - - Value cmp = rewriter.create(loc, arith::CmpIPredicate::sgt, - axisSize, one); - auto ifOp = rewriter.create(loc, cmp, false); - - // Create a loop only when the axis size is greater than 1. - rewriter.setInsertionPointToStart(ifOp.thenBlock()); - - // Use a forward loop, but handle reverse indexing inside the loop. - auto forOp = rewriter.create(loc, one, axisSize, one); - rewriter.setInsertionPointToStart(forOp.getBody()); - - Value k = forOp.getInductionVar(); - - if (reverse) { - // Reverse scanning: Convert the forward loop index to the actual - // reverse index. (axis_size - 1) - k - Value axisSizeVal = - rewriter.create(loc, baseShape[axis]); - Value axisSizeMinusOne = - rewriter.create(loc, axisSizeVal, one); - k = rewriter.create(loc, axisSizeMinusOne, k); - } - - llvm::SmallVector currIdx = baseIdxs; - if (axis <= currIdx.size()) { - currIdx.insert(currIdx.begin() + axis, k); - } else { - currIdx.push_back(k); - } - - Value prevIndex; - if (reverse) { - prevIndex = rewriter.create(loc, k, one); - } else { - prevIndex = rewriter.create(loc, k, one); - } - - llvm::SmallVector prevIdx = baseIdxs; - if (axis <= prevIdx.size()) { - prevIdx.insert(prevIdx.begin() + axis, prevIndex); - } else { - prevIdx.push_back(prevIndex); - } - - // 5.4 Load current elements and previous cumulative results - llvm::SmallVector currentVals; - llvm::SmallVector prevResults; - for (size_t i = 0; i < inputMemRefs.size(); ++i) { - currentVals.push_back( - rewriter.create(loc, inputMemRefs[i], currIdx)); - prevResults.push_back( - rewriter.create(loc, outputMemRefs[i], prevIdx)); - } - - // 5.5 Bind parameters for custom reduction logic - Region &combineRegion = op->getRegion(0); - if (combineRegion.empty()) { - op->emitError("Missing combine region in extended scan"); - loopResult = failure(); - return; - } - Block &combineBlock = combineRegion.front(); - // Validate that the number of reduction region arguments matches (number - // of previous results + number of current elements) - if (combineBlock.getNumArguments() != 2 * inputMemRefs.size()) { - op->emitError("Combine region arguments mismatch with input count"); - loopResult = failure(); - return; - } - IRMapping mapping; - for (size_t i = 0; i < inputMemRefs.size(); ++i) { - // Bind previous results (previous value of the i-th output) to the i-th - // argument of the reduction region - mapping.map(combineBlock.getArgument(i), prevResults[i]); - // Bind current elements (current value of the i-th input) to the i+N-th - // argument of the reduction region (N is the number of inputs) - mapping.map(combineBlock.getArgument(i + inputMemRefs.size()), - currentVals[i]); - } - - // 5.6 Clone all operations within the reduction region - for (Operation &innerOp : combineBlock.without_terminator()) { - rewriter.clone(innerOp, mapping); - } - - // 5.7 Extract reduction results and store them in outputMemRef - Operation *yieldOp = combineBlock.getTerminator(); - if (yieldOp->getNumOperands() != outputMemRefs.size()) { - op->emitError("Combine region returns mismatch with output count"); - loopResult = failure(); - return; - } - for (size_t i = 0; i < outputMemRefs.size(); ++i) { - Value resultVal = mapping.lookup(yieldOp->getOperand(i)); - rewriter.create(loc, resultVal, outputMemRefs[i], - currIdx); - } - - rewriter.setInsertionPointAfter(ifOp); - }; - - // 6. Generate nested loops for non-scan dimensions - llvm::SmallVector nonScanDims; - for (int i = 0; i < rank; ++i) { - if (i != axis) - nonScanDims.push_back(i); - } - createSimpleNestedLoops(rewriter, loc, outputMemRefs[0], nonScanDims, - processDimension); - - if (failed(loopResult)) { - return failure(); - } - - // 7. Convert multiple output MemRefs back to tensors and replace the - // original tt.scan operation - llvm::SmallVector outputTensors; - for (auto outputMemRef : outputMemRefs) { - mlir::Type resultType = mlir::memref::getTensorTypeFromMemRefType( - dyn_cast(outputMemRef.getType())); - outputTensors.push_back(rewriter.create( - loc, resultType, outputMemRef, true)); - } - rewriter.replaceOp(op, outputTensors); - - return success(); -} - -LogicalResult ExternElementwiseClOpConverter::matchAndRewrite( - triton::ExternElementwiseOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - if (!op.getPure()) { - op->emitWarning() << "impure elementwise op!"; - return failure(); - } - if (op.getSymbol().contains("__hmf_")) { - Type dstTy = op.getResult().getType(); - bool isDstScalar = !isa(dstTy); - Type dstElemTy = - isDstScalar ? dstTy : cast(dstTy).getElementType(); - SmallVector srcElemTys; - SmallVector srcs; - for (auto src : op.getSrcs()) { - if (!isa(src.getType())) { - src = rewriter.create( - op.getLoc(), RankedTensorType::get({int64_t{1}}, src.getType()), - src); - } - srcs.push_back(src); - srcElemTys.push_back( - cast(src.getType()).getElementType()); - } - - // extern libdevice ops -> hivm.hir.custom - static constexpr llvm::StringLiteral libdeviceSuffixes[] = { - "_fp32", "_fp16", "_bf16", "_i32", "_i64", "_u32", "_u64"}; - bool isLibdeviceOp = - llvm::any_of(libdeviceSuffixes, [&](llvm::StringRef suffix) { - return op.getSymbol().ends_with(suffix); - }); - if (isLibdeviceOp) { - auto originalTensorType = isDstScalar - ? RankedTensorType::get({1}, dstElemTy) - : cast(dstTy); - auto originalShape = llvm::to_vector(originalTensorType.getShape()); - bool needsExpand = originalTensorType.getRank() != 1; - bool isI1Result = dstElemTy.isInteger(1); - - Type customElemType = dstElemTy; - if (isI1Result) - customElemType = rewriter.getI32Type(); - - // Flatten inputs to 1D - SmallVector collapsedInputs; - - for (Value operand : srcs) { - RankedTensorType tensorTy = cast(operand.getType()); - if (tensorTy.getRank() <= 1) { - collapsedInputs.push_back(operand); - continue; - } - - int64_t totalSize = 1; - for (int64_t dim : tensorTy.getShape()) { - if (ShapedType::isDynamic(dim)) { - totalSize = ShapedType::kDynamic; - break; - } - totalSize *= dim; - } - - ReassociationIndices indices; - for (int i = 0; i < tensorTy.getRank(); ++i) - indices.push_back(i); - - RankedTensorType collapse1DTy = - RankedTensorType::get({totalSize}, tensorTy.getElementType()); - Value collapsedVal = rewriter.create( - op.getLoc(), collapse1DTy, operand, indices); - collapsedInputs.push_back(collapsedVal); - } - - int64_t outputSize = 1; - for (int64_t dim : originalTensorType.getShape()) { - if (ShapedType::isDynamic(dim)) { - outputSize = ShapedType::kDynamic; - break; - } - outputSize *= dim; - } - auto customOutputType = - RankedTensorType::get({outputSize}, customElemType); - - SmallVector dynamicSizes; - if (ShapedType::isDynamic(outputSize)) { - for (Value collapsedInput : collapsedInputs) { - auto collapsedInputType = - dyn_cast(collapsedInput.getType()); - if (!collapsedInputType || collapsedInputType.getRank() != 1 || - !collapsedInputType.isDynamicDim(0)) - continue; - - dynamicSizes.push_back( - rewriter.create(op.getLoc(), collapsedInput, 0)); - break; - } - - if (dynamicSizes.empty()) - return rewriter.notifyMatchFailure( - op, "cannot determine the runtime size of a dynamic " - "libdevice output"); - } - - auto outputMemRefType = - MemRefType::get(customOutputType.getShape(), customElemType); - Value outputMemRef = rewriter.create( - op.getLoc(), outputMemRefType, dynamicSizes); - Value outputTensor = rewriter.create( - op.getLoc(), customOutputType, outputMemRef, true, true); - - ReassociationIndices outputReassociation; - if (needsExpand) { - for (int i = 0; i < originalTensorType.getRank(); ++i) - outputReassociation.push_back(i); - } - - std::string sym = - llvm::join(llvm::split(op.getSymbol().str(), "__hmf_"), ""); - auto customOp = rewriter.create( - op.getLoc(), sym, ValueRange{collapsedInputs}, - ValueRange{outputTensor}); - - auto argAttrsArray = mlir::ArrayAttr::get(customOp->getContext(), {}); - auto pipeAttr = - hivm::PipeAttr::get(customOp->getContext(), hivm::PIPE::PIPE_V); - auto tcoreTypeAttr = hivm::TCoreTypeAttr::get(customOp->getContext(), - hivm::TCoreType::VECTOR); - auto vfModeAttr = - hivm::VFModeAttr::get(customOp->getContext(), hivm::VFMode::SIMD); - - customOp->setAttr("bitcode", - mlir::StringAttr::get(customOp->getContext(), "")); - customOp->setAttr("hivm.pipe", pipeAttr); - customOp->setAttr("hivm.tcore_type", tcoreTypeAttr); - customOp->setAttr("hivm.vf_mode", vfModeAttr); - customOp->setAttr("symbol", - mlir::StringAttr::get(customOp->getContext(), sym)); - customOp->setAttr("arg_attrs", argAttrsArray); - customOp.setInlineMode(hivm::InlineMode::AlwaysInline); - - // Restore the result's shape and element type - Value finalResult = customOp.getResults().front(); - if (needsExpand) { - auto expandedType = - RankedTensorType::get(originalShape, customElemType); - finalResult = rewriter.create( - op->getLoc(), expandedType, finalResult, outputReassociation); - } - - if (isI1Result) { - auto resultTensorType = cast(finalResult.getType()); - auto zeroAttr = DenseElementsAttr::get( - RankedTensorType::get(resultTensorType.getShape(), - rewriter.getI32Type()), - 0); - Value zeroTensor = - rewriter.create(op->getLoc(), zeroAttr); - finalResult = rewriter.create( - op->getLoc(), arith::CmpIPredicate::ne, finalResult, zeroTensor); - } - - if (isDstScalar) { - Value zero = rewriter.create( - op->getLoc(), rewriter.getIndexAttr(0)); - finalResult = - rewriter.create(op->getLoc(), finalResult, zero); - } - - rewriter.replaceOp(op, finalResult); - return success(); - } - - // 1. get or create the declaration of external elementwise function - FunctionType elemFuncType = - FunctionType::get(rewriter.getContext(), srcElemTys, {dstElemTy}); - auto mod = SymbolTable::getNearestSymbolTable(op); - auto extFunc = dyn_cast_or_null( - SymbolTable::lookupSymbolIn(mod, op.getSymbol())); - // std::string symbol = op.getSymbol().str(); - if (!extFunc) { - OpBuilder::InsertionGuard guard(rewriter); - rewriter.setInsertionPointToStart(&mod->getRegion(0).front()); - extFunc = rewriter.create(rewriter.getUnknownLoc(), - op.getSymbol(), elemFuncType); - extFunc.setPrivate(); - extFunc->setAttr(LLVM::LLVMDialect::getReadnoneAttrName(), - UnitAttr::get(rewriter.getContext())); - // set coreType for external func, otherwise InferFuncCoreTypePass will - // fail - } - assert(isa( - SymbolTable::lookupSymbolIn(mod, op.getSymbol()))); - // 2. prepare the output tensor - Value output; - if (isDstScalar) { - dstTy = RankedTensorType::get({(int64_t)1}, dstElemTy); - } - bool found = false; - for (Value v : srcs) { - if (v.getType() == dstTy) { - found = true; - output = v; - break; - } - } - if (!found) { - output = rewriter.create( - op.getLoc(), cast(dstTy).getShape(), dstElemTy); - } - // 3. create the linalg.map op - auto mapOp = rewriter.create( - loc, - /*inputs=*/srcs, - /*init=*/output, - /*bodyBuilder=*/ - [&](OpBuilder &builder, Location loc, ValueRange regionArgs) { - auto elemOp = builder.create(loc, - /*name=*/op.getSymbol(), - /*resultType=*/dstElemTy, - /*operands=*/regionArgs); - builder.create(loc, elemOp->getResults()); - }); - if (isDstScalar) { - // need to convert tensor back to scalar - auto indexType = rewriter.getIndexType(); - Value zeroConstant = rewriter.create( - loc, indexType, rewriter.getIntegerAttr(indexType, 0)); - auto extractOp = rewriter.create( - loc, mapOp.getResults()[0], zeroConstant); - rewriter.replaceOp(op, extractOp); - } else { - rewriter.replaceOp(op, mapOp); - } - return success(); - } - return failure(); -} - -LogicalResult UnrealizedCastConverter::matchAndRewrite( - UnrealizedConversionCastOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - rewriter.eraseOp(op); - return success(); -} - -LogicalResult -JoinConverter::matchAndRewrite(triton::JoinOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Value opa = op.getLhs(); - Value opb = op.getRhs(); - auto loc = op.getLoc(); - - auto resType = dyn_cast(op.getResult().getType()); - Value emptyOp = rewriter.create(loc, resType.getShape(), - resType.getElementType()); - - auto shape = dyn_cast(opa.getType()).getShape(); - auto sizes = llvm::map_to_vector(shape, [&](int64_t t) { - return OpFoldResult(rewriter.getI64IntegerAttr(t)); - }); - sizes.push_back(rewriter.getI64IntegerAttr(1)); - - int64_t rank = resType.getRank(); - - // Set last dimension stride to 2 in layout - // As last dimension size is always 1, last dimension stride here could be - // either 1 or 2, while stride `2` could carry interleave trait and it's - // convenient for next lower. - SmallVector strides(rank, rewriter.getIndexAttr(1)); - strides.back() = rewriter.getIndexAttr(2); - - SmallVector offsets(rank, rewriter.getIndexAttr(0)); - - auto insert0 = rewriter.create( - loc, opa, emptyOp, offsets, sizes, strides); - - offsets.back() = rewriter.getIndexAttr(1); - auto insert1 = rewriter.create( - loc, opb, insert0, offsets, sizes, strides); - rewriter.replaceOp(op, insert1); - return success(); -} - -LogicalResult -CatConverter::matchAndRewrite(triton::CatOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Value opa = op.getLhs(); - Value opb = op.getRhs(); - auto loc = op.getLoc(); - - auto resType = dyn_cast(op.getResult().getType()); - if (!resType || resType.getRank() != 1) { - return rewriter.notifyMatchFailure(op, "only support 1D cat"); - } - - auto inputTypeA = dyn_cast(opa.getType()); - auto inputTypeB = dyn_cast(opb.getType()); - if (!inputTypeA || !inputTypeB || inputTypeA.getRank() != 1 || - inputTypeB.getRank() != 1) { - return rewriter.notifyMatchFailure(op, "inputs must be 1D tensors"); - } - - int64_t sizeA = inputTypeA.getShape()[0]; - int64_t sizeB = inputTypeB.getShape()[0]; - - // Only handle the case where both inputs have size 1 (i.e., scalar-like) - if (sizeA == 1 && sizeB == 1) { - // Use scalar extract + insert - auto emptyOp = rewriter.create(loc, resType.getShape(), - resType.getElementType()); - - Value zero = rewriter.create(loc, 0); - Value one = rewriter.create(loc, 1); - - Value scalarA = rewriter.create(loc, opa, zero); - Value scalarB = rewriter.create(loc, opb, zero); - - Value inserted0 = - rewriter.create(loc, scalarA, emptyOp, zero); - Value inserted1 = - rewriter.create(loc, scalarB, inserted0, one); - - rewriter.replaceOp(op, inserted1); - return success(); - } - - // General case: use tensor.insert_slice - auto emptyOp = rewriter.create(loc, resType.getShape(), - resType.getElementType()); - - auto rank = resType.getRank(); - SmallVector offsets(rank, rewriter.getIndexAttr(0)); - SmallVector strides(rank, rewriter.getIndexAttr(1)); - - auto inputType = dyn_cast(opa.getType()); - - SmallVector sizes = - llvm::map_to_vector(inputType.getShape(), [&](int64_t t) { - return OpFoldResult(rewriter.getI64IntegerAttr(t)); - }); - - auto insert0 = rewriter.create( - loc, opa, emptyOp, offsets, sizes, strides); - - offsets[0] = - rewriter.getIndexAttr(inputType.getRank() ? inputType.getShape()[0] : 1); - auto insert1 = rewriter.create( - loc, opb, insert0, offsets, sizes, strides); - - rewriter.replaceOp(op, insert1); - return success(); -} - -/// @brief Convert tt.gather to func.call. BiShengIR captures the func -/// with assumed semantics. -/// @param op The `triton::GatherOp` operation to be rewritten. -/// @param adaptor An adaptor for the operation's operands. -/// @param rewriter A pattern rewriter used to modify the IR. -/// @return A `LogicalResult` indicating whether the rewrite was successful. -LogicalResult -GatherConverter::matchAndRewrite(triton::GatherOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - Value src = adaptor.getSrc(); - Value idx = adaptor.getIndices(); - Value res = op.getResult(); - auto gatherAxis = op.getAxis(); - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - llvm::SmallString funcName = gatherFuncNameBase; - int uniqueId = 0; - while (SymbolTable::lookupSymbolIn(moduleOp, funcName)) { - funcName = gatherFuncNameBase; - funcName += ("_" + std::to_string(uniqueId++)); - } - - auto resTy = res.getType(); - auto libFnType = rewriter.getFunctionType( - {src.getType(), idx.getType(), rewriter.getI32Type()}, {resTy}); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - - rewriter.setInsertionPoint(op); - Value axis = rewriter.create(loc, gatherAxis, 32); - auto callOp = rewriter.create(loc, funcOp.getSymNameAttr(), - TypeRange({resTy}), - ValueRange({src, idx, axis})); - - rewriter.replaceOp(op, callOp); - - return success(); -} - -LogicalResult -SplitConverter::matchAndRewrite(triton::SplitOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Value input = op.getSrc(); - auto loc = op.getLoc(); - auto inputType = cast(input.getType()); - - int64_t rank = inputType.getRank(); - SmallVector offsets(rank, rewriter.getIndexAttr(0)); - // Similar to JoinConverter, here adjust last dimension stride - SmallVector strides(rank, rewriter.getIndexAttr(1)); - strides.back() = rewriter.getIndexAttr(2); - - auto outType = dyn_cast(op.getOutLHS().getType()); - auto sizes = llvm::map_to_vector(outType.getShape(), [&](int64_t t) { - return OpFoldResult(rewriter.getIndexAttr(t)); - }); - sizes.push_back(rewriter.getIndexAttr(1)); - - auto slice0 = rewriter.create( - loc, outType, input, offsets, sizes, strides); - - offsets.back() = rewriter.getIndexAttr(1); - auto slice1 = rewriter.create( - loc, outType, input, offsets, sizes, strides); - - SmallVector slices = {slice0.getResult(), slice1.getResult()}; - rewriter.replaceOp(op, ValueRange(slices)); - return success(); -} - -/* -the element-wise most significant N bits of the 2N-bit product of x and y -%x:2 = arith.mulsi_extended %y, %z : tensor<4x?xi32> -*/ -LogicalResult TritonMulhiuiConverter::matchAndRewrite( - triton::MulhiUIOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - Value opl = op.getX(); - Value opr = op.getY(); - Value res = op.getResult(); - auto newMulOp = rewriter.create( - loc, res.getType(), res.getType(), opl, opr); - // triton only need the high value - rewriter.replaceOp(op, ValueRange{newMulOp.getHigh()}); - return success(); -} - -LogicalResult TritonPreciseSqrtConverter::matchAndRewrite( - triton::PreciseSqrtOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - rewriter.replaceOpWithNewOp(op, adaptor.getOperands()); - return success(); -} - -LogicalResult DevicePrintConverter::matchAndRewrite( - triton::PrintOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - SmallVector inputTypes; - for (auto arg : op.getArgs()) { - inputTypes.push_back(arg.getType()); - } - auto libFnType = rewriter.getFunctionType(inputTypes, {}); - // See ScanConverter: the name is uniqued before the op is created so that a - // kernel named "triton_print" cannot make SymbolTable assert. - auto funcName = generateUniqueFuncName(moduleOp, printFuncNameBase); - auto funcOp = - rewriter.create(op.getLoc(), funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - auto prefixAttr = op.getPrefixAttr(); - funcOp->setAttr(prefixAttrName, prefixAttr); - auto hexAttr = op.getHexAttr(); - funcOp->setAttr(hexAttrName, hexAttr); - - rewriter.setInsertionPoint(op); - rewriter.create(op.getLoc(), funcOp, op.getArgs()); - - rewriter.eraseOp(op); - return success(); -} - -LogicalResult DeviceAssertConverter::matchAndRewrite( - triton::AssertOp op, OpAdaptor adaptor, - mlir::ConversionPatternRewriter &rewriter) const { - auto msgAttr = op.getMessageAttr(); - // The frontend marks only sanitize_overflow assertions. A user may choose - // the same text for tl.device_assert, so message matching is not a safe - // provenance test here or in the graph rewrite. - if (op->hasAttr("tt.auto_overflow_assert")) { - rewriter.eraseOp(op); - return success(); - } - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - auto conditionType = op.getCondition().getType(); - - auto libFnType = rewriter.getFunctionType({conditionType}, {}); - // See ScanConverter: the name is uniqued before the op is created so that a - // kernel named "triton_assert" cannot make SymbolTable assert. - auto funcName = generateUniqueFuncName(moduleOp, printFuncNameBase); - auto funcOp = - rewriter.create(op.getLoc(), funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - funcOp->setAttr(msgAttrName, msgAttr); - - rewriter.setInsertionPoint(op); - rewriter.create(op.getLoc(), funcOp, - ValueRange{op.getCondition()}); - - rewriter.eraseOp(op); - return success(); -} - -LogicalResult -MatmulConverter::matchAndRewrite(triton::DotOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto opa = adaptor.getA(); - auto opb = adaptor.getB(); - auto opc = adaptor.getC(); - auto dstType = cast(op.getType()); - auto elemTy = dstType.getElementType(); - auto inputPrec = op.getInputPrecision(); - // Ascend does not support tf32; map it to hf32 which provides similar - // functionality.HF32 is only valid for fp32 x fp32 inputs; for other - // dtypes, fall back to ieee. - if (inputPrec == InputPrecision::TF32) { - op->emitWarning("Ascend does not support tf32; map it to hf32."); - inputPrec = InputPrecision::HF32; - } - if (inputPrec == InputPrecision::HF32) { - auto opaElemTy = cast(opa.getType()).getElementType(); - auto opbElemTy = cast(opb.getType()).getElementType(); - if (!opaElemTy.isF32() || !opbElemTy.isF32()) { - inputPrec = InputPrecision::IEEE; - } - } - auto createOp = [&](auto &&rewriter, ValueRange operands, - ValueRange results) -> Operation * { - if (dstType.getRank() == 2) - return rewriter.template create(op.getLoc(), operands, - results); - else if (dstType.getRank() == 3) - return rewriter.template create(op.getLoc(), - operands, results); - llvm_unreachable("Datatype of DotOp operands could only be 2D or 3D"); - }; - - auto replaceOp = [&](auto &&rewriter, ValueRange operands, - ValueRange results) -> Operation * { - if (dstType.getRank() == 2) - return rewriter.template replaceOpWithNewOp( - op, operands, results); - else if (dstType.getRank() == 3) - return rewriter.template replaceOpWithNewOp( - op, operands, results); - llvm_unreachable("Datatype of DotOp operands could only be 2D or 3D"); - }; - - Operation *matmulOp; - if (mlir::isa(elemTy) && !elemTy.isF32()) { - RankedTensorType opcFp32Ty = - RankedTensorType::get(dstType.getShape(), rewriter.getF32Type()); - Value opcFp32 = rewriter.create(op.getLoc(), opcFp32Ty, opc); - matmulOp = createOp(rewriter, ValueRange{opa, opb}, ValueRange{opcFp32}); - auto roundModeAttr = hfusion::RoundModeAttr::get(rewriter.getContext(), - hfusion::RoundMode::RINT); - auto truncOp = rewriter.replaceOpWithNewOp( - op, dstType, matmulOp->getResult(0)); - truncOp->setAttr("round_mode", roundModeAttr); - } else { - matmulOp = replaceOp(rewriter, ValueRange{opa, opb}, ValueRange{opc}); - } - matmulOp->setAttr("input_precision", - rewriter.getStringAttr(stringifyInputPrecision(inputPrec))); - return success(); -} - -// Fractal (zN) block geometry for ascend.dot lowering comes from Utils.h, -// shared with DotOp::inferReturnTypes. - -// zN block column count b = 32 bytes / element bytes. -static int64_t nzBlockCol(Type elemTy) { - return triton::ascend::kBytesPerFractalCol / - (elemTy.getIntOrFloatBitWidth() / triton::ascend::kBitsPerByte); -} - -static hivm::DataLayoutAttr dotNdLayout(MLIRContext *ctx) { - return hivm::DataLayoutAttr::get(ctx, hivm::DataLayout::ND); -} - -static hivm::DataLayoutAttr dotFractalLayout(MLIRContext *ctx, int64_t blockRow, - int64_t blockCol) { - return hivm::DataLayoutAttr::get( - ctx, hivm::DataLayout::Fractal, /*transpose=*/nullptr, - DenseI64ArrayAttr::get(ctx, {blockRow, blockCol})); -} - -// A 4D fractal dot operand -> 2D ND (no-op if already ND). Both operands are zN -// (non-transpose): lhs [K1,M1,16,b] -> [M,K] and rhs [N1,K1,16,b] -> [K,N], -// each = [d1*d2, d0*d3]. -static Value dotFractalToND(ConversionPatternRewriter &rewriter, Location loc, - Value operand, bool isLhs) { - auto operandTy = cast(operand.getType()); - if (operandTy.getRank() != 4) - return operand; - auto blockedShape = operandTy.getShape(); - SmallVector ndShape{blockedShape[1] * blockedShape[2], - blockedShape[0] * blockedShape[3]}; - MLIRContext *ctx = rewriter.getContext(); - return rewriter.create( - loc, RankedTensorType::get(ndShape, operandTy.getElementType()), operand, - /*srcLayout=*/dotFractalLayout(ctx, blockedShape[2], blockedShape[3]), - /*dstLayout=*/dotNdLayout(ctx), DenseI64ArrayAttr::get(ctx, ndShape), - /*output_shape=*/ValueRange{}); -} - -// If `v` is a freshly-loaded buffer used only here -- to_tensor of an alloc -// with a single writer copy (the load) -- return that load copy so its DMA can -// be retargeted into the padded buffer (one copy, not load-then-copy); else -// null. -static memref::CopyOp findRedirectableLoad(Value v) { - auto toTensor = v.getDefiningOp(); - if (!toTensor || v.hasNUsesOrMore(2)) - return nullptr; - Value srcBuf = toTensor.getBuffer(); - if (!srcBuf.getDefiningOp()) - return nullptr; - memref::CopyOp loadCopy = nullptr; - for (Operation *user : srcBuf.getUsers()) { - if (user == toTensor.getOperation()) - continue; - // Bail on a non-copy user, a source-use of srcBuf, or a second writer. - auto copyOp = dyn_cast(user); - if (!copyOp || copyOp.getTarget() != srcBuf || loadCopy) - return nullptr; - loadCopy = copyOp; - } - return loadCopy; -} - -// Zero-pad `v`'s dimension `dim` up to `target` (no-op if already that size). -// Uses memref.alloc + fill(0) + subview + copy, NOT tensor.insert_slice: the -// latter lowers (in AscendNPU-IR AutoVectorizeV2) to a scalar copy loop that -// blows up cbuf. -static Value zeroPadDimTo(Value v, int64_t dim, int64_t target, - ConversionPatternRewriter &rewriter, Location loc) { - auto t = cast(v.getType()); - if (t.getShape()[dim] == target) - return v; - Type elemTy = t.getElementType(); - - // Push the pad through a transpose (pad the mapped input dim, re-transpose). - if (auto trans = v.getDefiningOp()) { - ArrayRef perm = trans.getPermutation(); - Value paddedInput = - zeroPadDimTo(trans.getInput(), perm[dim], target, rewriter, loc); - SmallVector outShape(t.getShape().begin(), t.getShape().end()); - outShape[dim] = target; - Value init = rewriter.create(loc, outShape, elemTy); - return rewriter.create(loc, paddedInput, init, perm) - .getResults()[0]; - } - - SmallVector paddedShape(t.getShape().begin(), t.getShape().end()); - paddedShape[dim] = target; - - // Allocate the padded buffer and zero-fill it. - auto paddedMemTy = MemRefType::get(paddedShape, elemTy); - Value padded = rewriter.create(loc, paddedMemTy); - Value zero = - rewriter.create(loc, rewriter.getZeroAttr(elemTy)); - rewriter.create(loc, ValueRange{zero}, ValueRange{padded}); - - // Subview covering the real (unpadded) region, then fill it with one copy. - int64_t rank = t.getRank(); - SmallVector offsets(rank, rewriter.getIndexAttr(0)); - SmallVector strides(rank, rewriter.getIndexAttr(1)); - SmallVector sizes; - for (int64_t d : t.getShape()) - sizes.push_back(rewriter.getIndexAttr(d)); - auto subviewTy = - memref::SubViewOp::inferResultType(paddedMemTy, offsets, sizes, strides); - Value subview = rewriter.create( - loc, cast(subviewTy), padded, offsets, sizes, strides); - - if (memref::CopyOp loadCopy = findRedirectableLoad(v)) { - rewriter.create(loc, loadCopy.getSource(), subview); - rewriter.eraseOp(loadCopy); - } else { - auto srcBufferTy = MemRefType::get(t.getShape(), elemTy); - Value srcBuffer = - rewriter.create(loc, srcBufferTy, v); - rewriter.create(loc, srcBuffer, subview); - } - - return rewriter.create( - loc, RankedTensorType::get(paddedShape, elemTy), padded, - /*restrict=*/true, /*writable=*/true); -} - -// Reconcile the ND operands' contraction (K) dim (a/b mutated in place). When -// exactly one operand is fractal, its K is block-rounded while the ND side may -// be shorter; zero-padding the ND side to the block-aligned K is safe (the -// extra K products are * 0). Only an in-one-block gap is padding; else error. -static LogicalResult -reconcileDotContractionK(triton::ascend::DotOp op, Value &a, Value &b, - Type elemTy, ConversionPatternRewriter &rewriter, - Location loc) { - if (op.getFractalA() == op.getFractalB()) - return success(); - int64_t kA = cast(a.getType()).getShape()[1]; // A_nd [M,kA] - int64_t kB = cast(b.getType()).getShape()[0]; // B_nd [kB,N] - if (kA == kB) - return success(); - int64_t paddedK = std::max(kA, kB), shortK = std::min(kA, kB); - int64_t blockK = nzBlockCol(elemTy); - if (paddedK % blockK != 0 || paddedK - shortK >= blockK) - return op.emitError() - << "dot: contraction dim mismatch (A K=" << kA << ", B K=" << kB - << ") is larger than fractal padding; pad the operand " - "explicitly"; - op.emitWarning() << "dot: auto zero-padding the non-fractal operand's K from " - << shortK << " to " << paddedK - << " to match the fractal operand's block-aligned K"; - a = zeroPadDimTo(a, /*dim=*/1, paddedK, rewriter, loc); // A_nd [M,K] - b = zeroPadDimTo(b, /*dim=*/0, paddedK, rewriter, loc); // B_nd [K,N] - return success(); -} - -// Decompose `ascend.dot` (D = A * B): fractal inputs -> ND, a plain -// linalg.matmul, then (if fractal_c) the ND result back to fractal. -LogicalResult -DotConverter::matchAndRewrite(triton::ascend::DotOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Location loc = op.getLoc(); - MLIRContext *ctx = rewriter.getContext(); - auto resTy = cast(op.getType()); - // The result carries the cube accumulator dtype (f32/i32); the matmul inputs - // and K-padding use the operands' own (input) dtype. - Type accTy = resTy.getElementType(); - - Value a = op.getFractalA() - ? dotFractalToND(rewriter, loc, adaptor.getA(), /*isLhs=*/true) - : adaptor.getA(); - Value b = op.getFractalB() - ? dotFractalToND(rewriter, loc, adaptor.getB(), /*isLhs=*/false) - : adaptor.getB(); - Type inElemTy = cast(a.getType()).getElementType(); - if (failed(reconcileDotContractionK(op, a, b, inElemTy, rewriter, loc))) - return failure(); - - // Uninitialized accumulator (accTy): mmadL1 (accumulate=false) overwrites it. - int64_t numRows = cast(a.getType()).getShape()[0]; - int64_t numCols = cast(b.getType()).getShape()[1]; - auto ndResultTy = RankedTensorType::get({numRows, numCols}, accTy); - Value acc = rewriter.create( - loc, ArrayRef{numRows, numCols}, accTy); - auto matmul = rewriter.create( - loc, TypeRange{ndResultTy}, ValueRange{a, b}, ValueRange{acc}); - matmul->setAttr("input_precision", rewriter.getStringAttr("ieee")); - Value ndResult = matmul->getResult(0); - - if (!op.getFractalC()) { - rewriter.replaceOp(op, ndResult); - return success(); - } - // fractal_c: ND result -> the L0C accumulator fractal (block [16, 16]). - Value fractalResult = rewriter.create( - loc, resTy, ndResult, /*srcLayout=*/dotNdLayout(ctx), - /*dstLayout=*/ - dotFractalLayout(ctx, triton::ascend::kFractalBlock, - triton::ascend::kFractalBlock), - DenseI64ArrayAttr::get(ctx, resTy.getShape()), /*output_shape=*/ - ValueRange{}); - rewriter.replaceOp(op, fractalResult); - return success(); -} - -LogicalResult -FlipOpConverter::matchAndRewrite(triton::ascend::FlipOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Value src = adaptor.getSrc(); - auto rankedSrcTy = cast(src.getType()); - - MLIRContext *ctx = rewriter.getContext(); - - Type valuesTy = src.getType(); - Location loc = op.getLoc(); - - auto dimAttr = op->getAttrOfType("dim"); - if (!dimAttr) { - op->emitError("missing 'dim' attribute"); - return failure(); - } - - auto moduleOp = op->getParentOfType(); - if (!moduleOp) { - op->emitError("must be inside a module"); - return failure(); - } - - // Unique callee name: triton_flip, triton_flip_1, … - std::string funcName = baseFuncName.str(); - int uniqueId = 0; - while (SymbolTable::lookupSymbolIn(moduleOp, funcName)) - funcName = (baseFuncName + Twine("_") + Twine(uniqueId++)).str(); - - auto i64Ty = IntegerType::get(ctx, 64); - auto libFnType = - rewriter.getFunctionType({rankedSrcTy, i64Ty}, {rankedSrcTy}); - - // Declare the callee - auto moduleIP = rewriter.saveInsertionPoint(); - rewriter.setInsertionPointToEnd(moduleOp.getBody()); - auto funcOp = rewriter.create(loc, funcName, libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - rewriter.restoreInsertionPoint(moduleIP); - - // dim constant - Value dimVal = - rewriter.create(loc, dimAttr.getInt(), 64); - - // Call the backend function - auto callee = SymbolRefAttr::get(ctx, funcOp.getSymName()); - auto callOp = rewriter.create( - loc, TypeRange({rankedSrcTy}), callee, ValueRange({src, dimVal})); - - Value finalValues = callOp.getResult(0); - - rewriter.replaceOp(op, {finalValues}); - return success(); -} - -LogicalResult -SortOpConverter::matchAndRewrite(triton::ascend::SortOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Value src = adaptor.getSrc(); - auto rankedSrcTy = cast(src.getType()); - auto srcElemTy = rankedSrcTy.getElementType(); - auto srcShape = rankedSrcTy.getShape(); - auto srcEnc = rankedSrcTy.getEncoding(); - - MLIRContext *ctx = rewriter.getContext(); - - Type backendElemTy = srcElemTy; - if (srcElemTy.isInteger(8)) { - backendElemTy = Float16Type::get(ctx); // i8 -> f16 - } else if (srcElemTy.isInteger(16)) { - backendElemTy = Float32Type::get(ctx); // i16 -> f32 - } - Type backendTensorTy = RankedTensorType::get(srcShape, backendElemTy, srcEnc); - - Type valuesTy = src.getType(); - - Location loc = op.getLoc(); - auto dimAttr = op->getAttrOfType("dim"); - auto descAttr = op->getAttrOfType("descending"); - if (!dimAttr || !descAttr) { - op->emitError("missing 'dim' or 'descending' attribute"); - return failure(); - } - - auto moduleOp = op->getParentOfType(); - if (!moduleOp) { - op->emitError("must be inside a module"); - return failure(); - } - - llvm::SmallString<64> baseName("triton_sort"); - llvm::SmallString<64> funcName = baseName; - int uniqueId = 0; - while (SymbolTable::lookupSymbolIn(moduleOp, funcName)) { - funcName = baseName; - funcName += ("_" + std::to_string(uniqueId++)); - } - - auto i64Ty = IntegerType::get(ctx, 64); - auto i1Ty = IntegerType::get(ctx, 1); - auto libFnType = rewriter.getFunctionType({backendTensorTy, i64Ty, i1Ty}, - {backendTensorTy}); - - auto moduleIP = rewriter.saveInsertionPoint(); - rewriter.setInsertionPointToEnd(moduleOp.getBody()); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - rewriter.restoreInsertionPoint(moduleIP); - - Value srcForCall = src; - if (backendElemTy != srcElemTy) { - srcForCall = rewriter.create(loc, backendTensorTy, src); - } - - Value dimVal = - rewriter.create(loc, dimAttr.getInt(), 64); - Value descVal = rewriter.create( - loc, descAttr.getValue() ? 1 : 0, 1); - - auto callee = SymbolRefAttr::get(ctx, funcOp.getSymName()); - auto callOp = - rewriter.create(loc, TypeRange({backendTensorTy}), callee, - ValueRange({srcForCall, dimVal, descVal})); - - Value valuesFloat = callOp.getResult(0); // tensor - - Value finalValues = valuesFloat; - if (backendElemTy != srcElemTy) { - finalValues = rewriter.create(loc, valuesTy, valuesFloat); - } - - rewriter.replaceOp(op, {finalValues}); - - return success(); -} - -LogicalResult -DotScaledConverter::matchAndRewrite(triton::DotScaledOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Location loc = op.getLoc(); - - Value lhs = adaptor.getA(); - Value rhs = adaptor.getB(); - Value c = adaptor.getC(); - Value lhsScale = adaptor.getAScale(); - Value rhsScale = adaptor.getBScale(); - RankedTensorType dstType = cast(op.getType()); - - auto lhsElemType = op.getAElemType(); - auto rhsElemType = op.getBElemType(); - - bool isFP8Input = (lhsElemType == triton::ScaleDotElemType::E4M3 || - lhsElemType == triton::ScaleDotElemType::E5M2) && - (rhsElemType == triton::ScaleDotElemType::E4M3 || - rhsElemType == triton::ScaleDotElemType::E5M2); - bool isFP4Input = (lhsElemType == triton::ScaleDotElemType::E2M1) && - (rhsElemType == triton::ScaleDotElemType::E2M1); - if ((isFP8Input || isFP4Input) && !lhsScale) { - auto lhsTy = cast(lhs.getType()); - int64_t k = lhsTy.getShape().back(); - if (lhsElemType == triton::ScaleDotElemType::E2M1 && op.getLhsKPack()) - k *= 2; - auto defaultScaleTy = RankedTensorType::get( - {dstType.getDimSize(dstType.getRank() - 2), (k + 31) / 32}, - rewriter.getI8Type()); - // The native MX path uses E8M0, whose encoding of one is 127. - Value defaultScaleVal = - rewriter.create(loc, rewriter.getI8IntegerAttr(127)); - Value defaultScaleEmpty = rewriter.create( - loc, defaultScaleTy.getShape(), defaultScaleTy.getElementType()); - lhsScale = rewriter - .create(loc, ValueRange{defaultScaleVal}, - ValueRange{defaultScaleEmpty}) - .getResult(0); - } - if (isFP8Input || isFP4Input) { - if (!rhsScale) { - auto rhsTy = cast(rhs.getType()); - int64_t k = rhsTy.getDimSize(rhsTy.getRank() - 2); - if (rhsElemType == triton::ScaleDotElemType::E2M1 && op.getRhsKPack()) - k *= 2; - auto defaultScaleTy = RankedTensorType::get( - {dstType.getShape().back(), (k + 31) / 32}, rewriter.getI8Type()); - // E8M0 encodes a scale factor of one with exponent bias 127. - Value defaultScaleVal = rewriter.create( - loc, rewriter.getI8IntegerAttr(127)); - Value defaultScaleEmpty = rewriter.create( - loc, defaultScaleTy.getShape(), defaultScaleTy.getElementType()); - rhsScale = rewriter - .create(loc, ValueRange{defaultScaleVal}, - ValueRange{defaultScaleEmpty}) - .getResult(0); - } - Value acc = c ? c - : rewriter.create(loc, dstType.getShape(), - dstType.getElementType()); - - // Get the ScaleDotElemType enum - // Helper to convert - auto convertFormat = - [&](triton::ScaleDotElemType ty) -> mlir::hfusion::DataformatAttr { - auto ctx = rewriter.getContext(); - switch (ty) { - case triton::ScaleDotElemType::E2M1: - return mlir::hfusion::DataformatAttr::get( - ctx, mlir::hfusion::Dataformat::FP4E2M1_T); - case triton::ScaleDotElemType::E4M3: - return mlir::hfusion::DataformatAttr::get( - ctx, mlir::hfusion::Dataformat::FP8E4M3_T); - case triton::ScaleDotElemType::E5M2: - return mlir::hfusion::DataformatAttr::get( - ctx, mlir::hfusion::Dataformat::FP8E5M2_T); - default: - llvm_unreachable("unsupported ScaleDotElemType"); - } - }; - - Value matmulMxResult = rewriter.create( - loc, dstType, lhs, rhs, lhsScale, rhsScale, acc, - /*lhsFormat(optional)*/ convertFormat(lhsElemType), - /*rhsFormat(optional)*/ convertFormat(rhsElemType)); - - Value finalResult = matmulMxResult; - if (dstType.getElementType().isBF16()) { - finalResult = - rewriter.create(loc, dstType, matmulMxResult); - } - rewriter.replaceOp(op, finalResult); - return success(); - } - - if (!lhsScale) { - return op.emitError("lhsScale is required for non-FP8 input"); - } - - RankedTensorType lhsTy = cast(lhs.getType()); - RankedTensorType lhsScaleTy = cast(lhsScale.getType()); - RankedTensorType rhsScaleTy = - rhsScale ? cast(rhsScale.getType()) : nullptr; - RankedTensorType rhsTy = cast(rhs.getType()); - - Value lhsScaleOut; - Value rhsScaleOut; - Value c127 = rewriter.create( - op.getLoc(), rewriter.getI16Type(), rewriter.getI16IntegerAttr(127)); - Value c7 = rewriter.create( - op.getLoc(), rewriter.getI16Type(), rewriter.getI16IntegerAttr(7)); - Type i16Ty = rewriter.getI16Type(); - Type bf16Ty = rewriter.getBF16Type(); - Type fp16Ty = rewriter.getF16Type(); - Type fp32Ty = rewriter.getF32Type(); - bool fastMath = op.getFastMath(); - - auto createNanSplat = [&](RankedTensorType tensorTy) -> Value { - auto floatTy = cast(tensorTy.getElementType()); - auto nanAttr = rewriter.getFloatAttr( - floatTy, APFloat::getNaN(floatTy.getFloatSemantics())); - Value empty = rewriter.create(loc, tensorTy.getShape(), - tensorTy.getElementType()); - return rewriter - .create( - loc, ValueRange{rewriter.create(loc, nanAttr)}, - ValueRange{empty}) - .getResult(0); - }; - - auto createNaNMask = [&](Value scaleTensor, - RankedTensorType scaleTy) -> Value { - if (scaleTy.getElementType().isIntOrIndex()) { - auto bitWidth = scaleTy.getElementTypeBitWidth(); - auto allOnes = APInt::getAllOnes(bitWidth); - auto sentinel = rewriter.create( - loc, scaleTy, DenseElementsAttr::get(scaleTy, allOnes)); - return rewriter - .create(loc, arith::CmpIPredicate::eq, scaleTensor, - sentinel) - .getResult(); - } - - return rewriter - .create(loc, arith::CmpFPredicate::UNO, scaleTensor, - scaleTensor) - .getResult(); - }; - - auto applyNaNMask = [&](Value valueTensor, Value maskTensor) -> Value { - auto valueTy = cast(valueTensor.getType()); - Value nanTensor = createNanSplat(valueTy); - return rewriter - .create(loc, maskTensor, nanTensor, valueTensor) - .getResult(); - }; - - if (lhsScaleTy.getElementType().isIntOrIndex()) { - RankedTensorType lhsScaleI16Ty = - RankedTensorType::get(lhsScaleTy.getShape(), i16Ty); - Value lhsScaleI16; - lhsScaleI16 = - rewriter.create(op.getLoc(), lhsScaleI16Ty, lhsScale); - - Value lhsShift7Empty = rewriter.create( - op.getLoc(), lhsScaleI16Ty.getShape(), i16Ty); - Value lhsShift7 = rewriter - .create(op.getLoc(), ValueRange{c7}, - ValueRange{lhsShift7Empty}) - .getResult(0); - Value lhsScaleI16Shifted = - rewriter.create(op.getLoc(), lhsScaleI16, lhsShift7); - - RankedTensorType lhsScaleBF16Ty = - RankedTensorType::get(lhsScaleTy.getShape(), bf16Ty); - Value lhsScaleBF16 = rewriter.create( - op.getLoc(), lhsScaleBF16Ty, lhsScaleI16Shifted); - if (lhsTy.getElementType() == fp16Ty) { - RankedTensorType lhsScaleFp32Ty = - RankedTensorType::get(lhsScaleTy.getShape(), fp32Ty); - Value lhsScaleFp32 = rewriter.create( - op.getLoc(), lhsScaleFp32Ty, lhsScaleBF16); - RankedTensorType lhsScaleFp16Ty = - RankedTensorType::get(lhsScaleTy.getShape(), fp16Ty); - lhsScaleOut = rewriter.create( - op.getLoc(), lhsScaleFp16Ty, lhsScaleFp32); - } else { - lhsScaleOut = lhsScaleBF16; - } - } else { - lhsScaleOut = - rewriter - .create( - op.getLoc(), - RankedTensorType::get(lhsScaleTy.getShape(), fp32Ty), lhsScale) - .getResult(); - } - - if (rhsScale && rhsScaleTy.getElementType().isIntOrIndex()) { - if (rhsScaleTy.getRank() != 2) { - return op.emitError("rhsScale must be 2D for transpose"); - } - - SmallVector transposedShape = {rhsScaleTy.getShape()[1], - rhsScaleTy.getShape()[0]}; - RankedTensorType transposedRhsScaleTy = - RankedTensorType::get(transposedShape, rhsScaleTy.getElementType()); - - Value transposedRhsScale = rewriter.create( - op.getLoc(), transposedRhsScaleTy, rhsScale, - DenseI32ArrayAttr::get(rewriter.getContext(), ArrayRef{1, 0})); - RankedTensorType rhsScaleI16Ty = - RankedTensorType::get(transposedShape, i16Ty); - Value rhsScaleI16; - rhsScaleI16 = rewriter.create(op.getLoc(), rhsScaleI16Ty, - transposedRhsScale); - - Value rhsShift7Empty = rewriter.create( - op.getLoc(), rhsScaleI16Ty.getShape(), i16Ty); - Value rhsShift7 = rewriter - .create(op.getLoc(), ValueRange{c7}, - ValueRange{rhsShift7Empty}) - .getResult(0); - Value rhsScaleI16Shifted = - rewriter.create(op.getLoc(), rhsScaleI16, rhsShift7); - - RankedTensorType rhsScaleBF16Ty = - RankedTensorType::get(transposedShape, bf16Ty); - Value rhsScaleBF16 = rewriter.create( - op.getLoc(), rhsScaleBF16Ty, rhsScaleI16Shifted); - - if (rhsTy.getElementType() == fp16Ty) { - RankedTensorType rhsScaleFp32Ty = - RankedTensorType::get(transposedShape, fp32Ty); - Value rhsScaleFp32 = rewriter.create( - op.getLoc(), rhsScaleFp32Ty, rhsScaleBF16); - RankedTensorType rhsScaleFp16Ty = - RankedTensorType::get(transposedShape, fp16Ty); - rhsScaleOut = rewriter.create( - op.getLoc(), rhsScaleFp16Ty, rhsScaleFp32); - } else { - rhsScaleOut = rhsScaleBF16; - } - int64_t rhsD0 = rhsScaleTy.getShape()[1]; - int64_t rhsD1 = rhsScaleTy.getShape()[0]; - SmallVector rhsExpandedShape1 = {rhsD0, rhsD1, 1}; - RankedTensorType rhsExpandedTy1 = - RankedTensorType::get(rhsExpandedShape1, rhsTy.getElementType()); - Value rhsExpanded1 = rewriter - .create( - op.getLoc(), rhsExpandedTy1, rhsScaleOut, - rewriter.getI32IntegerAttr(2)) - .getResult(); - - int64_t rhsDim1 = rhsTy.getShape()[0]; - if (rhsDim1 % rhsD0 != 0) { - return op.emitError( - "rhs dim0 must be an integer multiple of rhsScale dim0"); - } - int64_t rhsD2 = rhsDim1 / rhsD0; - SmallVector rhsBroadcastShape = {rhsD0, rhsD1, rhsD2}; - RankedTensorType rhsBroadcastTy = - RankedTensorType::get(rhsBroadcastShape, rhsTy.getElementType()); - Value rhsBroadcasted = rewriter - .create( - op.getLoc(), rhsBroadcastTy, rhsExpanded1) - .getResult(); - - SmallVector transposeOrder = {0, 2, 1}; - Value transposedBroadcasted = rewriter.create( - op.getLoc(), - RankedTensorType::get({rhsD0, rhsD2, rhsD1}, rhsTy.getElementType()), - rhsBroadcasted, - DenseI32ArrayAttr::get(rewriter.getContext(), transposeOrder)); - SmallVector rhsReassociation; - rhsReassociation.push_back({0, 1}); - rhsReassociation.push_back({2}); - - Value scaledRhs = rewriter - .create( - op.getLoc(), - RankedTensorType::get({rhsD0 * rhsD2, rhsD1}, - rhsTy.getElementType()), - transposedBroadcasted, rhsReassociation) - .getResult(); - - rhs = - rewriter.create(op.getLoc(), rhs, scaledRhs).getResult(); - - if (!fastMath) { - Value rhsScaleNaNMask = - createNaNMask(transposedRhsScale, transposedRhsScaleTy); - Value rhsExpandedMask = - rewriter - .create( - op.getLoc(), - RankedTensorType::get(rhsExpandedShape1, - rewriter.getI1Type()), - rhsScaleNaNMask, rewriter.getI32IntegerAttr(2)) - .getResult(); - Value rhsBroadcastMask = - rewriter - .create( - op.getLoc(), - RankedTensorType::get(rhsBroadcastShape, - rewriter.getI1Type()), - rhsExpandedMask) - .getResult(); - Value transposedBroadcastMask = - rewriter - .create( - op.getLoc(), - RankedTensorType::get({rhsD0, rhsD2, rhsD1}, - rewriter.getI1Type()), - rhsBroadcastMask, - DenseI32ArrayAttr::get(rewriter.getContext(), transposeOrder)) - .getResult(); - Value collapsedRhsMask = - rewriter - .create( - op.getLoc(), - RankedTensorType::get({rhsD0 * rhsD2, rhsD1}, - rewriter.getI1Type()), - transposedBroadcastMask, rhsReassociation) - .getResult(); - rhs = applyNaNMask(rhs, collapsedRhsMask); - } - } - - int64_t D0 = lhsScaleTy.getShape()[0]; - int64_t D1 = lhsScaleTy.getShape()[1]; - SmallVector expandedShape1 = {D0, D1, 1}; - RankedTensorType expandedTy1 = - RankedTensorType::get(expandedShape1, lhsTy.getElementType()); - Value expanded1 = - rewriter - .create(op.getLoc(), expandedTy1, lhsScaleOut, - rewriter.getI32IntegerAttr(2)) - .getResult(); - - int64_t lhsDim1 = lhsTy.getShape()[1]; - if (lhsDim1 % D1 != 0) { - return op.emitError( - "lhs dim1 must be an integer multiple of lhsScale dim1"); - } - int64_t D2 = lhsDim1 / D1; - SmallVector broadcastShape = {D0, D1, D2}; - RankedTensorType broadcastTy = - RankedTensorType::get(broadcastShape, lhsTy.getElementType()); - Value broadcasted = - rewriter.create(op.getLoc(), broadcastTy, expanded1) - .getResult(); - - SmallVector reassociation; - reassociation.push_back({0}); - reassociation.push_back({1, 2}); - - Value scaledLhs = - rewriter - .create( - op.getLoc(), - RankedTensorType::get({D0, D1 * D2}, lhsTy.getElementType()), - broadcasted, reassociation) - .getResult(); - - Value scaledLhsFinal = - rewriter.create(op.getLoc(), lhs, scaledLhs).getResult(); - - if (!fastMath) { - Value lhsScaleNaNMask = createNaNMask(lhsScale, lhsScaleTy); - Value lhsExpandedMask = - rewriter - .create( - op.getLoc(), - RankedTensorType::get(expandedShape1, rewriter.getI1Type()), - lhsScaleNaNMask, rewriter.getI32IntegerAttr(2)) - .getResult(); - Value lhsBroadcastMask = - rewriter - .create( - op.getLoc(), - RankedTensorType::get(broadcastShape, rewriter.getI1Type()), - lhsExpandedMask) - .getResult(); - Value collapsedLhsMask = - rewriter - .create( - op.getLoc(), - RankedTensorType::get({D0, D1 * D2}, rewriter.getI1Type()), - lhsBroadcastMask, reassociation) - .getResult(); - scaledLhsFinal = applyNaNMask(scaledLhsFinal, collapsedLhsMask); - } - - Operation *matmulOp; - if (dstType.getRank() == 2) { - matmulOp = rewriter.create( - op.getLoc(), ValueRange{scaledLhsFinal, rhs}, ValueRange{c}); - } else if (dstType.getRank() == 3) { - matmulOp = rewriter.create( - op.getLoc(), ValueRange{scaledLhsFinal, rhs}, ValueRange{c}); - } else { - return op.emitError("DotScaledOp only support 2D or 3D tensor"); - } - - rewriter.replaceOp(op, matmulOp->getResults()); - return success(); -} - -LogicalResult -IntToPtrConverter::matchAndRewrite(triton::IntToPtrOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto pointerType = dyn_cast(op.getType()); - if (!pointerType || isa(pointerType.getPointeeType())) - return rewriter.notifyMatchFailure( - op, "only scalar pointer reconstruction is supported"); - - const TypeConverter *typeConverter = getTypeConverter(); - if (!typeConverter) - return rewriter.notifyMatchFailure(op, "requires a type converter"); - auto resultType = - dyn_cast(typeConverter->convertType(op.getType())); - if (!resultType) - return rewriter.notifyMatchFailure( - op, "pointer result did not convert to a memref type"); - - // Rebuild the memref only after the integer address has crossed control - // flow. Selecting an i64 address is a pure SSA operation and avoids the - // backend interpreting a memref-valued merge as a GM-to-UB copy. - auto pointerCast = createScalarPointerCast(rewriter, op.getLoc(), resultType, - adaptor.getSrc()); - rewriter.replaceOp(op, pointerCast.getResult()); - return success(); -} - -LogicalResult -PtrToIntConverter::matchAndRewrite(triton::PtrToIntOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - FailureOr address = - materializePointerAddress(adaptor.getSrc(), op.getLoc(), rewriter); - if (failed(address)) { - return rewriter.notifyMatchFailure(op, "input is not a memref type"); - } - - rewriter.replaceOp(op, *address); - return success(); -} - -LogicalResult -IndexPutConverter::matchAndRewrite(triton::ascend::IndexPutOp op, - OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - auto funcName = generateUniqueFuncName(moduleOp, funcNameBase); - - auto ptr = adaptor.getPtr(); - auto index = op.getIndex(); - auto value = op.getValue(); - auto dim = op.getDim(); - auto indexBoundary = op.getIndexBoundary(); - auto endOffset = op.getEndOffset(); - auto startOffset = op.getStartOffset(); - auto dstStride = adaptor.getDstStride(); - - // convert !tt.ptr to memref - auto ptrTy = dyn_cast(ptr.getType()); - if (!ptrTy) { - return rewriter.notifyMatchFailure(op, "expected MemRefType for ptr"); - } - SmallVector inputTypes({ptrTy, index.getType(), value.getType(), - dim.getType(), indexBoundary.getType()}); - inputTypes.append(endOffset.getTypes().begin(), endOffset.getTypes().end()); - inputTypes.append(startOffset.getTypes().begin(), - startOffset.getTypes().end()); - inputTypes.append(dstStride.getTypes().begin(), dstStride.getTypes().end()); - auto libFnType = rewriter.getFunctionType(inputTypes, {}); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - - rewriter.setInsertionPoint(op); - SmallVector inputVals({ptr, index, value, dim, indexBoundary}); - inputVals.append(endOffset.begin(), endOffset.end()); - inputVals.append(startOffset.begin(), startOffset.end()); - inputVals.append(dstStride.begin(), dstStride.end()); - rewriter.create(loc, funcOp.getSymNameAttr(), TypeRange({}), - inputVals); - rewriter.eraseOp(op); - return success(); -} - -LogicalResult GatherOutToUbConverter::matchAndRewrite( - triton::ascend::GatherOutToUbOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - auto funcName = generateUniqueFuncName(moduleOp, funcNameBase); - - auto src = adaptor.getSrc(); - auto index = op.getIndex(); - auto indexBoundary = op.getIndexBoundary(); - auto dim = op.getDim(); - auto srcStride = op.getSrcStride(); - auto endOffset = op.getEndOffset(); - auto startOffset = op.getStartOffset(); - auto other = op.getOther(); - - auto res = op.getResult(); - auto resTy = res.getType(); - - // convert !tt.ptr to memref - auto srcTy = dyn_cast(src.getType()); - if (!srcTy) { - return rewriter.notifyMatchFailure(op, "expected MemRefType for src"); - } - - SmallVector inputTypes( - {srcTy, index.getType(), indexBoundary.getType(), dim.getType()}); - inputTypes.append(srcStride.getTypes().begin(), srcStride.getTypes().end()); - inputTypes.append(endOffset.getTypes().begin(), endOffset.getTypes().end()); - inputTypes.append(startOffset.getTypes().begin(), - startOffset.getTypes().end()); - if (other) - inputTypes.push_back(other.getType()); - - auto libFnType = rewriter.getFunctionType(inputTypes, {resTy}); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - - rewriter.setInsertionPoint(op); - SmallVector inputVals({src, index, indexBoundary, dim}); - inputVals.append(srcStride.begin(), srcStride.end()); - inputVals.append(endOffset.begin(), endOffset.end()); - inputVals.append(startOffset.begin(), startOffset.end()); - if (other) - inputVals.push_back(other); - auto callOp = rewriter.create(loc, funcOp.getSymNameAttr(), - TypeRange({resTy}), inputVals); - rewriter.replaceOp(op, callOp); - return success(); -} - -LogicalResult ScatterUbToOutConverter::matchAndRewrite( - triton::ascend::ScatterUbToOutOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - auto funcName = generateUniqueFuncName(moduleOp, funcNameBase); - - auto ptr = adaptor.getPtr(); - auto value = op.getValue(); - auto index = op.getIndex(); - auto indexBoundary = op.getIndexBoundary(); - auto dim = op.getDim(); - auto dstStride = op.getDstStride(); - auto endOffset = op.getEndOffset(); - auto startOffset = op.getStartOffset(); - - // convert !tt.ptr to memref - auto ptrTy = dyn_cast(ptr.getType()); - if (!ptrTy) { - return rewriter.notifyMatchFailure(op, "expected MemRefType for ptr"); - } - - SmallVector inputTypes({ptrTy, value.getType(), index.getType(), - indexBoundary.getType(), dim.getType()}); - inputTypes.append(dstStride.getTypes().begin(), dstStride.getTypes().end()); - inputTypes.append(endOffset.getTypes().begin(), endOffset.getTypes().end()); - inputTypes.append(startOffset.getTypes().begin(), - startOffset.getTypes().end()); - - auto libFnType = rewriter.getFunctionType(inputTypes, {}); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - - rewriter.setInsertionPoint(op); - SmallVector inputVals({ptr, value, index, indexBoundary, dim}); - inputVals.append(dstStride.begin(), dstStride.end()); - inputVals.append(endOffset.begin(), endOffset.end()); - inputVals.append(startOffset.begin(), startOffset.end()); - rewriter.create(loc, funcOp.getSymNameAttr(), TypeRange({}), - inputVals); - rewriter.eraseOp(op); - return success(); -} - -LogicalResult IndirectLoadConverter::matchAndRewrite( - triton::ascend::IndirectLoadOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - auto funcName = generateUniqueFuncName(moduleOp, funcNameBase); - - auto src = adaptor.getSrc(); - auto offsets = op.getOffsets(); - auto mask = op.getMask(); - auto other = op.getOther(); - auto res = op.getResult(); - auto resTy = res.getType(); - - // The adaptor may already expose the dynamic carrier created by - // IntToPtrConverter. Re-materialize that narrow provenance before the - // MemRefType fast path, otherwise the indirect helper receives memref - // instead of the required one-element identity descriptor. - if (isIntToPtrBasedScalarPointer(op.getSrc()) || - (op.getIsVolatile() && isVolatileBitcastScalarPointer(op.getSrc()))) { - llvm::SmallDenseMap known; - FailureOr materialized = - BlockDataParser::materializePointer(op.getSrc(), rewriter, known); - if (failed(materialized)) - return rewriter.notifyMatchFailure( - op, "unable to materialize int_to_ptr indirect-load source"); - src = *materialized; - } else if (!isa(src.getType())) { - llvm::SmallDenseMap known; - FailureOr materialized = - BlockDataParser::materializePointer(op.getSrc(), rewriter, known); - if (failed(materialized)) - return rewriter.notifyMatchFailure( - op, "unable to materialize indirect-load source as a memref"); - src = *materialized; - } - // convert !tt.ptr to memref - auto srcTy = dyn_cast(src.getType()); - if (!srcTy) { - return rewriter.notifyMatchFailure(op, "expected MemRefType for src"); - } - SmallVector inputTypes({srcTy, offsets.getType()}); - if (mask) - inputTypes.push_back(mask.getType()); - if (other) - inputTypes.push_back(other.getType()); - auto libFnType = rewriter.getFunctionType(inputTypes, {resTy}); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - auto isVolatileAttr = rewriter.getBoolAttr(op.getIsVolatile()); - funcOp->setAttr("isVolatile", isVolatileAttr); - - rewriter.setInsertionPoint(op); - SmallVector inputVals({src, offsets}); - if (mask) - inputVals.push_back(mask); - if (other) - inputVals.push_back(other); - auto callOp = rewriter.create(loc, funcOp.getSymNameAttr(), - TypeRange({resTy}), inputVals); - callOp->setAttr("isVolatile", isVolatileAttr); - rewriter.replaceOp(op, callOp); - return success(); -} - -LogicalResult StrideLoadConverter::matchAndRewrite( - triton::ascend::StrideLoadOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - auto funcName = generateUniqueFuncName(moduleOp, funcNameBase); - - auto src = adaptor.getSrc(); - auto offset = adaptor.getOffset(); - auto other = adaptor.getOther(); - auto strides = adaptor.getStride(); - auto numels = adaptor.getNumel(); - auto resTy = op.getResult().getType(); - - auto srcTy = dyn_cast(src.getType()); - if (!srcTy) { - return rewriter.notifyMatchFailure(op, "expected MemRefType for src"); - } - - SmallVector inputTypes({srcTy}); - inputTypes.push_back(offset.getType()); - inputTypes.push_back(other.getType()); - for (Value stride : strides) - inputTypes.push_back(stride.getType()); - for (Value numel : numels) - inputTypes.push_back(numel.getType()); - auto libFnType = rewriter.getFunctionType(inputTypes, {resTy}); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - - SmallVector inputVals({src}); - inputVals.push_back(offset); - inputVals.push_back(other); - inputVals.append(strides.begin(), strides.end()); - inputVals.append(numels.begin(), numels.end()); - - rewriter.setInsertionPoint(op); - auto callOp = rewriter.create(loc, funcOp.getSymNameAttr(), - TypeRange({resTy}), inputVals); - rewriter.replaceOp(op, callOp); - return success(); -} - -LogicalResult StrideStoreConverter::matchAndRewrite( - triton::ascend::StrideStoreOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - auto funcName = generateUniqueFuncName(moduleOp, funcNameBase); - - auto dst = adaptor.getDst(); - auto src = adaptor.getSrc(); - auto offset = adaptor.getOffset(); - auto strides = adaptor.getStride(); - auto numels = adaptor.getNumel(); - - auto dstTy = dyn_cast(dst.getType()); - if (!dstTy) { - return rewriter.notifyMatchFailure(op, "expected MemRefType for dst"); - } - - SmallVector inputTypes({dstTy, src.getType(), offset.getType()}); - for (Value stride : strides) - inputTypes.push_back(stride.getType()); - for (Value numel : numels) - inputTypes.push_back(numel.getType()); - auto libFnType = rewriter.getFunctionType(inputTypes, {}); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - - SmallVector inputVals({dst, src, offset}); - inputVals.append(strides.begin(), strides.end()); - inputVals.append(numels.begin(), numels.end()); - - rewriter.setInsertionPoint(op); - rewriter.create(loc, funcOp.getSymNameAttr(), TypeRange({}), - inputVals); - rewriter.eraseOp(op); - return success(); -} - -LogicalResult IndirectStoreConverter::matchAndRewrite( - triton::ascend::IndirectStoreOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - auto moduleOp = op->getParentOfType(); - rewriter.setInsertionPoint(moduleOp.getBody(), - std::prev(moduleOp.getBody()->end())); - - auto funcName = generateUniqueFuncName(moduleOp, funcNameBase); - - auto src = adaptor.getSrc(); - auto offsets = op.getOffsets(); - auto value = op.getValue(); - auto mask = op.getMask(); - - // Keep the indirect-store ABI consistent with indirect-load for a scalar - // address reconstructed by tt.int_to_ptr. Ordinary dynamic memrefs retain - // the existing direct path. - if (isIntToPtrBasedScalarPointer(op.getSrc())) { - llvm::SmallDenseMap known; - FailureOr materialized = - BlockDataParser::materializePointer(op.getSrc(), rewriter, known); - if (failed(materialized)) - return rewriter.notifyMatchFailure( - op, "unable to materialize int_to_ptr indirect-store source"); - src = *materialized; - } else if (!isa(src.getType())) { - llvm::SmallDenseMap known; - FailureOr materialized = - BlockDataParser::materializePointer(op.getSrc(), rewriter, known); - if (failed(materialized)) - return rewriter.notifyMatchFailure( - op, "unable to materialize indirect-store source as a memref"); - src = *materialized; - } - // convert !tt.ptr to memref - auto srcTy = dyn_cast(src.getType()); - if (!srcTy) { - return rewriter.notifyMatchFailure(op, "expected MemRefType for src"); - } - SmallVector inputTypes({srcTy, offsets.getType(), value.getType()}); - if (mask) - inputTypes.push_back(mask.getType()); - - auto libFnType = rewriter.getFunctionType(inputTypes, {}); - auto funcOp = rewriter.create(loc, funcName.str(), libFnType); - SymbolTable::setSymbolVisibility(funcOp, SymbolTable::Visibility::Private); - - rewriter.setInsertionPoint(op); - SmallVector inputVals({src, offsets, value}); - if (mask) - inputVals.push_back(mask); - rewriter.create(loc, funcOp.getSymNameAttr(), TypeRange({}), - inputVals); - rewriter.eraseOp(op); - return success(); -} - -IndexSelectSimdConverter::IndexSelectSimdConverter(MLIRContext *context) - : OpConversionPattern(context) {} - -LogicalResult IndexSelectSimdConverter::matchAndRewrite( - triton::ascend::IndexSelectSimdOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - - // Get converted operands - Value src = adaptor.getSrc(); - Value indexTensor = adaptor.getIndex(); - auto srcShapeVals = adaptor.getSrcShape(); - auto srcOffsetVals = adaptor.getSrcOffset(); - auto readShapeAttr = op.getReadShape(); - int32_t dim = op.getDim(); - - // Get result type - auto resultTensorType = cast(op.getResult().getType()); - static constexpr llvm::StringLiteral wasBoolToInt8AttrName = - "was_bool_to_int8"; - bool wasBoolToInt8 = op->hasAttr(wasBoolToInt8AttrName); - - auto elemType = resultTensorType.getElementType(); - auto resultShape = resultTensorType.getShape(); - - // Convert src (tt.ptr -> memref) to the correct memref shape - // src is now memref after type conversion, need to reinterpret to full - // shape - auto srcMemRefType = cast(src.getType()); - - // DenseI32ArrayAttr can be implicitly converted to ArrayRef - ArrayRef readShape = readShapeAttr; - - // Helper lambda to convert Value to Index type if needed - auto toIndexValue = [&](Value val) -> Value { - if (!val.getType().isIndex()) { - return rewriter.create(loc, rewriter.getIndexType(), - val); - } - return val; - }; - - // Build multi-dimensional memref type and determine static sizes - // Merge two passes: build fullSrcShape and staticSizes in one loop - SmallVector fullSrcShape; - SmallVector staticSizes; - SmallVector sizes; - - for (size_t i = 0; i < srcShapeVals.size(); ++i) { - bool isDynamicDim = (i == static_cast(dim) && readShape[i] == -1); - int64_t staticSize; - - if (isDynamicDim) { - // Dynamic dimension: readShape[i] == -1 indicates dynamic - staticSize = ShapedType::kDynamic; - fullSrcShape.push_back(ShapedType::kDynamic); - sizes.push_back(toIndexValue(srcShapeVals[i])); - } else if (auto constOp = - srcShapeVals[i].getDefiningOp()) { - // Static dimension: use constant value - staticSize = constOp.value(); - fullSrcShape.push_back(staticSize); - } else { - // Runtime value: must use dynamic - staticSize = ShapedType::kDynamic; - fullSrcShape.push_back(ShapedType::kDynamic); - sizes.push_back(toIndexValue(srcShapeVals[i])); - } - staticSizes.push_back(staticSize); - } - auto fullSrcMemRefType = MemRefType::get(fullSrcShape, elemType); - - // Build static offsets, sizes, and strides for ReinterpretCastOp - SmallVector offsets, strides; - SmallVector staticOffsets, staticStrides; - staticOffsets.push_back(0); // offsets are 0 - - // Calculate static strides: stride[i] = product of all dimensions after i - for (size_t i = 0; i < srcShapeVals.size(); ++i) { - int64_t staticStride = 1; - bool isDynamic = false; - - // Check if stride needs to be dynamic (any dimension after i is dynamic) - for (size_t j = i + 1; j < srcShapeVals.size(); ++j) { - if (staticSizes[j] == ShapedType::kDynamic) { - isDynamic = true; - break; - } - staticStride *= staticSizes[j]; - } - - if (isDynamic) { - staticStride = ShapedType::kDynamic; - // Compute stride dynamically: stride[i] = product of sizes after i - Value strideVal = rewriter.create(loc, 1); - for (size_t j = i + 1; j < srcShapeVals.size(); ++j) { - if (staticSizes[j] != ShapedType::kDynamic) { - strideVal = rewriter.create( - loc, strideVal, - rewriter.create(loc, staticSizes[j])); - } else { - // Dynamic dimension: use runtime value - strideVal = rewriter.create( - loc, strideVal, toIndexValue(srcShapeVals[j])); - } - } - strides.push_back(strideVal); - } - staticStrides.push_back(staticStride); - } - - auto srcMemRef = rewriter.create( - loc, fullSrcMemRefType, src, offsets, sizes, strides, staticOffsets, - staticSizes, staticStrides); - - // Allocate output buffer - auto resultMemRefType = MemRefType::get(resultShape, elemType); - auto outputBuffer = rewriter.create(loc, resultMemRefType); - - // Get indices tensor type for extracting - auto indicesTensorType = cast(indexTensor.getType()); - int64_t numIndices = indicesTensorType.getShape()[0]; - - // Create for loop - auto zeroIdx = rewriter.create(loc, 0); - auto numIndicesVal = rewriter.create(loc, numIndices); - auto stepOne = rewriter.create(loc, 1); - auto forOp = - rewriter.create(loc, zeroIdx, numIndicesVal, stepOne); - - // Mark as parallel loop - forOp->setAttr("hivm.parallel_loop", rewriter.getUnitAttr()); - - // Build loop body - Block *loopBody = forOp.getBody(); - auto savedInsertionPoint = rewriter.saveInsertionPoint(); - rewriter.setInsertionPointToStart(loopBody); - - // Remove the terminator temporarily - Operation *terminator = &loopBody->back(); - rewriter.setInsertionPoint(terminator); - - Value iv = forOp.getInductionVar(); - - // Extract index from indices tensor - Value selectedIdx = - rewriter.create(loc, indexTensor, ValueRange{iv}); - Value selectedIdxAsIndex = rewriter.create( - loc, rewriter.getIndexType(), selectedIdx); - - // Build source subview offsets/sizes/strides - SmallVector srcSubviewOffsets, srcSubviewSizes, - srcSubviewStrides; - - for (size_t i = 0; i < srcOffsetVals.size(); ++i) { - if (i == static_cast(dim)) { - // Use the selected index for this dimension - srcSubviewOffsets.push_back(selectedIdxAsIndex); - srcSubviewSizes.push_back(rewriter.getIndexAttr(1)); - } else { - // Use provided offset and read size for other dimensions - Value offsetVal = srcOffsetVals[i]; - if (!offsetVal.getType().isIndex()) { - offsetVal = rewriter.create( - loc, rewriter.getIndexType(), offsetVal); - } - srcSubviewOffsets.push_back(offsetVal); - srcSubviewSizes.push_back(rewriter.getIndexAttr(readShape[i])); - } - srcSubviewStrides.push_back(rewriter.getIndexAttr(1)); - } - - auto srcSubview = rewriter.create( - loc, srcMemRef, srcSubviewOffsets, srcSubviewSizes, srcSubviewStrides); - - // Build destination subview - SmallVector dstSubviewOffsets, dstSubviewSizes, - dstSubviewStrides; - for (size_t i = 0; i < resultShape.size(); ++i) { - if (i == static_cast(dim)) { - dstSubviewOffsets.push_back(iv); - dstSubviewSizes.push_back(rewriter.getIndexAttr(1)); - } else { - dstSubviewOffsets.push_back(rewriter.getIndexAttr(0)); - dstSubviewSizes.push_back(rewriter.getIndexAttr(readShape[i])); - } - dstSubviewStrides.push_back(rewriter.getIndexAttr(1)); - } - - auto dstSubview = rewriter.create( - loc, outputBuffer, dstSubviewOffsets, dstSubviewSizes, dstSubviewStrides); - - // Check if index_select is on the trailing axis (last dimension) - if (static_cast(dim) == fullSrcShape.size() - 1) { - // For index_select on the trailing axis, mark as discrete memory access - // This degrades to scalar read/write handling to avoid alignment issues - auto copyOp = rewriter.create(loc, srcSubview, dstSubview); - if (wasBoolToInt8) - copyOp->setAttr(wasBoolToInt8AttrName, rewriter.getBoolAttr(true)); - copyOp->setAttr(ConverterUtils::discreteAttrName, rewriter.getUnitAttr()); - } else { - // For index_select on non-trailing axes, add stride alignment annotation - // This tells the backend to handle address alignment for DMA operations - auto dstMarkOp = rewriter.create(loc, dstSubview); - dstMarkOp->setAttr( - "hfusion.stride_align_dims", - rewriter.getDenseI32ArrayAttr({static_cast(dim)})); - dstMarkOp->setAttr("hfusion.stride_align_value_in_byte", - rewriter.getDenseI32ArrayAttr({32})); - - // Copy from source to destination - auto copyOp = rewriter.create(loc, srcSubview, dstSubview); - if (wasBoolToInt8) - copyOp->setAttr(wasBoolToInt8AttrName, rewriter.getBoolAttr(true)); - } - - // Restore insertion point - rewriter.restoreInsertionPoint(savedInsertionPoint); - - // Convert memref to tensor - auto resultTensor = rewriter.create( - loc, resultTensorType, outputBuffer, true, true); - if (wasBoolToInt8) - resultTensor->setAttr(wasBoolToInt8AttrName, rewriter.getBoolAttr(true)); - - // Mark as index_select_simd - resultTensor->setAttr("index_select_simd", rewriter.getUnitAttr()); - - // Replace the original op - rewriter.replaceOp(op, resultTensor); - - return success(); -} - -LogicalResult -HistogramConverter::matchAndRewrite(triton::HistogramOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - auto loc = op.getLoc(); - Value input = adaptor.getSrc(); - Value mask = adaptor.getMask(); - auto resultType = dyn_cast(op.getResult().getType()); - if (!resultType || !resultType.hasStaticShape()) { - return rewriter.notifyMatchFailure(op, - "expected static shaped tensor result"); - } - - int64_t numBins = resultType.getDimSize(0); - - auto outputTensor = rewriter.create( - loc, resultType.getShape(), resultType.getElementType()); - - auto zeroVal = rewriter.create( - loc, rewriter.getIntegerAttr(resultType.getElementType(), 0)); - auto fillOp = rewriter.create(loc, ValueRange{zeroVal}, - ValueRange{outputTensor}); - - Value numBinsVal = rewriter.create(loc, numBins, 64); - - SmallVector inputs = {input, numBinsVal}; - if (mask) { - inputs.push_back(mask); - } - - auto customOp = rewriter.create( - loc, TypeRange{resultType}, "__builtin_histogram", ValueRange{inputs}, - ValueRange{fillOp.getResult(0)}, ValueRange{}); - - customOp->setAttr("symbol", rewriter.getStringAttr("__builtin_histogram")); - customOp->setAttr( - "hivm.tcore_type", - hivm::TCoreTypeAttr::get(rewriter.getContext(), hivm::TCoreType::VECTOR)); - customOp->setAttr("hivm.vf_mode", hivm::VFModeAttr::get(rewriter.getContext(), - hivm::VFMode::SIMT)); - customOp->setAttr("hivm.pipe", hivm::PipeAttr::get(rewriter.getContext(), - hivm::PIPE::PIPE_V)); - customOp->setAttr("gm_addr_args_indices", - DenseI32ArrayAttr::get(rewriter.getContext(), {})); - - rewriter.replaceOp(op, customOp->getResult(0)); - return success(); -} -// ===----------------------------------------------------------------------=== -// MapElementwiseDecomposeConverter -// ===----------------------------------------------------------------------=== - -// Promote a single scalar op to tensor-level. Returns the tensor result -// values after recording them in valueMap. -SmallVector MapElementwiseDecomposeConverter::promoteOp( - Operation *op, llvm::DenseMap &valueMap, OpBuilder &builder, - Location loc, ArrayRef tensorShape) const { - - // ---- helpers ---- - auto getTensorType = [&](Type scalarTy) -> RankedTensorType { - return RankedTensorType::get(tensorShape, scalarTy); - }; - - auto createInit = [&](Type scalarTy) -> Value { - return builder.create(loc, tensorShape, scalarTy); - }; - - auto getOperand = [&](Value v) -> Value { - auto it = valueMap.find(v); - if (it != valueMap.end()) - return it->second; - // External constant defined outside the region — promote on first use - if (auto cst = v.getDefiningOp()) { - auto scalarTy = cst.getResult().getType(); - Value init = createInit(scalarTy); - auto fill = builder.create(loc, cst.getResult(), init); - valueMap[v] = fill.getResult(0); - return fill.getResult(0); - } - llvm_unreachable("operand not in valueMap and not a constant"); - }; - - auto record = [&](Value oldVal, Value newVal) -> SmallVector { - valueMap[oldVal] = newVal; - return {newVal}; - }; - - // ---- binary arith: Op(lhs, rhs) → tensor Op(lhs, rhs) ---- -#define PROMOTE_BINARY(OpTy) \ - if (auto a = dyn_cast(op)) { \ - auto r = builder.create(loc, getOperand(a.getLhs()), \ - getOperand(a.getRhs())); \ - return record(a.getResult(), r.getResult()); \ - } - - PROMOTE_BINARY(arith::AddIOp) - PROMOTE_BINARY(arith::SubIOp) - PROMOTE_BINARY(arith::MulIOp) - PROMOTE_BINARY(arith::DivSIOp) - PROMOTE_BINARY(arith::DivUIOp) - PROMOTE_BINARY(arith::RemSIOp) - PROMOTE_BINARY(arith::RemUIOp) - PROMOTE_BINARY(arith::MaxSIOp) - PROMOTE_BINARY(arith::MinSIOp) - PROMOTE_BINARY(arith::MaxUIOp) - PROMOTE_BINARY(arith::MinUIOp) - PROMOTE_BINARY(arith::AddFOp) - PROMOTE_BINARY(arith::SubFOp) - PROMOTE_BINARY(arith::MulFOp) - PROMOTE_BINARY(arith::DivFOp) - PROMOTE_BINARY(arith::RemFOp) - PROMOTE_BINARY(arith::MaxNumFOp) - PROMOTE_BINARY(arith::MinNumFOp) - PROMOTE_BINARY(arith::MaximumFOp) - PROMOTE_BINARY(arith::MinimumFOp) - PROMOTE_BINARY(arith::AndIOp) - PROMOTE_BINARY(arith::OrIOp) - PROMOTE_BINARY(arith::XOrIOp) - PROMOTE_BINARY(arith::ShLIOp) - PROMOTE_BINARY(arith::ShRSIOp) - PROMOTE_BINARY(arith::ShRUIOp) -#undef PROMOTE_BINARY - - // ---- unary arith: Op(operand) → tensor Op(operand) ---- - if (auto a = dyn_cast(op)) { - auto r = builder.create(loc, getOperand(a.getOperand())); - return record(a.getResult(), r.getResult()); - } - - // ---- comparisons: preserve predicate ---- - if (auto a = dyn_cast(op)) { - auto r = builder.create( - loc, a.getPredicate(), getOperand(a.getLhs()), getOperand(a.getRhs())); - return record(a.getResult(), r.getResult()); - } - if (auto a = dyn_cast(op)) { - auto r = builder.create( - loc, a.getPredicate(), getOperand(a.getLhs()), getOperand(a.getRhs())); - return record(a.getResult(), r.getResult()); - } - - // ---- select ---- - if (auto a = dyn_cast(op)) { - auto r = builder.create(loc, getOperand(a.getCondition()), - getOperand(a.getTrueValue()), - getOperand(a.getFalseValue())); - return record(a.getResult(), r.getResult()); - } - - // ---- type casts: Op(in) → Op(in) with tensor result type ---- -#define PROMOTE_CAST(OpTy) \ - if (auto a = dyn_cast(op)) { \ - auto r = builder.create(loc, getTensorType(a.getOut().getType()), \ - getOperand(a.getIn())); \ - return record(a.getResult(), r.getResult()); \ - } - - PROMOTE_CAST(arith::SIToFPOp) - PROMOTE_CAST(arith::UIToFPOp) - PROMOTE_CAST(arith::FPToSIOp) - PROMOTE_CAST(arith::FPToUIOp) - PROMOTE_CAST(arith::ExtSIOp) - PROMOTE_CAST(arith::ExtUIOp) - PROMOTE_CAST(arith::ExtFOp) - PROMOTE_CAST(arith::TruncIOp) - PROMOTE_CAST(arith::TruncFOp) -#undef PROMOTE_CAST - - // ---- constant: splat via linalg.fill ---- - if (auto a = dyn_cast(op)) { - auto scalarTy = a.getResult().getType(); - Value init = createInit(scalarTy); - auto fill = builder.create(loc, a.getResult(), init); - return record(a.getResult(), fill.getResult(0)); - } - - // ---- math ops: directly on tensors (same as PROMOTE_BINARY) ---- -#define PROMOTE_MATH(OpTy) \ - if (auto a = dyn_cast(op)) { \ - auto r = builder.create(loc, getOperand(a.getOperand())); \ - return record(a.getResult(), r.getResult()); \ - } - - PROMOTE_MATH(math::ExpOp) - PROMOTE_MATH(math::Exp2Op) - PROMOTE_MATH(math::SqrtOp) - PROMOTE_MATH(math::RsqrtOp) - PROMOTE_MATH(math::LogOp) - PROMOTE_MATH(math::Log2Op) - PROMOTE_MATH(math::SinOp) - PROMOTE_MATH(math::CosOp) - PROMOTE_MATH(math::ErfOp) - PROMOTE_MATH(math::CeilOp) - PROMOTE_MATH(math::FloorOp) - PROMOTE_MATH(math::AbsFOp) - PROMOTE_MATH(math::AbsIOp) -#undef PROMOTE_MATH - - // ---- scf.if → arith.select ---- - if (auto a = dyn_cast(op)) { - Value cond = getOperand(a.getCondition()); - // Process then/else regions with inherited valueMap - SmallVector thenResults = promoteRegionBody( - a.getThenRegion(), valueMap, builder, loc, tensorShape); - SmallVector elseResults = promoteRegionBody( - a.getElseRegion(), valueMap, builder, loc, tensorShape); - for (unsigned i = 0; i < thenResults.size(); i++) { - auto sel = builder.create(loc, cond, thenResults[i], - elseResults[i]); - valueMap[a.getResult(i)] = sel.getResult(); - } - SmallVector results; - for (unsigned i = 0; i < thenResults.size(); i++) - results.push_back(valueMap[a.getResult(i)]); - return results; - } - - // ---- scf.for ---- - if (auto a = dyn_cast(op)) { - SmallVector tensorInits; - for (Value v : a.getInitArgs()) - tensorInits.push_back(getOperand(v)); - auto newFor = builder.create( - loc, a.getLowerBound(), a.getUpperBound(), a.getStep(), tensorInits, - [&](OpBuilder &b, Location l, Value iv, ValueRange iterArgs) { - llvm::DenseMap loopMap = valueMap; - loopMap[a.getInductionVar()] = iv; - for (auto [oldArg, newArg] : - llvm::zip(a.getRegionIterArgs(), iterArgs)) - loopMap[oldArg] = newArg; - SmallVector yielded = - promoteRegionBody(a.getRegion(), loopMap, b, l, tensorShape); - b.create(l, yielded); - }); - for (auto [oldR, newR] : llvm::zip(a.getResults(), newFor.getResults())) - valueMap[oldR] = newR; - SmallVector results; - for (auto r : newFor.getResults()) - results.push_back(r); - return results; - } - - if (isa(op)) { - op->emitError( - "scf.while is not supported inside map_elementwise " - "(scf.condition requires scalar i1, cannot promote to tensor)"); - llvm_unreachable(""); - } - op->emitError("MapElementwiseDecomposeConverter: unsupported op"); - llvm_unreachable("unhandled op in map_elementwise region"); - llvm_unreachable("unhandled op in map_elementwise region"); -} - -SmallVector MapElementwiseDecomposeConverter::promoteRegionBody( - Region ®ion, llvm::DenseMap &valueMap, OpBuilder &builder, - Location loc, ArrayRef tensorShape) const { - llvm::DenseMap localMap = valueMap; - Block &block = region.front(); - for (Operation &op : block.without_terminator()) - promoteOp(&op, localMap, builder, loc, tensorShape); - - // Helper to get-or-create a tensor version of any value (external - // constants are promoted on first use). - auto getTensorValue = [&](Value v) -> Value { - auto it = localMap.find(v); - if (it != localMap.end()) - return it->second; - // External constant — promote to tensor - if (auto cst = v.getDefiningOp()) { - auto scalarTy = cst.getResult().getType(); - Value init = builder.create(loc, tensorShape, scalarTy); - auto fill = builder.create(loc, cst.getResult(), init); - localMap[v] = fill.getResult(0); - return fill.getResult(0); - } - // External but not a constant — use as-is (e.g., SSA values from - // enclosing scope that are already tensors or valid in this context). - return v; - }; - - Operation *term = block.getTerminator(); - SmallVector results; - for (Value v : term->getOperands()) - results.push_back(getTensorValue(v)); - return results; -} - -LogicalResult MapElementwiseDecomposeConverter::matchAndRewrite( - triton::MapElementwiseOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const { - Location loc = op.getLoc(); - - auto tensorType = cast(op.getOperand(0).getType()); - auto shape = tensorType.getShape(); - int64_t rank = shape.size(); - int64_t pack = op.getPack(); - int64_t numInputs = op.getNumOperands(); - int64_t numOutputs = op.getNumResults(); - - assert(rank > 0 && "map_elementwise requires ranked tensor inputs"); - - Region ®ion = op.getRegion(); - assert(region.hasOneBlock() && "expected single-block region"); - Block &body = region.getBlocks().front(); - unsigned numRegionArgs = body.getNumArguments(); - - // Build initial ValueMap: block_arg[i] → input_tensor[i/pack] - llvm::DenseMap valueMap; - for (unsigned i = 0; i < numRegionArgs; i++) { - unsigned inputIdx = i / pack; - valueMap[body.getArgument(i)] = adaptor.getOperands()[inputIdx]; - } - - // Promote all non-terminator ops - for (Operation &innerOp : body.without_terminator()) - promoteOp(&innerOp, valueMap, rewriter, loc, shape); - - // Terminator → results - Operation *terminator = body.getTerminator(); - SmallVector results; - for (unsigned i = 0; i < numOutputs; i++) - results.push_back(valueMap[terminator->getOperand(i * pack)]); - - rewriter.replaceOp(op, results); - return success(); -} - -} // namespace TTOpConverters diff --git a/third_party/ascend/lib/TritonToLinalg/TritonToLinalgPass.cpp b/third_party/ascend/lib/TritonToLinalg/TritonToLinalgPass.cpp deleted file mode 100644 index 7ed436f1ce..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/TritonToLinalgPass.cpp +++ /dev/null @@ -1,2471 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -#include "TritonControlFlowOpt/ControlFlowRewrite.h" -#include "TritonToGraph/LayoutMemoryOptimization.h" -#include "TritonToLinalg/BlockPtrAnalysis.h" -#include "ascend/include/Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "ascend/include/TritonToLinalg/ArgMinMaxConverter.h" -#include "ascend/include/TritonToLinalg/CanonicalizeDebugLocationsPass.h" -#include "ascend/include/TritonToLinalg/DeduplicateDebugNopsPass.h" -#include "ascend/include/TritonToLinalg/DescriptorConverter.h" -#include "ascend/include/TritonToLinalg/DevicePrintOffsetRewrite.h" -#include "ascend/include/TritonToLinalg/FunctionConverter.h" -#include "ascend/include/TritonToLinalg/HoistBroadcast.h" -#include "ascend/include/TritonToLinalg/ImplicitPermute.h" -#include "ascend/include/TritonToLinalg/LoadStoreConverter.h" -#include "ascend/include/TritonToLinalg/MarkTensorKindPass.h" -#include "ascend/include/TritonToLinalg/TritonOpConverter.h" -#include "ascend/include/TritonToLinalg/TritonToLinalgPass.h" -#include "ascend/include/TritonToLinalg/UseAnalysis.h" -#include "ascend/include/TritonToStructured/CannonicalizerConverter.h" -#include "ascend/include/TritonToUnstructure/UnstructureConversionPass.h" -#include "ascend/include/Utils/InterleaveOptimization.h" - -#include "bishengir/Dialect/HFusion/IR/HFusion.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/Operation.h" -#include "mlir/IR/TypeRange.h" -#include "mlir/IR/TypeUtilities.h" -#include "mlir/IR/Types.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" - -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "bishengir/Dialect/Annotation/IR/Annotation.h" -#include "bishengir/Dialect/HIVM/IR/HIVM.h" -#include "bishengir/Dialect/Scope/IR/Scope.h" - -#include "mlir/Dialect/Affine/IR/AffineOps.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Linalg/Transforms/Transforms.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinTypeInterfaces.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/IRMapping.h" -#include "mlir/IR/Visitors.h" -#include "mlir/Pass/PassManager.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" -#include "mlir/Transforms/Passes.h" - -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/LogicalResult.h" - -#include -#include -#include -#include - -#define DEBUG_TYPE "triton-to-linalg" - -using namespace mlir; -using namespace triton; - -int nd2nzFlag = 0; -bool compileOn91095Flag = false; -bool existDotFlag = false; -triton::ascend::CompileMode compileModeFlag = triton::ascend::CompileMode::Simd; - -static bool containsTritonPointer(Type type) { - if (isa(type)) - return true; - auto shapedType = dyn_cast(type); - return shapedType && isa(shapedType.getElementType()); -} - -static bool hasPointerFreeTypes(TypeRange types) { - return llvm::none_of(types, containsTritonPointer); -} - -template static bool hasPointerFreeValues(RangeT values) { - return llvm::none_of(values, [](auto value) { - return containsTritonPointer(value.getType()); - }); -} - -// Checks every structural edge of a marker loop. Looking only at the loop -// operation's operands/results misses region arguments and terminator operands, -// which can otherwise leave a half-converted pointer boundary. -static bool hasPointerFreeControlFlowBoundary(LoopLikeOpInterface loopOp) { - Operation *loop = loopOp.getOperation(); - if (auto forOp = dyn_cast(loop)) { - return hasPointerFreeValues(forOp.getInitArgs()) && - hasPointerFreeValues(forOp.getRegionIterArgs()) && - hasPointerFreeValues(forOp.getYieldedValues()) && - hasPointerFreeTypes(forOp.getResultTypes()); - } - if (auto whileOp = dyn_cast(loop)) { - return hasPointerFreeValues(whileOp.getInits()) && - hasPointerFreeValues(whileOp.getBeforeArguments()) && - hasPointerFreeValues(whileOp.getConditionOp().getArgs()) && - hasPointerFreeValues(whileOp.getAfterArguments()) && - hasPointerFreeTypes(whileOp.getYieldOp().getOperandTypes()) && - hasPointerFreeTypes(whileOp.getResultTypes()); - } - return false; -} - -// Adds only the values that can produce one SSA result. Structured control -// flow is followed by result index so preserving one descriptor slot does not -// retain unrelated loop accumulators or sibling results. -static LogicalResult -appendProducerOperands(Value value, SmallVectorImpl &producerWorklist) { - if (auto argument = dyn_cast(value)) { - Operation *parent = argument.getOwner()->getParentOp(); - unsigned argumentIndex = argument.getArgNumber(); - if (auto forOp = dyn_cast_or_null(parent)) { - if (argumentIndex == 0) - return success(); - unsigned iterIndex = argumentIndex - 1; - if (iterIndex >= forOp.getInitArgs().size()) - return failure(); - producerWorklist.push_back(forOp.getInitArgs()[iterIndex]); - producerWorklist.push_back(forOp.getYieldedValues()[iterIndex]); - return success(); - } - if (auto whileOp = dyn_cast_or_null(parent)) { - if (argumentIndex >= whileOp.getInits().size()) - return failure(); - if (argument.getOwner() == whileOp.getBeforeBody()) { - producerWorklist.push_back(whileOp.getInits()[argumentIndex]); - producerWorklist.push_back( - whileOp.getYieldOp().getOperand(argumentIndex)); - return success(); - } - if (argument.getOwner() == whileOp.getAfterBody()) { - producerWorklist.push_back( - whileOp.getConditionOp().getArgs()[argumentIndex]); - return success(); - } - } - return success(); - } - - Operation *producer = value.getDefiningOp(); - if (!producer) - return success(); - - auto result = dyn_cast(value); - if (auto forOp = dyn_cast(producer)) { - if (!result || result.getResultNumber() >= forOp.getNumResults()) - return failure(); - unsigned index = result.getResultNumber(); - producerWorklist.push_back(forOp.getInitArgs()[index]); - producerWorklist.push_back(forOp.getYieldedValues()[index]); - return success(); - } - if (auto whileOp = dyn_cast(producer)) { - if (!result || result.getResultNumber() >= whileOp.getNumResults()) - return failure(); - unsigned index = result.getResultNumber(); - producerWorklist.push_back(whileOp.getInits()[index]); - producerWorklist.push_back(whileOp.getConditionOp().getArgs()[index]); - producerWorklist.push_back(whileOp.getYieldOp().getOperand(index)); - return success(); - } - if (auto ifOp = dyn_cast(producer)) { - if (!result || result.getResultNumber() >= ifOp.getNumResults() || - !ifOp.elseBlock()) - return failure(); - unsigned index = result.getResultNumber(); - producerWorklist.push_back(ifOp.thenYield().getOperand(index)); - producerWorklist.push_back(ifOp.elseYield().getOperand(index)); - return success(); - } - - producerWorklist.append(producer->operand_begin(), producer->operand_end()); - return success(); -} - -// Visits every structural value represented by one dynamic descriptor slot. -// The result type is the slot contract; init values, region arguments and -// terminator operands must all agree with it exactly. Keeping this traversal -// shared prevents entry validation and later MetaUse preservation from -// interpreting PointerDescriptorBoundary differently. -template -static LogicalResult -visitPointerDescriptorBoundarySlotValues(Operation *loop, int32_t slot, - VisitorT &&visit) { - if (!isa(loop) || slot < 0 || - static_cast(slot) >= loop->getNumResults()) - return failure(); - - Type expectedType = loop->getResult(slot).getType(); - if (containsTritonPointer(expectedType)) - return failure(); - - auto visitSlot = [&](auto values) { - if (static_cast(slot) >= values.size() || - values[slot].getType() != expectedType) - return failure(); - visit(values[slot]); - return success(); - }; - - visit(loop->getResult(slot)); - if (auto forOp = dyn_cast(loop)) { - return success(succeeded(visitSlot(forOp.getInitArgs())) && - succeeded(visitSlot(forOp.getRegionIterArgs())) && - succeeded(visitSlot(forOp.getYieldedValues()))); - } - - auto whileOp = cast(loop); - return success(succeeded(visitSlot(whileOp.getInits())) && - succeeded(visitSlot(whileOp.getBeforeArguments())) && - succeeded(visitSlot(whileOp.getConditionOp().getArgs())) && - succeeded(visitSlot(whileOp.getAfterArguments())) && - succeeded(visitSlot(whileOp.getYieldOp().getOperands()))); -} - -// Validates one CFO-owned loop boundary without changing the IR. An empty -// DenseI32ArrayAttr is valid and means every descriptor component is invariant -// and therefore absent from the loop signature. -static LogicalResult validatePointerDescriptorBoundary(Operation *loop) { - if (!isa(loop)) - return failure(); - - Attribute marker = loop->getAttr(controlflow::kPointerDescriptorBoundaryAttr); - if (!marker) - return failure(); - auto descriptorSlots = dyn_cast(marker); - if (!descriptorSlots || - !hasPointerFreeControlFlowBoundary(cast(loop))) - return failure(); - - llvm::SmallDenseSet seenSlots; - for (int32_t slot : descriptorSlots.asArrayRef()) { - if (!seenSlots.insert(slot).second || - failed( - visitPointerDescriptorBoundarySlotValues(loop, slot, [](Value) {}))) - return failure(); - } - return success(); -} - -// Validates a complete descriptor reconstruction root without changing the -// IR. Rebuild roots need operands even when the corresponding loop marker is -// empty because those operands preserve the invariant descriptor components. -static LogicalResult validatePointerDescriptorRebuild(Operation *op) { - Attribute marker = op->getAttr(controlflow::kPointerDescriptorRebuildAttr); - if (!marker || !isa(marker) || op->getNumOperands() == 0 || - !llvm::any_of(op->getResultTypes(), containsTritonPointer)) - return failure(); - - Attribute offsetForm = - op->getAttr(controlflow::kPointerDescriptorOffsetFormAttr); - if (!offsetForm) - return success(); - auto form = dyn_cast(offsetForm); - auto addPtr = dyn_cast(op); - if (!form || form.getValue() != controlflow::kStrided1DOffsetForm || !addPtr) - return failure(); - - auto resultType = dyn_cast(addPtr.getType()); - auto offsetType = dyn_cast(addPtr.getOffset().getType()); - auto offsetElementType = - offsetType ? dyn_cast(offsetType.getElementType()) - : IntegerType(); - auto baseSplat = addPtr.getPtr().getDefiningOp(); - return success(resultType && resultType.getRank() == 1 && offsetType && - isa(resultType.getElementType()) && - offsetElementType && offsetElementType.getWidth() <= 64 && - resultType.getShape() == offsetType.getShape() && - resultType.getEncoding() == offsetType.getEncoding() && - baseSplat && - isa(baseSplat.getSrc().getType())); -} - -// Returns true when a pointer consumer is reached through ordinary addptr -// operations from a CFO descriptor reconstruction root. The walk is purposely -// limited to addptr: following arbitrary pointer-producing operations would -// make the handoff contract retain computations it does not own. -static bool hasPointerDescriptorRebuildProvenance(Value pointer) { - llvm::DenseSet visited; - while (pointer && visited.insert(pointer).second) { - Operation *producer = pointer.getDefiningOp(); - if (!producer) - return false; - if (producer->hasAttr(controlflow::kPointerDescriptorRebuildAttr)) - return true; - auto addPtr = dyn_cast(producer); - if (!addPtr) - return false; - pointer = addPtr.getPtr(); - } - return false; -} - -// Checks the complete CFO-to-TritonToLinalg handoff before any rewrite can -// fold away a malformed marker. This function is deliberately read-only so it -// can run at pass entry, before UseAnalysis has produced MetaUse attributes. -static LogicalResult -validatePointerDescriptorHandoffMetadata(ModuleOp moduleOp) { - bool valid = true; - moduleOp.walk([&](Operation *op) { - if (op->hasAttr(controlflow::kPointerDescriptorBoundaryAttr) && - failed(validatePointerDescriptorBoundary(op))) - valid = false; - if (op->hasAttr(controlflow::kPointerDescriptorRebuildAttr) && - failed(validatePointerDescriptorRebuild(op))) - valid = false; - if (op->hasAttr(controlflow::kPointerDescriptorOffsetFormAttr) && - !op->hasAttr(controlflow::kPointerDescriptorRebuildAttr)) - valid = false; - }); - return success(valid); -} - -// PointerDescriptorBoundary identifies the dynamic loop slots used to rebuild -// pointers. PointerDescriptorRebuild operands are the complete descriptor -// roots, including invariant components omitted from an empty or minimal slot -// list. After UseAnalysis, preserve exactly those producer chains so -// MetaUseEraser cannot discard values required by conversion. -static LogicalResult preservePointerDescriptorComputations(ModuleOp moduleOp) { - if (failed(validatePointerDescriptorHandoffMetadata(moduleOp))) - return failure(); - - bool valid = true; - SmallVector producerWorklist; - moduleOp.walk([&](Operation *loop) { - Attribute marker = - loop->getAttr(controlflow::kPointerDescriptorBoundaryAttr); - if (!marker) - return; - auto descriptorSlots = cast(marker); - - loop->removeAttr("MetaUse"); - for (int32_t slot : descriptorSlots.asArrayRef()) { - if (failed(visitPointerDescriptorBoundarySlotValues( - loop, slot, - [&](Value value) { producerWorklist.push_back(value); }))) - valid = false; - } - }); - - moduleOp.walk([&](Operation *op) { - if (!op->hasAttr(controlflow::kPointerDescriptorRebuildAttr)) - return; - - // The rebuild operation itself and each of its operands are exact live - // conversion roots. This includes an invariant opaque tensor base as well - // as a scalar address behind int_to_ptr+splat. UseAnalysis has already - // propagated MetaUse by this point; preserving only the offset would leave - // a live ptr operand's producer eligible for erasure. This operand-local - // closure is still narrower than retaining all loop operands/terminators. - op->removeAttr("MetaUse"); - producerWorklist.append(op->operand_begin(), op->operand_end()); - }); - - // UseAnalysis classifies load/store masks and load fallback values as - // pointer metadata. A descriptor rebuild, however, is converted to a memref - // while its consumer still needs those tensor values to form subviews and - // padding. Preserve these exact consumer operands when the pointer is owned - // by the CFO handoff; otherwise MetaUseEraser can leave an unconvertible - // memref-to-pointer materialization at the live memory operation. - moduleOp.walk([&](triton::LoadOp load) { - if (!hasPointerDescriptorRebuildProvenance(load.getPtr())) - return; - if (Value mask = load.getMask()) - producerWorklist.push_back(mask); - if (Value other = load.getOther()) - producerWorklist.push_back(other); - }); - moduleOp.walk([&](triton::StoreOp store) { - if (hasPointerDescriptorRebuildProvenance(store.getPtr()) && - store.getMask()) - producerWorklist.push_back(store.getMask()); - }); - moduleOp.walk([&](triton::AtomicRMWOp atomic) { - if (hasPointerDescriptorRebuildProvenance(atomic.getPtr()) && - atomic.getMask()) - producerWorklist.push_back(atomic.getMask()); - }); - - llvm::DenseSet visitedValues; - while (!producerWorklist.empty()) { - Value value = producerWorklist.pop_back_val(); - if (!value || !visitedValues.insert(value).second) - continue; - if (Operation *producer = value.getDefiningOp()) - producer->removeAttr("MetaUse"); - if (failed(appendProducerOperands(value, producerWorklist))) - valid = false; - } - return success(valid); -} - -// The generic canonicalizer may remove forwarded scf.for/scf.while iter-args, -// which changes a marker loop's positional signature without updating its -// external descriptor-slot metadata. Marker-bearing modules therefore use this -// narrow pre-clean: CSE remains enabled, and only constant scf.if regions are -// inlined so use analysis does not visit unreachable branches. -class FoldConstantIfBeforeUseAnalysis final - : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(scf::IfOp ifOp, - PatternRewriter &rewriter) const override { - auto condition = ifOp.getCondition().getDefiningOp(); - if (!condition) - return failure(); - auto conditionValue = dyn_cast(condition.getValue()); - if (!conditionValue || !conditionValue.getType().isInteger(1)) - return failure(); - - Block *selectedBlock = - conditionValue.getValue().isOne() ? ifOp.thenBlock() : ifOp.elseBlock(); - if (!selectedBlock) { - if (ifOp.getNumResults() != 0) - return failure(); - rewriter.eraseOp(ifOp); - return success(); - } - - auto yield = cast(selectedBlock->getTerminator()); - SmallVector replacements(yield.getOperands()); - rewriter.inlineBlockBefore(selectedBlock, ifOp->getBlock(), - ifOp->getIterator()); - rewriter.eraseOp(yield); - rewriter.replaceOp(ifOp, replacements); - return success(); - } -}; - -// Inspect Operation directly so discovery cannot silently miss an SCF marker -// or an if-only rebuild root. Any surviving handoff attribute disables generic -// pre-clean canonicalization because its validated slot positions and rebuild -// operands must remain stable until MetaUse preservation. -static bool containsPointerDescriptorHandoff(ModuleOp moduleOp) { - bool found = false; - moduleOp.walk([&](Operation *op) { - if (op->hasAttr(controlflow::kPointerDescriptorBoundaryAttr) || - op->hasAttr(controlflow::kPointerDescriptorRebuildAttr) || - op->hasAttr(controlflow::kPointerDescriptorOffsetFormAttr)) - found = true; - }); - return found; -} - -// A converted make_range may leave a tensor-valued SCF state behind even -// though no loop result or loop-body operation observes it. Keep this cleanup -// deliberately narrow: it recognizes only statically shaped integer tensors -// whose entire loop-carried use chain consists of uniform integer updates. -static bool isDeadRangeUpdate(Value value, scf::ForOp loop, unsigned slot, - llvm::SmallPtrSetImpl &visited) { - if (!visited.insert(value).second) - return false; - - auto yield = cast(loop.getBody()->getTerminator()); - bool sawYield = false; - for (OpOperand &use : value.getUses()) { - Operation *user = use.getOwner(); - if (user == yield) { - if (use.getOperandNumber() != slot) - return false; - sawYield = true; - continue; - } - - Value updateResult; - Value lhs; - Value rhs; - if (auto add = dyn_cast(user)) { - updateResult = add.getResult(); - lhs = add.getLhs(); - rhs = add.getRhs(); - } else if (auto sub = dyn_cast(user)) { - updateResult = sub.getResult(); - lhs = sub.getLhs(); - rhs = sub.getRhs(); - } else { - return false; - } - if (updateResult.getType() != value.getType()) - return false; - - bool usesValueAsLhs = lhs == value; - bool usesValueAsRhs = rhs == value; - if (usesValueAsLhs == usesValueAsRhs) - return false; - Value delta = usesValueAsLhs ? rhs : lhs; - auto deltaType = dyn_cast(delta.getType()); - if (!deltaType || deltaType != value.getType()) - return false; - - Operation *deltaProducer = delta.getDefiningOp(); - bool isUniformDelta = false; - if (auto constant = dyn_cast_or_null(deltaProducer)) { - if (auto dense = dyn_cast(constant.getValue())) - isUniformDelta = dense.isSplat(); - } - if (auto fill = dyn_cast_or_null(deltaProducer)) - isUniformDelta = fill.getInputs().size() == 1 && - fill.getOutputs().size() == 1 && - fill.getOutputs().front().getType() == value.getType(); - if (!isUniformDelta) - return false; - - if (!isDeadRangeUpdate(updateResult, loop, slot, visited)) - return false; - } - return sawYield; -} - -static bool isDeadRangeCarrier(scf::ForOp loop, unsigned slot) { - if (slot >= loop.getInitArgs().size() || - slot >= loop.getRegionIterArgs().size() || - slot >= loop.getResults().size() || - slot >= loop.getYieldedValues().size() || - !loop.getResult(slot).use_empty()) - return false; - - auto type = dyn_cast(loop.getInitArgs()[slot].getType()); - auto iterType = - dyn_cast(loop.getRegionIterArgs()[slot].getType()); - if (!type || !iterType || type != iterType || !type.hasStaticShape() || - !isa(type.getElementType())) - return false; - - Operation *producer = loop.getInitArgs()[slot].getDefiningOp(); - if (!producer || - (!isa(producer) && - !producer->hasAttr("tt.from_make_range") && - !producer->hasAttr("tt.make_range_offset") && - !producer->hasAttr("tt.make_range_size"))) - return false; - - llvm::SmallPtrSet visited; - return isDeadRangeUpdate(loop.getRegionIterArgs()[slot], loop, slot, visited); -} - -// Remove only fully dead range-like SCF state. Body arguments and yield -// operands are removed first, then the loop is rebuilt with surviving values. -static void eraseDeadRangeCarriers(ModuleOp moduleOp) { - SmallVector loops; - moduleOp.walk([&](scf::ForOp loop) { loops.push_back(loop); }); - - for (scf::ForOp loop : loops) { - if (!loop || loop->getParentOp() == nullptr) - continue; - llvm::BitVector dead(loop.getInitArgs().size()); - for (unsigned i = 0; i < dead.size(); ++i) { - if (isDeadRangeCarrier(loop, i)) - dead.set(i); - } - if (dead.none()) - continue; - - auto yield = cast(loop.getBody()->getTerminator()); - yield->eraseOperands(dead); - loop.getBody()->eraseArguments([&](BlockArgument arg) { - unsigned argNumber = arg.getArgNumber(); - return argNumber != 0 && dead.test(argNumber - 1); - }); - - llvm::BitVector operandIndices(loop->getNumOperands()); - for (auto [i, init] : llvm::enumerate(loop.getInitArgsMutable())) { - if (dead.test(i)) - operandIndices.set(init.getOperandNumber()); - } - loop->eraseOperands(operandIndices); - - OperationState state(loop.getLoc(), loop->getName(), loop->getOperands(), - loop.getInitArgs().getTypes(), loop->getAttrs()); - state.addRegion()->takeBody(loop.getBodyRegion()); - OpBuilder builder(loop); - auto newLoop = cast(builder.create(state)); - - unsigned newResultIndex = 0; - for (auto [i, result] : llvm::enumerate(loop.getResults())) { - if (dead.test(i)) { - assert(result.use_empty() && "dead range result still has uses"); - continue; - } - result.replaceAllUsesWith(newLoop.getResult(newResultIndex++)); - } - loop.erase(); - } -} - -static LogicalResult preCleanBeforeUseAnalysis(ModuleOp moduleOp) { - bool hasPointerDescriptorHandoff = containsPointerDescriptorHandoff(moduleOp); - - PassManager csePipeline(moduleOp.getContext(), moduleOp.getOperationName()); - csePipeline.addPass(createCSEPass()); - if (!hasPointerDescriptorHandoff) - csePipeline.addPass(createCanonicalizerPass()); - if (failed(csePipeline.run(moduleOp))) - return failure(); - - if (!hasPointerDescriptorHandoff) - return success(); - RewritePatternSet patterns(moduleOp.getContext()); - patterns.add(moduleOp.getContext()); - return applyPatternsGreedily(moduleOp, std::move(patterns)); -} - -// Recomputes the result layouts of subviews whose source descriptor has been -// rebased. A memref.subview result layout is derived from both its mixed -// offsets/strides and its source layout. For example, changing the source from -// `memref<32xf32, strided<[1], offset: ?>>` to the equivalent rebased -// `memref<32xf32, strided<[1]>>` changes a zero-offset subview result from a -// dynamic offset to offset zero. Merely changing the source SSA value leaves -// the old result type behind and makes SubViewOp verification fail. -// -// Subviews may be chained, so every updated result becomes a new worklist -// source. Rank-reduced subviews retain their existing result shape while their -// layout is inferred again from the rebased source. -static LogicalResult propagateRebasedSubviewTypes(Value rebasedSource, - IRRewriter &rewriter) { - SmallVector sources{rebasedSource}; - llvm::SmallPtrSet visited; - - while (!sources.empty()) { - Value source = sources.pop_back_val(); - for (Operation *user : source.getUsers()) { - auto subview = dyn_cast(user); - if (!subview || !visited.insert(user).second) - continue; - - auto sourceType = dyn_cast(subview.getSource().getType()); - auto oldResultType = dyn_cast(subview.getResult().getType()); - if (!sourceType || !oldResultType) - return failure(); - - Type inferredType; - if (sourceType.getRank() == oldResultType.getRank()) { - inferredType = memref::SubViewOp::inferResultType( - sourceType, subview.getMixedOffsets(), subview.getMixedSizes(), - subview.getMixedStrides()); - } else { - inferredType = memref::SubViewOp::inferRankReducedResultType( - oldResultType.getShape(), sourceType, subview.getMixedOffsets(), - subview.getMixedSizes(), subview.getMixedStrides()); - } - - auto inferredMemRefType = dyn_cast(inferredType); - if (!inferredMemRefType) - return failure(); - if (inferredMemRefType != oldResultType) { - rewriter.modifyOpInPlace( - subview, [&] { subview.getResult().setType(inferredMemRefType); }); - } - sources.push_back(subview.getResult()); - } - } - return success(); -} - -// Returns true when rebasing a descriptor would change a type owned by a -// different operation. A subview chain ending in direct memref loads/stores is -// local to this rewrite. Calls, returns, SCF/CFG boundaries, and every other -// user retain their existing descriptor layout and therefore stop rebasing. -static bool reachesLayoutSensitiveBoundary(Value root) { - SmallVector worklist{root}; - llvm::DenseSet visited; - while (!worklist.empty()) { - Value value = worklist.pop_back_val(); - if (!visited.insert(value).second) - continue; - for (Operation *user : value.getUsers()) { - if (auto subview = dyn_cast(user)) { - if (subview.getSource() == value) { - worklist.push_back(subview.getResult()); - continue; - } - } - if (auto load = dyn_cast(user)) { - if (load.getMemRef() == value) - continue; - } - if (auto store = dyn_cast(user)) { - if (store.getMemRef() == value) - continue; - } - return true; - } - } - return false; -} - -// Convert structured custom ops after operand type converted, -// for example tt.ptr converted to memref. -template -class StructuredCustomOpConverter : public OpConversionPattern { -public: - using OpConversionPattern::OpConversionPattern; - - LogicalResult - matchAndRewrite(CustomOpT op, typename CustomOpT::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - BlockDataParser::rewriteStructuredCustomOp(op, adaptor, rewriter); - return success(); - } -}; - -// Collect the ops in a scan body that actually contribute to the yielded -// result: compute the backward slice from the terminator operands, then drop -// pure type-cast ops. Mirrors ReductionOpBaseConverter::getRealReductionOps so -// that auxiliary ops (e.g. the overflow asserts injected by TRITON_DEBUG) -// never feed the yielded result and are ignored here as well. -static llvm::SmallVector getRealScanBodyOps(triton::ScanOp op) { - Block *body = op.getBody(); - Operation *terminator = body->getTerminator(); - - llvm::DenseSet liveOps; - llvm::SmallVector worklist(terminator->getOperands()); - while (!worklist.empty()) { - Value val = worklist.pop_back_val(); - if (auto *defOp = val.getDefiningOp()) { - if (defOp->getBlock() == body && liveOps.insert(defOp).second) { - for (auto operand : defOp->getOperands()) - worklist.push_back(operand); - } - } - } - - llvm::SmallVector realOps; - for (Operation &bodyOp : body->without_terminator()) { - if (!liveOps.contains(&bodyOp)) - continue; - if (isa(&bodyOp)) - continue; - realOps.push_back(&bodyOp); - } - return realOps; -} - -// A tt.scan that is (1) a plain cumsum (combine body is a single add, matching -// ScanConverter's triton_cumsum selection) and (2) collapses to a 1-D scan -// after backend lowering, i.e. every dim except the scan axis has extent 1 -// (e.g. [1,1,128,1] with axis=2). Only this case is routed to the SIMT -// (Sklansky) cumsum template; cumprod / generic scans and multi-dim cumsum stay -// on SIMD. -static bool isSimt1DCumsum(triton::ScanOp op) { - // (1) Must be a single-add combine body (only ops that feed the yielded - // result count, mirroring getRealReductionOps). - llvm::SmallVector realOps = getRealScanBodyOps(op); - if (realOps.size() != 1 || - !isa(realOps.front())) - return false; - - // (2) Must be the 1-D scenario: all non-scan dims are unit-sized. - auto srcTy = dyn_cast(op.getOperand(0).getType()); - if (!srcTy || !srcTy.hasRank()) - return false; - int64_t axis = op.getAxis(); - ArrayRef shape = srcTy.getShape(); - if (axis < 0 || axis >= static_cast(shape.size())) - return false; - for (int64_t i = 0; i < static_cast(shape.size()); ++i) { - if (i != axis && shape[i] != 1) - return false; - } - return true; -} - -static bool isCustomOpOperandTypesLegal(TypeRange types) { - return llvm::all_of(types, [](Type t) { - if (isa(t)) { - return false; - } - if (auto shapedType = dyn_cast(t)) { - return !isa(shapedType.getElementType()); - } - return true; - }); -} - -static bool isSIMTOp(Operation *op) { - if (auto custom_op = dyn_cast(op)) { - return custom_op.getCoreType() == hivm::TCoreType::VECTOR && - custom_op.getVFMode() == hivm::VFMode::SIMT; - } - - if (isa(op) && compileOn91095Flag) { - return true; - } - - if (isa(op) && compileOn91095Flag) { - return true; - } - - // tt.scan: only a 1-D cumsum is treated as a SIMT op (drives the kernel - // parallel_mode -> mix_simd_simt -> enable_simt). Everything else stays SIMD. - if (compileOn91095Flag) { - if (auto scan = dyn_cast(op)) { - return isSimt1DCumsum(scan); - } - } - - // math.sin / math.cos on f16/f32 inputs: downstream (Ascend 950 RegBase - // normalize, enable-high-precision defaults to true) rewrites them into a - // Payne-Hanek range reduction that looks up a 320xi32 2/pi limbs table with - // two hfusion.gather ops per collapsed region. Match that scenario here and - // route it to the SIMT template so the table gathers run in SIMT. - if (compileOn91095Flag && (isa(op) || isa(op))) { - Type inElem = getElementTypeOrSelf(op->getOperand(0).getType()); - if (inElem.isF16() || inElem.isF32()) { - return true; - } - } - - return isa(op); -} - -TritonTypeConverter::TritonTypeConverter() { - addConversion([](Type type) { return type; }); - - addConversion([](triton::PointerType ptrType) { - Type elem = ptrType.getPointeeType(); - // Handling special case: ptr -> memref - if (auto it = dyn_cast(elem); it && it.getWidth() == 1) { - elem = IntegerType::get(ptrType.getContext(), 8); - LLVM_DEBUG({ - llvm::dbgs() << "[TritonTypeConverter] Normalize i1 pointer to i8 " - "memref. elemType=" - << elem << "\n"; - }); - } - return MemRefType::get({ShapedType::kDynamic}, elem); - }); - - addConversion([](TensorType tensorType) -> Type { - auto elemType = tensorType.getElementType(); - if (auto ptrType = dyn_cast(elemType)) { - elemType = ptrType.getPointeeType(); - } - // Handling special case: tensor -> memref - if (auto it = dyn_cast(elemType); it && it.getWidth() == 1) { - elemType = IntegerType::get(tensorType.getContext(), 8); - LLVM_DEBUG({ - llvm::dbgs() << "[TritonTypeConverter] Normalize i1 tensor to i8 " - "memref. elemType=" - << elemType << "\n"; - }); - } - return MemRefType::get(tensorType.getShape(), elemType); - }); - - // A pointer-descriptor boundary can intentionally remain a legal SCF op - // carrying pointer-free tensor values while function signature conversion - // changes the corresponding argument to a memref. Materialize only the - // canonical numerical memref-to-ranked-tensor pair produced by the - // conversion above. The exact type check deliberately excludes pointer - // tensors, encoded tensors, i1-to-i8 normalization, non-identity layouts, - // and non-default memory spaces, so this does not become a general SCF - // type-conversion path. - addSourceMaterialization([](OpBuilder &builder, RankedTensorType resultType, - ValueRange inputs, Location loc) -> Value { - if (inputs.size() != 1 || resultType.getEncoding() || - isa(resultType.getElementType())) - return nullptr; - - auto inputType = dyn_cast(inputs.front().getType()); - if (!inputType || !inputType.getLayout().isIdentity() || - inputType.getMemorySpace()) - return nullptr; - - auto expectedInputType = - MemRefType::get(resultType.getShape(), resultType.getElementType()); - if (inputType != expectedInputType) - return nullptr; - - return builder.create( - loc, resultType, inputs.front(), /*restrict=*/false, - /*writable=*/false); - }); -} - -void TritonToLinalgPass::addProgramInfo(triton::FuncOp func, - bool globalKernel) { - OpBuilder b(func); - - auto origFuncType = func.getFunctionType(); - auto origInputTypes = origFuncType.getInputs(); - SmallVector newInputTypes(origInputTypes); - newInputTypes.append(TRITON_PROGRAM_INFO_ARG_COUNT, b.getI32Type()); - - auto newFuncType = - b.getFunctionType(newInputTypes, origFuncType.getResults()); - - func.setFunctionType(newFuncType); - - // If argument attributes exist, extend attribute list. - if (func.getAllArgAttrs()) { - SmallVector newArgAttrs; - func.getAllArgAttrs(newArgAttrs); - newArgAttrs.append(TRITON_PROGRAM_INFO_ARG_COUNT, DictionaryAttr()); - func.setAllArgAttrs(newArgAttrs); - } - - // Append the arguments to the entry block. - for (unsigned i = 0; i < TRITON_PROGRAM_INFO_ARG_COUNT; i++) { - func.getBody().front().addArgument(b.getI32Type(), func.getLoc()); - } - - if (globalKernel) { - func->setAttr(globalKernelAttr, b.getStringAttr("")); - } else { - func->setAttr(globalKernelAttr, b.getStringAttr("local")); - } -} - -LogicalResult -TritonToLinalgPass::convertMultipleBlockControlFlow(Operation *funcOp, - OpBuilder &builder) { - if (!isa(funcOp)) { - funcOp->emitError( - "convertMultipleBlockControlFlow can only process func::FuncOp!"); - return failure(); - } - - SmallVector candidate; - SmallVector eraseBlocks; - for (Block &block : dyn_cast(funcOp).getBody()) { - auto curTerminator = block.getTerminator(); - if (isa(curTerminator)) { - candidate.push_back(curTerminator); - } else if (isa(curTerminator)) { - if (candidate.empty()) { - curTerminator->emitError( - "funcOp has more than one Block but got an early 'tt.return' Op."); - return failure(); - } - } else if (!isa(curTerminator)) { - funcOp->emitError( - "funcOp has more than one Block but found unsupported Terminator: ") - << *curTerminator; - return failure(); - } - - if (!block.isEntryBlock()) - eraseBlocks.push_back(&block); - } - - LLVM_DEBUG({ - llvm::dbgs() << "Found " << candidate.size() - << " candidate cond_branch operations to convert.\n"; - }); - - if (candidate.empty()) { - funcOp->emitError("funcOp has more than one Block but no candidate " - "Terminator was found!"); - return failure(); - } - - llvm::BitVector visitFlag(candidate.size(), false); - - // Recursive function to convert all cf::CondBranchOp to scf::IfOp - std::function convertToSCF = - [&](Operation *op, Operation *insertPosOp) -> void { - auto condBranchOp = dyn_cast_if_present(op); - auto iter = llvm::find(candidate, condBranchOp); - if (!(condBranchOp && iter != candidate.end())) { - op->emitError( - "convertToSCF must process with condBranchOp in candidates!"); - return; - } - visitFlag.set(iter - candidate.begin()); - - OpBuilder::InsertionGuard guard(builder); - builder.setInsertionPointAfter(insertPosOp); - - // Well, here force to destory original control flow - builder.create( - condBranchOp->getLoc(), condBranchOp.getCondition(), - /*thenBuilder=*/ - [&](OpBuilder &builder, Location loc) { - SmallVector movedOps = llvm::map_to_vector( - condBranchOp.getTrueDest()->without_terminator(), - [](Operation &op) { return &op; }); - for (auto *innerOp : movedOps) { - innerOp->moveBefore(builder.getInsertionBlock(), - builder.getInsertionPoint()); - } - - auto blockTerm = condBranchOp.getTrueDest()->getTerminator(); - if (auto nextCond = dyn_cast(blockTerm)) { - if (movedOps.empty()) { - blockTerm->emitError("movedOps can not be empty before entering " - "convertToSCF (then)!"); - return; - } - convertToSCF(nextCond, movedOps.back()); - } else if (!isa(blockTerm)) { - blockTerm->emitError( - "Unsupported terminator in then branch after structuring"); - } - - builder.create(loc); - }, - /*elseBuilder=*/ - [&](OpBuilder &builder, Location loc) { - SmallVector movedOps = llvm::map_to_vector( - condBranchOp.getFalseDest()->without_terminator(), - [](Operation &op) { return &op; }); - for (auto *innerOp : movedOps) { - innerOp->moveBefore(builder.getInsertionBlock(), - builder.getInsertionPoint()); - } - - auto blockTerm = condBranchOp.getFalseDest()->getTerminator(); - if (auto nextCond = dyn_cast(blockTerm)) { - if (movedOps.empty()) { - blockTerm->emitError("movedOps can not be empty before entering " - "convertToSCF (else)!"); - return; - } - convertToSCF(nextCond, movedOps.back()); - } else if (!isa(blockTerm)) { - blockTerm->emitError( - "Unsupported terminator in else branch after structuring"); - } - builder.create(loc); - }); - }; - - Block::iterator insertOp(candidate.front()); - if (insertOp == candidate.front()->getBlock()->begin()) { - // if the first operation is a cond_branch, we need to insert before it - convertToSCF(candidate.front(), candidate.front()); - } else { - --insertOp; - convertToSCF(candidate.front(), &(*insertOp)); - } - - if (!visitFlag.all()) { - funcOp->emitError("Not all cf.cond_br converted!"); - return failure(); - } - - OpBuilder::InsertionGuard guard(builder); - builder.setInsertionPoint(candidate.front()); - builder.create(candidate.front()->getLoc()); - - for (Operation *eachTerm : candidate) - eachTerm->erase(); - for (Block *block : llvm::reverse(eraseBlocks)) - block->erase(); - - return success(); -} - -void TritonToLinalgPass::convertTTFunc(triton::FuncOp func, const bool existDot, - const bool existSIMTOp) { - OpBuilder builder(func); - - auto name = func.getName(); - auto type = func.getFunctionType(); - - SmallVector argAttrs, resAttrs; - func.getAllArgAttrs(argAttrs); - func.getAllResultAttrs(resAttrs); - - // Special handling for bit-casted tt.ptr arguments - SmallVector inputTypes{type.getInputs()}; - SmallVector retTypes{type.getResults()}; - if (func.getSymVisibility() == "public" && !func.isDeclaration()) { - for (size_t i = 0; i < func.getNumArguments(); ++i) { - auto arg = func.getArgument(i); - // Special method for i1 arg - if (!isa(arg.getType()) || - dyn_cast(arg.getType()).getElementTypeBitWidth() != - 1) { - continue; - } - - SmallVector argVaildUser{arg.getUsers()}; - llvm::erase_if(argVaildUser, [](Operation *op) -> bool { - return isOpTriviallyDead(op); - }); - - if (!argVaildUser.empty()) { - LLVM_DEBUG({ - auto &os = llvm::dbgs(); - os << arg << " has users:\n"; - int cnt = 0; - for (auto it : argVaildUser) { - os << "users[" << cnt++ << "] = " << *it; - } - }); - if (llvm::all_of(argVaildUser, [](Operation *userOp) { - return isa(userOp); - })) { - auto castOp = cast(*argVaildUser.begin()); - if (castOp.getInputs().size() == 1 && - castOp.getOutputs().size() == 1) { - arg.setType(castOp.getOutputs()[0].getType()); - inputTypes[i] = arg.getType(); - } - } else { - func->emitError(Twine("Unsupported use of func arg at index ") + - Twine(i)); - } - } else { - // Process unused bool ptr type specially, which guarantees bool pointer - // argument's type is realistic and don't mislead backend compiler. - // realistic memory layout of bool pointer is 8 bit width - auto memType = dyn_cast(arg.getType()) - .cloneWith(std::nullopt, builder.getI8Type()); - arg.setType(memType); - inputTypes[i] = arg.getType(); - } - } - } - auto castType = FunctionType::get(func.getContext(), inputTypes, retTypes); - - auto funcFunc = builder.create(func.getLoc(), name, castType); - funcFunc.setAllArgAttrs(argAttrs); - funcFunc.setAllResultAttrs(resAttrs); - auto kernelAttr = func->getAttr(globalKernelAttr); - if (kernelAttr) { - funcFunc->setAttr(globalKernelAttr, kernelAttr); - } - std::string kernelMixMode = "aiv"; - if (existDot) { - // mix also works for pure cube kernel by using the same MAGIC_ELF keyword - kernelMixMode = "mix"; - } - // Set mix_mode in the func attrs so that the backend could know - // the mix_mode by parse the func attrs. - // The backend needs to know the mix_mode because the host wrapper - // needs to set the devbin.magic. Check npu_utils.cpp. - funcFunc->setAttr(kernelMixModeName, builder.getStringAttr(kernelMixMode)); - - std::string parallelMode = "simd"; - if (existSIMTOp) { - parallelMode = "mix_simd_simt"; - } - funcFunc->setAttr(kernelParallelModeName, - builder.getStringAttr(parallelMode)); - - auto autoBlockifyAttr = func->getAttr("auto_blockify_size"); - if (autoBlockifyAttr) - funcFunc->setAttr("auto_blockify_size", autoBlockifyAttr); - - auto &funcFuncBody = funcFunc.getBody(); - auto &funcBody = func.getBody(); - - IRMapping map; - funcBody.cloneInto(&funcFuncBody, map); - - if (!funcFuncBody.hasOneBlock()) { - if (failed(convertMultipleBlockControlFlow(funcFunc, builder))) { - llvm_unreachable("Encounter unsupported control flow"); - } - } - - for (Block &block : funcFuncBody.getBlocks()) { - auto term = block.getTerminator(); - builder.setInsertionPoint(term); - builder.create(func.getLoc(), term->getOperands()); - term->erase(); - } - func.erase(); -} - -void TritonToLinalgPass::addDynamicLegal( - ConversionTarget &target, TritonTypeConverter &tritonTypeConverter) { - target.addLegalDialect< - func::FuncDialect, arith::ArithDialect, math::MathDialect, - linalg::LinalgDialect, affine::AffineDialect, scf::SCFDialect, - cf::ControlFlowDialect, tensor::TensorDialect, LLVM::LLVMDialect, - bufferization::BufferizationDialect, memref::MemRefDialect, - annotation::AnnotationDialect, hivm::HIVMDialect, hfusion::HFusionDialect, - scope::ScopeDialect>(); - - // add legal dialect on condition - target.addLegalOp(); - - // decide which ops need conversion based on uses - target.addDynamicallyLegalOp( - [](mlir::Operation *op) { - if (op->use_empty()) { - return false; - } else { - return true; - } - }); - - target.addDynamicallyLegalOp([&](triton::FuncOp op) { - return tritonTypeConverter.isSignatureLegal(op.getFunctionType()); - }); - - // For CustomOp/CustomMacroOp, tt.ptr should be converted to memref. - target.addDynamicallyLegalOp([&](hivm::CustomOp op) { - return isCustomOpOperandTypesLegal(op->getOperandTypes()); - }); - target.addDynamicallyLegalOp( - [&](hivm::CustomMacroOp op) { - return isCustomOpOperandTypesLegal(op->getOperandTypes()); - }); - - target.addDynamicallyLegalOp([](arith::ConstantOp op) { - auto res = op.getResult(); - if (!isa(res.getType())) { - return true; - } - - if (auto denseAttr = dyn_cast(op.getValue())) { - if (!denseAttr.isSplat() || - !isa(denseAttr.getElementType())) { - return true; - } - if (res.hasOneUse() && isa(*res.user_begin())) { - return true; - } - return false; - } - return true; - }); - - target.addDynamicallyLegalOp( - [](scf::IfOp op) { return !TTOpConverters::hasScalarPointerResult(op); }); - - auto controlFlowTerminatorLegal = [](Operation *op) { - Operation *parent = op->getParentOp(); - if (parent && - parent->hasAttr(TTOpConverters::kScalarPointerCarrierBoundaryAttr)) { - if (auto parentIf = dyn_cast(parent)) - return llvm::equal(op->getOperandTypes(), parentIf.getResultTypes()); - } - - if (parent && parent->hasAttr(controlflow::kPointerDescriptorBoundaryAttr)) - return hasPointerFreeControlFlowBoundary( - cast(parent)); - - return llvm::all_of(op->getOperandTypes(), [](Type t) { - if (isa(t)) - return false; - if (auto shapedType = dyn_cast(t)) - return shapedType.getElementType().isIntOrFloat(); - assert(t.isIntOrIndexOrFloat()); - return true; - }); - }; - - target.addDynamicallyLegalOp( - controlFlowTerminatorLegal); - - auto isArithOrMathOpLegal = [this](Operation *op) { - if (op->hasAttr("MetaUse")) - return false; - - if (isa(op)) - return true; - - bool operateOnTensors = llvm::all_of(op->getOperandTypes(), [](Type type) { - return isa(type); - }); - - return this->namedOps || !operateOnTensors; - }; - - // Numeric selects retain the existing Arith legality. Every scalar-pointer - // select uses the integer-address converter so no memref object crosses it. - target.addDynamicallyLegalOp( - [isArithOrMathOpLegal](arith::SelectOp op) { - if (TTOpConverters::isScalarPointerSelect(op)) - return false; - return isArithOrMathOpLegal(op); - }); - - target.addDynamicallyLegalDialect( - isArithOrMathOpLegal); -} - -namespace { - -/// Route the specific `splat(base) -> addptr(scan) -> addptr(constant) -> -/// store` form through the existing indirect-store operation before use -/// analysis. Lowering either addptr independently would materialize a memref -/// while the remaining operation still expects tensor. -class FoldScanOffsetAddPtrChain : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(triton::AddPtrOp op, - PatternRewriter &rewriter) const override { - if (!op->hasOneUse() || !op.getOffset().getDefiningOp()) - return failure(); - auto nextAddPtr = dyn_cast(*op->user_begin()); - if (!nextAddPtr || - !nextAddPtr.getOffset().getDefiningOp() || - op.getOffset().getType() != nextAddPtr.getOffset().getType() || - !nextAddPtr->hasOneUse()) - return failure(); - auto store = dyn_cast(*nextAddPtr->user_begin()); - auto baseSplat = op.getPtr().getDefiningOp(); - if (!store || !baseSplat || - !isa(baseSplat.getSrc().getType())) - return failure(); - - rewriter.setInsertionPoint(nextAddPtr); - Value combinedOffset = rewriter.create( - nextAddPtr.getLoc(), op.getOffset(), nextAddPtr.getOffset()); - rewriter.create( - store.getLoc(), baseSplat.getSrc(), combinedOffset, store.getValue(), - store.getMask()); - rewriter.eraseOp(store); - rewriter.eraseOp(nextAddPtr); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -// Version only the small, bufferized row-copy loops emitted for mixed-axis -// loads. The column extent is shared by every row: checking it once outside -// the loop lets full tiles recover a static copy while tails keep their mask. -// Run after memory conversion so both branches use the same outer allocation. -static void specializeFullRowCopies(ModuleOp moduleOp) { - SmallVector loops; - moduleOp.walk([&](scf::ForOp loop) { - if (loop->hasAttr("ExtractedLoadOrStore") && - loop->hasAttr("hivm.parallel_loop") && loop.getNumResults() == 0) - loops.push_back(loop); - }); - IRRewriter rewriter(moduleOp.getContext()); - for (scf::ForOp loop : loops) { - // Do not duplicate nested control flow, allocations, or other accesses. - memref::CopyOp copy; - bool supported = true; - unsigned numOps = 0; - for (Operation &op : loop.getBody()->without_terminator()) { - if (++numOps > 32 || op.getNumRegions() != 0) { - supported = false; - break; - } - if (auto candidate = dyn_cast(op)) { - if (copy) { - supported = false; - break; - } - copy = candidate; - } else if (!isMemoryEffectFree(&op)) { - supported = false; - break; - } - } - if (!supported || !copy) - continue; - - auto src = copy.getSource().getDefiningOp(); - auto dst = copy.getTarget().getDefiningOp(); - if (!src || !dst || src.getSourceType().getRank() != 2 || - dst.getSourceType().getRank() != 2 || src.getType().getRank() != 2 || - dst.getType().getRank() != 2) - continue; - auto shape = src.getSourceType().getShape(); - if (shape[0] != 1 || ShapedType::isDynamic(shape[1]) || shape[1] <= 1 || - dst.getSourceType().getShape() != shape) - continue; - int64_t width = shape[1]; - auto isZero = [](OpFoldResult v) { return isConstantIntValue(v, 0); }; - auto isOne = [](OpFoldResult v) { return isConstantIntValue(v, 1); }; - auto isRowPrefix = [&](memref::SubViewOp view) { - auto [strides, offset] = view.getSourceType().getStridesAndOffset(); - return strides.back() == 1 && - llvm::all_of(view.getMixedOffsets(), isZero) && - llvm::all_of(view.getMixedStrides(), isOne) && - isOne(view.getMixedSizes()[0]); - }; - if (!isRowPrefix(src) || !isRowPrefix(dst) || - src.getMixedSizes()[1] != dst.getMixedSizes()[1]) - continue; - auto extent = dyn_cast(src.getMixedSizes()[1]); - if (!extent || !loop.isDefinedOutsideOfLoop(extent)) - continue; - - // Require the existing load buffer and its exact row slice. This excludes - // stores and avoids moving/duplicating a per-row allocation into a branch. - auto row = dst.getSource().getDefiningOp(); - auto source = src.getSource().getDefiningOp(); - if (!row || !source || row.getSourceType().getRank() != 2 || - !row.getSource().getDefiningOp() || - !loop.isDefinedOutsideOfLoop(row.getSource()) || - !loop.isDefinedOutsideOfLoop(source.getSource()) || - row.getMixedOffsets()[0] != OpFoldResult(loop.getInductionVar()) || - !isZero(row.getMixedOffsets()[1]) || - !llvm::all_of(row.getMixedStrides(), isOne) || - !isOne(row.getMixedSizes()[0]) || - !isConstantIntValue(row.getMixedSizes()[1], width)) - continue; - - rewriter.setInsertionPoint(loop); - Location loc = copy.getLoc(); - Value fullWidth = rewriter.create(loc, width); - Value full = rewriter.create(loc, arith::CmpIPredicate::eq, - extent, fullWidth); - auto branch = - rewriter.create(loc, full, /*withElseRegion=*/true); - rewriter.setInsertionPointToStart(branch.thenBlock()); - IRMapping mapping; - auto fastLoop = cast(rewriter.clone(*loop, mapping)); - auto fastCopy = *fastLoop.getBody()->getOps().begin(); - // Under extent == width the two zero-offset subviews are the complete - // static rows. Bypass them explicitly; no branch-sensitive folding needed. - fastCopy->setOperand(0, mapping.lookupOrDefault(src.getSource())); - fastCopy->setOperand(1, mapping.lookupOrDefault(dst.getSource())); - loop->moveBefore(branch.elseBlock()->getTerminator()); - // The A5 backend must be able to specialize the full-width library call - // independently of the dynamic tail call that shares its callee. - moduleOp->setAttr("tt.full_row_copy", rewriter.getUnitAttr()); - } -} - -void TritonToLinalgPass::populateTritonToLinalgCanonicalizationPatterns( - RewritePatternSet &patterns) { - patterns.add, - LoadStoreConverter::LoadStoreCanonicalizer, - LoadStoreConverter::LoadStoreCanonicalizer, - LoadStoreConverter::LoadStoreCanonicalizer>( - patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add< - TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer< - arith:: - MulFOp>, // TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer, - TTOpConverters::ScalarMathCanonicalizer - // By test, the following ops do not need canonicalization. - // TTOpConverters::ScalarMathCanonicalizer - // TTOpConverters::ScalarMathCanonicalizer - // TTOpConverters::ScalarMathCanonicalizer - >(patterns.getContext()); - patterns.add( - patterns.getContext()); - if (this->enableSelectAnalysis) { - patterns.add(patterns.getContext()); - } -} - -void TritonToLinalgPass::populateTritonToLinalgConversionPatterns( - TypeConverter &typeConverter, RewritePatternSet &patterns, - unsigned int launchGridRank) { - nd2nzFlag = this->enableNd2nzOnVector; - populateFunctionOpInterfaceTypeConversionPattern( - patterns, typeConverter); - - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns - .add, - LoadStoreConverter::MemoryPointerConverter, - LoadStoreConverter::MemoryPointerConverter, - LoadStoreConverter::MemoryPointerConverter>( - patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - // reduce converters - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - - patterns.add(patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add>(patterns.getContext(), - PatternBenefit(2)); - patterns.add>( - patterns.getContext(), PatternBenefit(2)); - patterns.add(patterns.getContext(), - PatternBenefit(2)); - patterns.add( - typeConverter, patterns.getContext(), PatternBenefit(2)); - patterns.add(typeConverter, - patterns.getContext()); - - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(typeConverter, - patterns.getContext()); - - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - // On 950 (910B4/91095), histogram is lowered via hivm.custom builtin - // template. On other targets, histogram is handled by TritonToHFusion pass - // instead. - if (compileOn91095Flag) { - patterns.add(patterns.getContext()); - } - - // Add convert pattern for structured custom ops. - patterns.add, - StructuredCustomOpConverter>( - patterns.getContext()); - - if (!this->namedOps) { - linalg::populateElementwiseToLinalgConversionPatterns(patterns); - } -} - -void TritonToLinalgPass::getDependentDialects(DialectRegistry ®istry) const { - registry - .insert(); -} - -LogicalResult -TritonToLinalgPass::processDescriptorOperations(ModuleOp moduleOp) { - // --- ConversionTarget: dynamic legality checks --- - mlir::ConversionTarget target(getContext()); - target.addLegalDialect(); - - // Dialect-level dynamic legality: ops are legal if none of their - // operands/results use TensorDescType. - target.addDynamicallyLegalDialect< - mlir::arith::ArithDialect, mlir::scf::SCFDialect, triton::TritonDialect>( - [](mlir::Operation *op) { - return !DescriptorConverter::hasATensorDescriptorType( - op->getOperandTypes()) && - !DescriptorConverter::hasATensorDescriptorType( - op->getResultTypes()); - }); - // Function signature legality: Triton FuncOp is legal if its inputs/outputs - // contain no TensorDescType. - target.addDynamicallyLegalOp([](triton::FuncOp funcOp) { - return !DescriptorConverter::hasATensorDescriptorType( - funcOp.getFunctionType().getInputs()) && - !DescriptorConverter::hasATensorDescriptorType( - funcOp.getFunctionType().getResults()); - }); - target.addLegalOp(); - target.addIllegalOp(); - - // --- Patterns --- - mlir::RewritePatternSet patterns(&getContext()); - patterns.add( - patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add( - patterns.getContext()); - patterns.add( - patterns.getContext()); - - mlir::ConversionConfig config; - config.buildMaterializations = true; - if (failed(applyPartialConversion(moduleOp, target, std::move(patterns), - config))) { - moduleOp->emitError("failed to convert tensor descriptor operations"); - return failure(); - } - - return success(); -} - -LogicalResult -TritonToLinalgPass::processPtrBroadcastOperations(ModuleOp moduleOp) { - // --- ConversionTarget: dynamic legality checks --- - mlir::ConversionTarget target(getContext()); - target.addLegalOp(); - target.addLegalOp(); - target.addDynamicallyLegalOp([](triton::BroadcastOp op) { - if (op->hasAttr("MetaUse")) { - return true; - } - auto resultType = dyn_cast(op.getType()); - HoistBroadcast::BroadcastHoister hoister(op); - return !(isa(resultType.getElementType()) && - hoister.canBroadcast()); - }); - - // --- Patterns --- - mlir::RewritePatternSet patterns(&getContext()); - patterns.add(patterns.getContext()); - - if (failed(applyPartialConversion(moduleOp, target, std::move(patterns)))) { - moduleOp->emitError("failed to convert ptr broadcast operations"); - return failure(); - } - - return success(); -} - -LogicalResult -TritonToLinalgPass::processImplicitPermuteOperations(ModuleOp moduleOp) { - mlir::RewritePatternSet patterns(&getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add(patterns.getContext()); - patterns.add( - patterns.getContext()); - - if (failed(applyPatternsGreedily(moduleOp, std::move(patterns)))) { - LLVM_DEBUG({ llvm::dbgs() << "ImplicitPermute: rewrite MemOp failed\n"; }); - } - - mlir::PassManager pm(&getContext(), moduleOp.getOperationName()); - pm.addPass(createCSEPass()); - pm.addPass(createCanonicalizerPass()); - return runPipeline(pm, getOperation()); -} - -LogicalResult TritonToLinalgPass::processStridedLoadStoreRewriteOperations( - ModuleOp moduleOp) { - // The strided-axis rewrites below only apply in 950 SIMT mode. On other - // targets we leave strided loads to the legacy strided DMA lowering. - if (!(compileOn91095Flag && - triton::ascend::isSimtTemplateMode(compileModeFlag))) { - return success(); - } - - auto runLayoutMemoryPhase = - [&](cfg::LayoutMemoryCompatibilityPhase phase) -> LogicalResult { - mlir::PassManager phasePm(&getContext(), moduleOp.getOperationName()); - phasePm.addPass(cfg::createLayoutMemoryCompatibilityPass(phase)); - return runPipeline(phasePm, getOperation()); - }; - - // Keep the original insertion point after ImplicitPermute. Axis remains in - // the pre-Diagonal slot; the current target has no Diagonal migration, so - // the two compatibility phases run adjacently. - if (failed(runLayoutMemoryPhase( - cfg::LayoutMemoryCompatibilityPhase::BeforeDiagonal))) { - return failure(); - } - - if (failed(runLayoutMemoryPhase( - cfg::LayoutMemoryCompatibilityPhase::AfterDiagonal))) { - return failure(); - } - - // Mirror processImplicitPermuteOperations: clean up dead IR left behind by - // PtrAnalysis when the pattern decided not to rewrite (e.g. stride==1 case - // returns failure() but PtrAnalysis has already inserted helper ExtSI ops). - // Without this, downstream passes may trip on stale uses. - mlir::PassManager pm(&getContext(), moduleOp.getOperationName()); - pm.addPass(createCSEPass()); - pm.addPass(createCanonicalizerPass()); - return runPipeline(pm, getOperation()); -} - -LogicalResult -TritonToLinalgPass::processLegalStrideOperations(ModuleOp moduleOp) { - mlir::ConversionTarget target(getContext()); - target.addLegalOp(); - target.addDynamicallyLegalOp( - [](memref::ReinterpretCastOp op) { - return !LoadStoreConverter::ReinterpretCastStrideCanonicalizer:: - hasFixableZeroStride(op); - }); - - mlir::RewritePatternSet patterns(&getContext()); - patterns.add( - patterns.getContext()); - - if (failed(applyPartialConversion(moduleOp, target, std::move(patterns)))) { - moduleOp->emitError( - "failed to legalize reinterpret_cast dynamic stride(0) with size(1)"); - return failure(); - } - - return success(); -} - -void TritonToLinalgPass::runOnOperation() { - compileOn91095Flag = this->compileOn91095; - auto compileMode = triton::ascend::parseCompileMode(this->compileMode); - if (!compileMode) { - getOperation().emitError() - << "triton-to-linalg compile-mode is invalid: " << this->compileMode; - signalPassFailure(); - return; - } - compileModeFlag = *compileMode; - - auto moduleOp = getOperation(); - - // Validate the CFO handoff before descriptor conversion, canonicalization, - // or any other IR mutation can erase malformed metadata with dead code. - if (failed(validatePointerDescriptorHandoffMetadata(moduleOp))) { - moduleOp->emitError("invalid pointer descriptor handoff metadata"); - signalPassFailure(); - return; - } - - // Check if the kernel contains a cube op: tl.dot / tl.dot_scaled / al.dot - // decompose into a cube linalg.matmul, and conv1d/conv2d run on the cube - // unit. Without any of them the kernel would be tagged as a pure AIV kernel; - // with them it must be tagged mix mode, otherwise the cube tile-and-slice - // fails (cbuf overflow). - bool existDot = false; - moduleOp.walk([&](hivm::CustomOp customOp) { - if (customOp.getCoreType() == hivm::TCoreType::CUBE_AND_VECTOR || - customOp.getCoreType() == hivm::TCoreType::CUBE) { - existDot = true; - return WalkResult::interrupt(); - } - return WalkResult::advance(); - }); - moduleOp.walk([&](hivm::CustomMacroOp customMacroOp) { - if (customMacroOp.getCoreType() == hivm::TCoreType::CUBE_AND_VECTOR || - customMacroOp.getCoreType() == hivm::TCoreType::CUBE) { - existDot = true; - return WalkResult::interrupt(); - } - return WalkResult::advance(); - }); - moduleOp.walk([&](Operation *op) { - if (isa(op)) { - existDot = true; - return WalkResult::interrupt(); - } - return WalkResult::advance(); - }); - existDotFlag = existDot; - - // NOTE: existSIMTOp is intentionally computed AFTER - // processStridedLoadStoreRewriteOperations below, because that step - // materializes triton::ascend::IndirectLoadOp/IndirectStoreOp (which - // isSIMTOp() counts). Walking here (before the rewrite) would miss them and - // mislabel the kernel parallel_mode as "simd" instead of "mix_simd_simt"; - // then enable_simt would be false and the launch would not reserve - // localMemorySize for the SIMT templates -> VEC UB out-of-bounds (error 341) - // at runtime on mix-CV kernels. - bool existSIMTOp = false; - - // Execute tensor descriptor operations conversion - if (failed(processDescriptorOperations(moduleOp))) { - signalPassFailure(); - } - - // Execute implicit permute - if (failed(processImplicitPermuteOperations(moduleOp))) { - LLVM_DEBUG( - { llvm::dbgs() << "Failed to process implicit permute operations\n"; }); - signalPassFailure(); - } - - // SIMT IndirectLoad fast-path rewrite (runs after ImplicitPermute so the - // permuted access patterns have already been absorbed; this step only - // catches non-permuted last-axis stride > 1 loads). - if (failed(processStridedLoadStoreRewriteOperations(moduleOp))) { - LLVM_DEBUG({ - llvm::dbgs() << "Failed to process indirect-load rewrite operations\n"; - }); - signalPassFailure(); - } - - // Detect SIMT ops AFTER the indirect-load rewrite so the freshly materialized - // IndirectLoadOp/IndirectStoreOp are counted (drives parallel_mode -> - // "mix_simd_simt" -> enable_simt -> launch reserves localMemorySize). - moduleOp.walk([&](Operation *op) { - if (isSIMTOp(op)) { - existSIMTOp = true; - LLVM_DEBUG({ - auto &os = llvm::dbgs(); - os << "Found SIMT op in function: "; - os << op->getName(); - os << "\n"; - }); - return WalkResult::interrupt(); - } - return WalkResult::advance(); - }); - - // 0. Annotate Memory-Related Triton FuncOps with tensor_kind (used by - // profiling). - { - PassManager pm(&getContext(), moduleOp.getOperationName()); - pm.addPass(triton::createMarkTensorKindPass()); - if (failed(runPipeline(pm, moduleOp))) { - moduleOp->emitError("failed to run LoopCanonicalizerPass"); - signalPassFailure(); - return; - } - } - - RewritePatternSet canonicalizerPatterns(&getContext()); - // 1. Canonicalize load/store related patterns. - // The currently registered patterns rewrite pointer consumers but do not - // replace tt.make_tensor_ptr/tt.addptr descriptor rebuild roots. A future - // pattern that rewrites those producers must atomically transfer - // PointerDescriptorRebuild to its replacement; entry validation alone - // cannot preserve metadata across a producer rewrite. - this->populateTritonToLinalgCanonicalizationPatterns(canonicalizerPatterns); - if (failed( - applyPatternsGreedily(moduleOp, std::move(canonicalizerPatterns)))) { - moduleOp->emitError("failed to apply Canonicalizer Patterns"); - signalPassFailure(); - return; - } - - // Stage-1 may legitimately delete an unused, already validated boundary. - // Revalidate every surviving handoff so later phases never consume a marker - // whose structural edges were changed without updating its slot contract. - if (failed(validatePointerDescriptorHandoffMetadata(moduleOp))) { - moduleOp->emitError("invalid pointer descriptor handoff metadata"); - signalPassFailure(); - return; - } - - // 2.1 Pre-clean dead control flow before use analysis. Descriptor handoff - // attributes require stable loop positions and rebuild operands, so marked - // modules use a restricted cleanup that cannot rewrite either contract. - if (failed(preCleanBeforeUseAnalysis(moduleOp))) { - moduleOp->emitError( - "failed to pre-clean dead control-flow before use analysis"); - signalPassFailure(); - return; - } - - // 2. Perform use analysis on FuncOp. - moduleOp.walk([this](triton::FuncOp op) { - if (failed(runUseAnalysis(op))) { - signalPassFailure(); - } - }); - - if (failed(preservePointerDescriptorComputations(moduleOp))) { - moduleOp->emitError("invalid pointer descriptor handoff metadata"); - signalPassFailure(); - return; - } - - RewritePatternSet patterns(&getContext()); - ConversionTarget target(getContext()); - TritonTypeConverter tritonTypeConverter{}; - - // 3. Mark legal dialects and operations. - this->addDynamicLegal(target, tritonTypeConverter); - - // 4. Mark ops that must be converted explicitly (e.g. tt.scan). - auto loopOpLegalFn = [](LoopLikeOpInterface loopOp) { - Operation *op = loopOp.getOperation(); - if (op->hasAttr(controlflow::kPointerDescriptorBoundaryAttr)) { - return hasPointerFreeControlFlowBoundary(loopOp); - } - return !op->hasAttr("UnhandledLoopOp"); - }; - - target.addIllegalOp(); - target.addIllegalOp(); - target.addDynamicallyLegalOp(loopOpLegalFn); - target.addDynamicallyLegalOp(loopOpLegalFn); - - // 5. Register converters for all illegal Triton ops. - // Execute ptr broadcast operations conversion - if (failed(processPtrBroadcastOperations(moduleOp))) { - signalPassFailure(); - } - this->populateTritonToLinalgConversionPatterns(tritonTypeConverter, patterns, - LAUNCH_GRID_RANK); - - // 6. Inject program id / number of programs arguments into each Triton kernel - // function. - for (auto func : getOperation().getOps()) { - addProgramInfo(func, globalKernel); - } - - moduleOp.walk([this](LoopLikeOpInterface loopOp) { - auto *op = loopOp.getOperation(); - // CFO-expanded pointer loops already carry policy-owned descriptor - // components. They require ordinary nested-op conversion, not the legacy - // pointer-loop decomposition a second time. Unmarked loops are delegated - // only when their original boundary still carries Triton pointer/address - // state. A fixed-layout memref is already a complete SCF value; the fact - // that its init is produced by reinterpret_cast does not make it BlockData. - bool hasExpandedPointerDescriptor = - op->hasAttr(mlir::triton::controlflow::kPointerDescriptorBoundaryAttr); - if (!op->hasAttr("ExtractedLoadOrStore") && - needsLegacyBlockDataLoopRewrite(loopOp)) - op->setAttr("UnhandledLoopOp", UnitAttr::get(op->getContext())); - - if (hasExpandedPointerDescriptor) - return; - - for (auto res : loopOp->getResults()) { - if (auto tensorType = dyn_cast(res.getType()); - tensorType && - !isa(tensorType.getElementType())) { - IRRewriter rewriter(op->getContext()); - rewriter.setInsertionPointAfter(op); - auto newVal = - rewriter.create(op->getLoc(), res.getType(), res); - rewriter.replaceAllUsesExcept(res, newVal, newVal); - } - } - }); - - // 7. Convert ops. - LogicalResult conversionResult = - applyPartialConversion(moduleOp, target, std::move(patterns)); - - if (failed(conversionResult)) { - moduleOp->emitError("failed to apply Conversion Patterns"); - signalPassFailure(); - return; - } - - // These markers are contracts with this conversion pass. Remove them only - // after successful conversion so failure reproducers retain ownership, - // exact dynamic descriptor slots, and complete rebuild roots. - moduleOp.walk([](LoopLikeOpInterface loopOp) { - loopOp->removeAttr(controlflow::kPointerDescriptorBoundaryAttr); - }); - moduleOp.walk([](Operation *op) { - op->removeAttr(controlflow::kPointerDescriptorRebuildAttr); - op->removeAttr(controlflow::kPointerDescriptorOffsetFormAttr); - op->removeAttr(controlflow::kPointerDescriptorStructuredAxesAttr); - }); - moduleOp.walk([](Operation *op) { - op->removeAttr(TTOpConverters::kScalarPointerCarrierBoundaryAttr); - }); - - // Conversion can expose a make_range carrier whose loop result is already - // dead. Remove only the proven dead range/update chain before generic - // canonicalization; live tensor carriers and scalar address state remain. - eraseDeadRangeCarriers(moduleOp); - - // 7.1 Workaround: fold duplicated one-hot reconstruction emitted after - // ArgMax lowering. The issue is not in triton::ReduceOp semantics themselves; - // redundant value reconstruction is materialized later and can lower to - // incorrect code on Ascend, so this is fixed post-conversion on - // linalg::ReduceOp. - { - RewritePatternSet foldPatterns(&getContext()); - TTOpConverters::populatePostConversionCanonicalizationPatterns( - foldPatterns); - - if (failed(applyPatternsGreedily(moduleOp, std::move(foldPatterns)))) { - moduleOp->emitError("failed to fold one-hot gather after max_with_index"); - signalPassFailure(); - return; - } - } - - // Execute legal stride operations conversion - if (failed(processLegalStrideOperations(moduleOp))) { - signalPassFailure(); - } - - // 8. Convert function prologue/epilogue. - moduleOp.walk([&](triton::FuncOp func) { - this->convertTTFunc(func, existDot, existSIMTOp); - }); - - rewriteDevicePrintOffsets(moduleOp); - - // 9. Clean up dead code and simplify IR. - PassManager pm(&getContext(), moduleOp.getOperationName()); - pm.addPass(createCSEPass()); - pm.addPass(createCanonicalizerPass()); - if (failed(runPipeline(pm, getOperation()))) { - signalPassFailure(); - } - - specializeFullRowCopies(moduleOp); - - // 10. Collapses call-site locations whose callee is an inlined Triton stdlib - // helper (under site-packages) down to their caller (user-file) frame - // Opt-in via LLVM_EXTRACT_DI_LOCAL_VARIABLES=1. - { - PassManager pm(&getContext(), moduleOp.getOperationName()); - pm.addPass(triton::createCanonicalizeDebugLocationsPass()); - if (failed(runPipeline(pm, moduleOp))) { - moduleOp->emitWarning("CanonicalizeDebugLocationsPass pass failed"); - } - } - - // 11. Deduplicate debug NOPs inserted by converters. - // Opt-in via LLVM_EXTRACT_DI_LOCAL_VARIABLES=1. - { - PassManager pm(&getContext(), moduleOp.getOperationName()); - pm.addPass(triton::createDeduplicateDebugNopsPass()); - if (failed(runPipeline(pm, moduleOp))) { - moduleOp->emitWarning("DeduplicateDebugNops pass failed"); - // Non-fatal: dedup is a quality improvement, not a correctness pass. - } - } - - // Calculate size of PointerCastOp precisely - SmallVector castOps; - - moduleOp.walk([&](hivm::PointerCastOp op) { castOps.push_back(op); }); - - for (auto op : castOps) { - SmallVector reinterpretCastOps; - for (Operation *user : op->getUsers()) { - if (auto reinterpretCast = dyn_cast(user)) - reinterpretCastOps.push_back(reinterpretCast); - } - if (reinterpretCastOps.empty()) - continue; - - bool isScalarPointerCarrier = op->hasAttr(kScalarPointerCarrierAttr); - IRRewriter rewriter(&getContext()); - rewriter.setInsertionPointAfter(op); - Value addr = op.getAddrs()[0]; - auto elementType = - cast(op.getResult().getType()).getElementType(); - - for (memref::ReinterpretCastOp reinterpretCastOp : reinterpretCastOps) { - auto sizes = reinterpretCastOp.getStaticSizes(); - auto staticStrides = reinterpretCastOp.getStaticStrides(); - auto strides = reinterpretCastOp.getStrides(); - if (reinterpretCastOp.getStaticOffsets().size() != 1) { - reinterpretCastOp->emitError( - "IntToPtrOp must converted to PointerCastOp of " - "memref type"); - signalPassFailure(); - return; - } - int64_t castOpSize = 0; - SmallVector dynamicSizes; - for (const auto &[size, stride] : llvm::zip_equal(sizes, staticStrides)) { - assert(!ShapedType::isDynamic(size)); - if (ShapedType::isDynamic(stride)) - dynamicSizes.push_back(size); - else - castOpSize = size * stride; - } - rewriter.setInsertionPoint(reinterpretCastOp); - Value dynamicSize = rewriter.create( - op.getLoc(), rewriter.getIndexAttr(castOpSize)); - for (const auto &[size, stride] : - llvm::zip_equal(dynamicSizes, strides)) { - Value axisSize = rewriter.create( - op.getLoc(), rewriter.getIndexAttr(size)); - axisSize = - rewriter.create(op.getLoc(), stride, axisSize); - dynamicSize = - rewriter.create(op.getLoc(), dynamicSize, axisSize); - } - auto staticOffset = reinterpretCastOp.getStaticOffsets()[0]; - auto materializeOffset = [&](Type targetType) -> Value { - if (ShapedType::isDynamic(staticOffset)) { - Value offset = reinterpretCastOp.getOffsets()[0]; - if (offset.getType() != targetType) - offset = rewriter.create(op.getLoc(), - targetType, offset); - return offset; - } - if (targetType.isIndex()) - return rewriter.create(op.getLoc(), - staticOffset); - return rewriter.create( - op.getLoc(), rewriter.getIntegerAttr(targetType, staticOffset)); - }; - - auto memrefType = MemRefType::get({ShapedType::kDynamic}, elementType); - auto createPointerCast = [&](Value address, Value capacity) { - auto cast = rewriter.create( - op.getLoc(), memrefType, address, capacity); - auto mark = - rewriter.create(op.getLoc(), cast.getResult()); - mark->setAttr(hivm::AddressSpaceAttr::getMnemonic(), - {hivm::AddressSpaceAttr::get(rewriter.getContext(), - hivm::AddressSpace::GM)}); - return cast; - }; - - if (!isScalarPointerCarrier) { - // Preserve main-dev behavior for ordinary PointerCast operations. The - // provenance-gated branches below exist specifically for scalar - // pointer carriers introduced by this conversion pipeline. - Value offsetValue = materializeOffset(addr.getType()); - Value elementTypeSize; - if (auto intType = dyn_cast(elementType)) { - elementTypeSize = rewriter.create( - op.getLoc(), - rewriter.getIntegerAttr(addr.getType(), intType.getWidth() / 8)); - } else if (auto floatType = dyn_cast(elementType)) { - elementTypeSize = rewriter.create( - op.getLoc(), rewriter.getIntegerAttr(addr.getType(), - floatType.getWidth() / 8)); - } else { - llvm_unreachable("Cannot get memory size"); - } - offsetValue = rewriter.create(op.getLoc(), offsetValue, - elementTypeSize); - Value realAddr = - rewriter.create(op.getLoc(), addr, offsetValue); - auto newCastOp = createPointerCast(realAddr, dynamicSize); - - auto oldResultType = - cast(reinterpretCastOp.getResult().getType()); - MemRefType newResultType = oldResultType; - if (auto stridedLayout = - dyn_cast(oldResultType.getLayout())) { - if (!ShapedType::isDynamic(stridedLayout.getOffset())) { - auto newLayout = - StridedLayoutAttr::get(rewriter.getContext(), /*offset=*/0, - stridedLayout.getStrides()); - newResultType = MemRefType::get( - oldResultType.getShape(), oldResultType.getElementType(), - newLayout, oldResultType.getMemorySpace()); - } - } - rewriter.replaceOpWithNewOp( - reinterpretCastOp, newResultType, newCastOp, ValueRange({}), - reinterpretCastOp.getSizes(), reinterpretCastOp.getStrides(), - SmallVector({0}), reinterpretCastOp.getStaticSizes(), - reinterpretCastOp.getStaticStrides()); - continue; - } - - if (reachesLayoutSensitiveBoundary(reinterpretCastOp.getResult())) { - // Keep the original offset and result type at externally typed - // boundaries. The replacement PointerCast still starts at the old - // address, so its capacity must include the leading view displacement. - Value pointerCapacity = dynamicSize; - Value offsetElements = materializeOffset(pointerCapacity.getType()); - Value leadingExtent = offsetElements; - if (ShapedType::isDynamic(staticOffset)) { - Value zero = rewriter.create(op.getLoc(), 0); - leadingExtent = rewriter.create(op.getLoc(), - offsetElements, zero); - } - if (ShapedType::isDynamic(staticOffset) || staticOffset > 0) - pointerCapacity = rewriter.create( - op.getLoc(), pointerCapacity, leadingExtent); - auto newCastOp = createPointerCast(addr, pointerCapacity); - rewriter.modifyOpInPlace(reinterpretCastOp, [&] { - reinterpretCastOp.getSourceMutable().assign(newCastOp.getResult()); - }); - continue; - } - - Value offsetValue = materializeOffset(addr.getType()); - Value elementTypeSize; - if (auto intType = dyn_cast(elementType)) { - elementTypeSize = rewriter.create( - op.getLoc(), - rewriter.getIntegerAttr(addr.getType(), intType.getWidth() / 8)); - } else if (auto floatType = dyn_cast(elementType)) { - elementTypeSize = rewriter.create( - op.getLoc(), - rewriter.getIntegerAttr(addr.getType(), floatType.getWidth() / 8)); - } else { - llvm_unreachable("Cannot get memory size"); - } - offsetValue = rewriter.create(op.getLoc(), offsetValue, - elementTypeSize); - Value realAddr = - rewriter.create(op.getLoc(), addr, offsetValue); - auto newCastOp = createPointerCast(realAddr, dynamicSize); - // realAddr already includes the old reinterpret-cast offset in bytes. - // The replacement view therefore starts at offset zero, and its result - // type must describe the same rebased layout. Reusing the old type here - // would combine static_offsets=[0] with (for example) a type-level - // offset of 1, which is rejected by the ReinterpretCast verifier. - auto oldResultType = - cast(reinterpretCastOp.getResult().getType()); - SmallVector rebasedStrides( - oldResultType.getStridesAndOffset().first); - auto rebasedResultType = MemRefType::get( - oldResultType.getShape(), oldResultType.getElementType(), - StridedLayoutAttr::get(oldResultType.getContext(), /*offset=*/0, - rebasedStrides), - oldResultType.getMemorySpace()); - - // Keep the old result and replacement types equal while RAUW updates all - // users to the rebased descriptor type. - rewriter.modifyOpInPlace(reinterpretCastOp, [&] { - reinterpretCastOp.getResult().setType(rebasedResultType); - }); - auto rebasedReinterpretCast = - rewriter.replaceOpWithNewOp( - reinterpretCastOp, rebasedResultType, newCastOp, ValueRange({}), - reinterpretCastOp.getSizes(), reinterpretCastOp.getStrides(), - SmallVector({0}), reinterpretCastOp.getStaticSizes(), - reinterpretCastOp.getStaticStrides()); - if (failed(propagateRebasedSubviewTypes( - rebasedReinterpretCast.getResult(), rewriter))) { - rebasedReinterpretCast.emitError( - "failed to propagate rebased layout through subview users"); - signalPassFailure(); - return; - } - } - if (op->use_empty()) - rewriter.eraseOp(op); - } - - // ScalarPointerCarrier is a pass-local provenance marker. Keep it through - // PointerCast post-processing so only known scalar-address carriers use the - // new layout path. A carrier that is consumed directly (without an - // intermediate reinterpret_cast) is not rebuilt by the loop above, so give - // that remaining GM pointer the same address-space annotation before the - // provenance marker is removed. - moduleOp.walk([](hivm::PointerCastOp pointerCast) { - if (!pointerCast->hasAttr(kScalarPointerCarrierAttr)) - return; - OpBuilder builder(pointerCast); - builder.setInsertionPointAfter(pointerCast); - auto mark = builder.create(pointerCast.getLoc(), - pointerCast.getResult()); - mark->setAttr(hivm::AddressSpaceAttr::getMnemonic(), - {hivm::AddressSpaceAttr::get(pointerCast.getContext(), - hivm::AddressSpace::GM)}); - pointerCast->removeAttr(kScalarPointerCarrierAttr); - }); - - // Try interleave optimization - llvm::DenseMap> interleaveCandidate; - llvm::DenseMap> - interleaveCandidateWithMask; - moduleOp.walk([&](bufferization::MaterializeInDestinationOp materializeOp) { - if (auto reinterpretCastOp = - materializeOp.getDest() - .getDefiningOp()) { - if (llvm::isa(reinterpretCastOp.getSource()) && - reinterpretCastOp.getStaticStrides().back() == 2) { - interleaveCandidate[llvm::cast( - reinterpretCastOp.getSource())] - .push_back(materializeOp); - } - } - - // Difference is that converted op chain of store with mask has - // `memref::SubViewOp` - if (auto subviewOp = - materializeOp.getDest().getDefiningOp()) { - if (!llvm::isa( - materializeOp.getSource().getDefiningOp())) - return WalkResult::advance(); - - if (auto reinterpretCastOp = - subviewOp.getSource() - .getDefiningOp()) { - if (llvm::isa(reinterpretCastOp.getSource()) && - reinterpretCastOp.getStaticStrides().back() == 2) { - interleaveCandidateWithMask[llvm::cast( - reinterpretCastOp.getSource())] - .push_back(materializeOp); - } - } - } - - return WalkResult::advance(); - }); - - for (auto [blockArg, materializeVec] : interleaveCandidate) { - // Just enable optimization where exists double materializeOp with same - // block argument destination. - if (materializeVec.size() != 2) - continue; - auto result = InterleaveStatusOptimization(materializeVec); - } - - for (auto [blockArg, materializeVec] : interleaveCandidateWithMask) { - if (materializeVec.size() != 2) - continue; - auto result = InterleaveStatusWithMaskOptimization(materializeVec); - } - - // Force to add an argument at the beginning of function arguments, which - // represents stub arg for workspace. Default type is memref - for (auto func : getOperation().getOps()) { - if (!func->hasAttr("global_kernel")) - continue; - - auto context = func.getContext(); - constexpr int64_t syncBlockLockArgIdx = 0; - NamedAttribute syncBlockLockArgAttr( - StringAttr::get(context, "syncBlockLock"), UnitAttr::get(context)); - MemRefType syncBlockLockArgType = - MemRefType::get(SmallVector(1, ShapedType::kDynamic), - IntegerType::get(context, 8)); - llvm::LogicalResult syncBlockLockArg = - func.insertArgument(syncBlockLockArgIdx, // argIndex - syncBlockLockArgType, // argType - nullptr, func->getLoc()); // dicAttr - func->setAttr("SyncBlockLockArgIdx", - IntegerAttr::get(IntegerType::get(&getContext(), 64), - 0)); // 64: 64位整型 - - constexpr int64_t workspaceArgIdx = 1; - MemRefType workspaceArgType = - MemRefType::get(SmallVector(1, ShapedType::kDynamic), - IntegerType::get(context, 8)); - NamedAttribute workspaceArgAttr(StringAttr::get(context, "workspace"), - UnitAttr::get(context)); - - llvm::LogicalResult workspaceArg = - func.insertArgument(/*argIndex*/ workspaceArgIdx, - /*argType*/ workspaceArgType, - /*dicAttr*/ nullptr, func->getLoc()); - func->setAttr("WorkspaceArgIdx", - IntegerAttr::get(IntegerType::get(&getContext(), 64), - 1)); // 64: 64位整型 - } - - // Fix the Location info - moduleOp.walk([&](Operation *op) { - auto loc = op->getLoc(); - if (isa(loc)) { - llvm::SmallPtrSet stopOps; - traverseForwardUpdateUserChainIf( - op, - /*conditionFn*/ - [](Operation *curOp) { return false; }, - /*stopFn*/ - [](Operation *curOp) { return !isa(curOp->getLoc()); }, - /*actionFn*/ - nullptr, stopOps); - if (stopOps.empty()) { - op->emitWarning() << *op << " and its users all have no location!"; - } else { - Operation *goodOp = *stopOps.begin(); - op->setLoc(goodOp->getLoc()); - } - } - return WalkResult::advance(); - }); -} - -std::unique_ptr> -triton::createTritonToLinalgPass(bool globalKernel, bool namedOps, - bool enableNd2nzOnVector, - bool enableSelectAnalysis, bool compileOn91095, - const std::string &compileMode) { - return std::make_unique( - globalKernel, namedOps, enableNd2nzOnVector, enableSelectAnalysis, - compileOn91095, compileMode); -} - -std::unique_ptr> triton::createTritonToLinalgPass() { - return std::make_unique(); -} diff --git a/third_party/ascend/lib/TritonToLinalg/UseAnalysis.cpp b/third_party/ascend/lib/TritonToLinalg/UseAnalysis.cpp deleted file mode 100644 index 1dfacff1b1..0000000000 --- a/third_party/ascend/lib/TritonToLinalg/UseAnalysis.cpp +++ /dev/null @@ -1,636 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/TritonToLinalg/UseAnalysis.h" -#include "ascend/include/Utils/Utils.h" - -#include "bishengir/Dialect/HIVM/IR/HIVM.h" -#include "bishengir/Dialect/Scope/IR/Scope.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h" -#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h" - -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Debug.h" - -using namespace mlir; -using namespace triton; -using namespace dataflow; - -#define DEBUG_TYPE "triton-use-analysis" - -std::string stringifyUseType(UseType useTy) { - std::string ret; - if (useTy == UseType::MetaUse) { - ret = "MetaUse"; - } else if (useTy == UseType::DataUse) { - ret = "DataUse"; - } else if (useTy == UseType::MixUse) { - ret = "MixUse"; - } else if (useTy == UseType::Undefined) { - ret = "Undefined"; - } - return ret; -} - -#if LLVM_VERSION_MAJOR >= 20 -LogicalResult -triton::UseAnalysis::visitOperation(Operation *op, ArrayRef operands, - ArrayRef results) { -#else -void triton::UseAnalysis::visitOperation(Operation *op, - ArrayRef operands, - ArrayRef results) { -#endif - - if (op->getResults().size() == 1) { - auto resultType = dyn_cast(op->getResult(0).getType()); - if (resultType && isa(resultType.getElementType())) { - for (auto opnd : operands) { - propagateUse(opnd, UseType::MetaUse); - } - } - } - - TypeSwitch(op) - .Case([&](auto load) { - propagateUse(operands[0], UseType::MetaUse); - auto mask = load.getMask(); - auto other = load.getOther(); - if (mask) { - assert(mask != other && "mask and other cannot be the same"); - propagateUse(operands[1], UseType::MetaUse); - } - if (other) { - propagateUse(operands[2], UseType::MetaUse); - } - }) - .Case([&](auto print) { - for (auto operand : operands) - propagateUse(operand, UseType::DataUse); - }) - .Case( - [&](auto assert) { propagateUse(operands[0], UseType::DataUse); }) - .Case([&](auto store) { - propagateUse(operands[0], UseType::MetaUse); - propagateUse(operands[1], UseType::DataUse); - auto value = store.getValue(); - auto mask = store.getMask(); - if (mask) { - assert(mask != value && "mask and data cannot be the same"); - propagateUse(operands[2], UseType::MetaUse); - } - }) - .Case([&](auto store) { - propagateUse(operands[0], UseType::MetaUse); - propagateUse(operands[1], UseType::DataUse); - propagateUse(operands[2], UseType::DataUse); - auto mask = store.getMask(); - if (mask) { - propagateUse(operands[3], UseType::DataUse); - } - }) - // Consider triton::AtomicRMWOp as store operation - .Case([&](auto atomicOp) { - propagateUse(operands[0], UseType::MixUse); - propagateUse(operands[1], UseType::DataUse); - auto value = atomicOp.getVal(); - auto mask = atomicOp.getMask(); - if (mask) { - assert(mask != value && "mask and data cannot be the same"); - propagateUse(operands[2], UseType::MetaUse); - } - }) - .Case([&](auto atomicOp) { - propagateUse(operands[0], UseType::MetaUse); - propagateUse(operands[1], UseType::DataUse); - propagateUse(operands[2], UseType::DataUse); - auto value = atomicOp.getVal(); - }) - .Case([&](auto dot) { - propagateResults(operands[0], results); - propagateResults(operands[1], results); - - auto opc = dot.getC(); - triton::SplatOp splat; - if (opc) { - splat = opc.template getDefiningOp(); - } - - if (opc && splat && splat.getSrc().getDefiningOp()) { - propagateUse(operands[2], UseType::MetaUse); - } else { - propagateUse(operands[2], UseType::DataUse); - } - }) - .Case([&](auto loopOp) { - for (const auto &[yield, init, result] : llvm::zip_equal( - loopOp.getYieldedValues(), loopOp.getInits(), results)) { - propagateResults(getLatticeElement(yield), {result}); - propagateResults(getLatticeElement(init), {result}); - } - }) - .Case([&](auto reduceOp) { - for (auto operand : operands) { - propagateUse(operand, UseType::DataUse); - } - }) - .Case([&](auto extractOp) { - for (auto operand : operands) { - propagateUse(operand, UseType::DataUse); - } - }) - .Case( - [&](auto fixpipeOp) { propagateUse(operands[0], UseType::DataUse); }) - .Case( - [&](auto copyOp) { propagateUse(operands[0], UseType::DataUse); }) - .Case([&](auto customOp) { - for (auto operand : operands) { - propagateUse(operand, UseType::MixUse); - } - }) - .Default([&](Operation *op) { - // this condition account for tt.addptr - for (auto operand : operands) { - propagateResults(operand, results); - } - }); -#if LLVM_VERSION_MAJOR >= 20 - return success(); -#endif -} - -void setMixUseRecursively(Operation *rootOp, bool applyRoot = true) { - traverseBackwardUpdateOperandChainIf( - rootOp, - // ConditionFn - [rootOp, applyRoot](Operation *curOp) { - for (auto res : curOp->getResults()) { - auto tensorType = dyn_cast(res.getType()); - if (tensorType && - isa(tensorType.getElementType())) - return false; - } - return isMetaUse(curOp) && (curOp != rootOp || applyRoot); - }, - // StopFn - [rootOp](Operation *curOp) { - return isa(curOp) && curOp != rootOp; - }, - // ActionFn - [](OpBuilder &b, Operation *op) { - LLVM_DEBUG({ op->setAttr("MixUse", UnitAttr::get(b.getContext())); }); - op->removeAttr("MetaUse"); - }); -} - -static void setMixUseFromValue(Value v) { - if (auto *defOp = v.getDefiningOp()) { - setMixUseRecursively(defOp); - return; - } - - auto blockArg = dyn_cast(v); - if (!blockArg) { - return; - } - - auto *parentOp = blockArg.getOwner()->getParentOp(); - auto loopLikeOp = dyn_cast_or_null(parentOp); - if (!loopLikeOp) { - return; - } - - if (OpOperand *init = loopLikeOp.getTiedLoopInit(blockArg)) { - if (auto *initDefOp = init->get().getDefiningOp()) - setMixUseRecursively(initDefOp); - } - - if (OpOperand *yielded = loopLikeOp.getTiedLoopYieldedValue(blockArg)) { - if (auto *yieldDefOp = yielded->get().getDefiningOp()) - setMixUseRecursively(yieldDefOp); - } -} - -std::optional isIterArgMixUse(Value v, Value target, - const DataFlowSolver &solver) { - auto defOp = v.getDefiningOp(); - auto *use = solver.lookupState(v); - if ((use && use->type == UseType::DataUse) || - isa_and_nonnull(defOp)) - return true; - if (v == target) - return false; - if (!defOp) - return std::nullopt; - for (auto oper : defOp->getOperands()) { - auto res = isIterArgMixUse(oper, target, solver); - if (res.has_value()) - return res.value() || !isMetaUse(defOp); - } - return std::nullopt; -} - -void postProcessWhileOp(scf::WhileOp op, const DataFlowSolver &solver) { - for (const auto &[res, arg] : - llvm::zip_equal(op->getResults(), op.getConditionOp().getArgs())) { - auto *defOp = arg.getDefiningOp(); - if (!defOp) - continue; - auto *use = solver.lookupState(res); - if (use && use->type == UseType::DataUse) - setMixUseRecursively(defOp); - } - for (const auto &[yield, regionArg] : llvm::zip_equal( - op.getYieldOp().getOperands(), op.getBeforeArguments())) { - auto *defOp = yield.getDefiningOp(); - if (!defOp) - continue; - if (isIterArgMixUse(yield, regionArg, solver).value_or(false)) - setMixUseRecursively(defOp); - } -} - -void postProcessLoopOp(LoopLikeOpInterface loopOp, - const DataFlowSolver &solver) { - if (auto whileOp = dyn_cast(loopOp.getOperation())) { - postProcessWhileOp(whileOp, solver); - return; - } - for (const auto &[res, yield, regionArg] : - llvm::zip_equal(loopOp->getResults(), loopOp.getYieldedValues(), - loopOp.getRegionIterArgs())) { - auto *defOp = yield.getDefiningOp(); - if (!defOp) - continue; - auto *use = solver.lookupState(res); - if ((use && use->type == UseType::DataUse) || - isIterArgMixUse(yield, regionArg, solver).value_or(false)) - setMixUseRecursively(defOp); - } -} - -LogicalResult triton::runUseAnalysis(triton::FuncOp &funcOp) { - MLIRContext *context = funcOp.getContext(); - SymbolTableCollection symbolTable; - - DataFlowSolver solver; - solver.load(); - solver.load(); - solver.load(symbolTable); - if (failed(solver.initializeAndRun(funcOp))) { - return failure(); - } - auto &os = llvm::dbgs(); - // Walk the func op, convert tags on operands to tags on operations - funcOp.walk([&](Operation *op) { - LLVM_DEBUG({ os << "[UseAnalysis] op is " << *op << "\n"; }); - UseType useType = UseType::Undefined; - for (auto result : op->getResults()) { - LLVM_DEBUG({ os << "[UseAnalysis] ===> result is " << result << "\n"; }); - auto use = solver.lookupState(result); - assert(use && "Lattice value not found"); - auto thisUseType = use->type; - LLVM_DEBUG({ - os << "[UseAnalysis] ==========> useType is " - << stringifyUseType(thisUseType) << "\n"; - }); - if (thisUseType == UseType::Undefined) { - continue; - } - if (useType == UseType::Undefined) { - useType = thisUseType; - } - if (thisUseType == UseType::MixUse || thisUseType != useType) { - useType = UseType::MixUse; - break; - } - } - - if (useType == UseType::Undefined) { - LLVM_DEBUG({ op->setAttr("Undefined", UnitAttr::get(context)); }); - return; - } else if (useType == UseType::MetaUse) { - auto memEffect = dyn_cast(op); - if (memEffect) { - if (isa(op)) { - LLVM_DEBUG( - { os << "force protecting side-effect op:" << *op << "\n"; }); - op->setAttr("DataUse", UnitAttr::get(context)); - return; - } - } - if (!isa(op)) { - assert(op->getNumResults() == 1 && - "Ops used for meta computation are expected to have one result"); - } - for (auto it = 0; it < op->getNumResults(); ++it) { - // Only set the tag if the operation uses tensors - if (isa(op->getResult(it).getType()) || - (isa(op) && - op->hasAttr(ConverterUtils::discreteAttrName)) || - (isa(op) && - isa(op->getResult(it).getType()))) { - // Setting tag for erasing op later - op->setAttr("MetaUse", UnitAttr::get(context)); - } - } - return; - } else if (useType == UseType::DataUse) { - LLVM_DEBUG({ op->setAttr("DataUse", UnitAttr::get(context)); }); - return; - } - - assert(useType == UseType::MixUse); - - // If the operation only produces scalars, no need to clone it - bool shapedResult = true; - for (auto result : op->getResults()) - shapedResult &= isa(result.getType()); - if (!shapedResult || - isa(op)) { - LLVM_DEBUG({ op->setAttr("MixUse", UnitAttr::get(context)); }); - return; - } - llvm::SetVector metaUsers; - for (auto result : op->getResults()) { - for (auto user : result.getUsers()) { - TypeSwitch(user) - .Case([&](auto load) { - auto ptr = load.getPtr(); - auto mask = load.getMask(); - auto other = load.getOther(); - if (result == ptr || result == mask || result == other) { - metaUsers.insert(user); - } - }) - .Case([&](auto store) { - auto ptr = store.getPtr(); - auto mask = store.getMask(); - if (result == ptr || result == mask) { - metaUsers.insert(user); - } - }) - .Case([&](auto indirectstore) { - auto src = indirectstore.getSrc(); - // Only src is MetaUse (see visitOperation). offsets/value/mask - // are DataUse - if (result == src) { - metaUsers.insert(user); - } - }) - .Case([&](auto atomicOp) { - auto ptr = atomicOp.getPtr(); - auto mask = atomicOp.getMask(); - if (result == ptr || result == mask) - metaUsers.insert(user); - }) - .Case([&](auto atomicOp) { - auto ptr = atomicOp.getPtr(); - if (result == ptr) - metaUsers.insert(user); - }) - .Case([&](auto dot) { - auto opc = dot.getC(); - triton::SplatOp splat; - if (opc) { - splat = opc.template getDefiningOp(); - } - - if (opc && splat && - splat.getSrc().getDefiningOp()) { - metaUsers.insert(user); - } - }) - .Case([&](auto print) {}) - .Case([](auto) { - // Custom operands are MixUse in visitOperation, even when all - // results are used only as memory addresses. Keep the call on - // the materialized value, not a MetaUse clone that is erased - // before the custom operation's operands are converted. - }) - .Default([&](Operation *op) { - bool allMeta = true; - for (auto res : op->getResults()) { - auto resUse = solver.lookupState(res); - if (resUse->type != UseType::MetaUse) { - allMeta = false; - break; - } - } - if (allMeta) { - metaUsers.insert(user); - } - }); - } - } - - // If the operation doesn't have direct meta users, no need to clone it - if (metaUsers.empty()) { - LLVM_DEBUG({ op->setAttr("MixUse", UnitAttr::get(context)); }); - return; - } - - if (isa(op)) - return; - - if (isa(op)) - return; - - // Clone the operation; switch all meta users to use the clone - OpBuilder builder(op); - auto clone = builder.clone(*op); - LLVM_DEBUG({ op->setAttr("MixUse", UnitAttr::get(context)); }); - - // Setting tag for erasing op later - clone->setAttr("MetaUse", UnitAttr::get(context)); - - for (auto [res_i, result] : llvm::enumerate(op->getResults())) { - for (auto user : metaUsers) { - for (auto &operand : user->getOpOperands()) { - if (operand.get() == result) { - operand.set(clone->getResult(res_i)); - } - } - } - } - }); - LLVM_DEBUG({ - os << "[UseAnalysis] Before post-process, funcOp is " << *funcOp << "\n"; - }); - // Post-process - funcOp.walk([&](Operation *op) { - // Handle indirect load and store case. - // For example, load(1st) -> computeOp -> load(2nd), - // or load -> computeOp -> store - // The first load is IndirectLoadInterfaceOp. - // Do not inplace replace MetaUse by MixUse. Because the condition checking - // depends on that the op has the attr of MetaUse. - // Handle the indirect load interface op - // We first trace from the 1st load to the 2nd load with the ops between - // them marked as MixUse. Then we traceback from the 2nd load to mark defs - // MixUse. - if (opIsIndirectLoad(op) || opIsIndirectCalc(op)) { - LLVM_DEBUG({ - os << "[UseAnalysis] Found indirect load interface op: " << *op << "\n"; - }); - llvm::SmallPtrSet stopOps; - // Modify the users of this op's result. - traverseForwardUpdateUserChainIf( - op, - /*conditionFn*/ - [op](Operation *curOp) { return isMetaUse(curOp) && curOp != op; }, - /*stopFn*/ - [&](Operation *curOp) { - // triton::LoadOp or triton::StoreOp without MetaUse means - // it is an indirect load or store - // instead of the load providing the offset. - // The pattern is as follows, - // load -> ops -> load - // load -> ops -> store - // We need to ensure the intermediate ops are marked MixUse - // so that they will be replaced instead of be erased without - // conversion. - return (isa(curOp)) && - !isMetaUse(curOp); - }, - /*actionFn*/ - [](OpBuilder &b, Operation *op) { setMixUseRecursively(op); }, - stopOps); - LLVM_DEBUG({ - os << "[UseAnalysis] stopOps are \n"; - for (auto [idx, stopOp] : llvm::enumerate(stopOps)) - os << idx << ": " << *stopOp << "\n"; - }); - LLVM_DEBUG({ - os << "[UseAnalysis] After trace, funcOp is " << *funcOp << "\n"; - }); - for (auto *stopOp : stopOps) - setMixUseRecursively(stopOp, /*applyRoot=*/false); - LLVM_DEBUG({ - os << "[UseAnalysis] After traceback of stopOp, funcOp is " << *funcOp - << "\n"; - }); - // Modify this op. - LLVM_DEBUG({ op->setAttr("MixUse", UnitAttr::get(context)); }); - op->removeAttr("MetaUse"); - } - if (op->hasAttr(ConverterUtils::discreteAttrName)) - setMixUseRecursively(op); - if (auto loopOp = dyn_cast(op)) { - postProcessLoopOp(loopOp, solver); - } else if (auto ifOp = dyn_cast(op)) { - SmallVector yields(ifOp.thenYield().getOperands()); - if (!ifOp.getElseRegion().empty()) - yields.append(llvm::to_vector(ifOp.elseYield().getOperands())); - for (auto yield : yields) { - setMixUseFromValue(yield); - } - } else if (auto atomicRmwOp = dyn_cast(op)) { - auto mask = atomicRmwOp.getMask(); - if (mask && op->hasAttr(ConverterUtils::discreteMaskAttrName)) - setMixUseRecursively(mask.getDefiningOp()); - } - }); - // Remove MetaUse in case of MixUse existing in the op - funcOp.walk([&](Operation *op) { - if (isMetaUse(op) && isMixUse(op)) { - op->removeAttr("MetaUse"); - } - }); - // Masked load with non-scalar tensor `other` lowers as - // arith.select(mask, loaded, other) so the mask SSA must stay live. - funcOp.walk([&](triton::LoadOp load) { - Value mask = load.getMask(); - Value other = load.getOther(); - if (!mask || !other) - return; - if (other.getDefiningOp()) - return; - if (auto c = other.getDefiningOp()) { - if (auto dense = dyn_cast(c.getValue())) { - if (dense.isSplat()) - return; - } - } - if (auto *def = mask.getDefiningOp()) - setMixUseRecursively(def); - }); - // hivm.custom present library call, shouldn't be metause - funcOp.walk([&](hivm::CustomOp op) { - if (isMetaUse(op)) { - op->removeAttr("MetaUse"); - } - }); - funcOp.walk([&](hivm::CustomMacroOp op) { - if (isMetaUse(op)) { - op->removeAttr("MetaUse"); - } - }); - LLVM_DEBUG({ - os << "[UseAnalysis] After post-process, funcOp is " << *funcOp << "\n"; - }); - return success(); -} - -static bool isScalarI1ToI8PointerBitcast(Operation *op) { - auto bitcast = dyn_cast(op); - if (!bitcast) - return false; - - auto sourceType = dyn_cast(bitcast.getSrc().getType()); - auto resultType = dyn_cast(bitcast.getType()); - if (!sourceType || !resultType) - return false; - - return sourceType.getPointeeType().isInteger(1) && - resultType.getPointeeType().isInteger(8); -} - -MetaUseEraser::MetaUseEraser(MLIRContext *context) - : RewritePattern(MatchAnyOpTypeTag(), /*benefit=*/10, context) {} - -LogicalResult MetaUseEraser::matchAndRewrite(Operation *op, - PatternRewriter &rewriter) const { - LLVM_DEBUG({ - int64_t count = 0; - for (auto result : op->getResults()) { - count += std::distance(result.use_begin(), result.use_end()); - } - llvm::dbgs() << "Number of user: " << count << "\n"; - }); - if (isa(op)) { - return rewriter.notifyMatchFailure(op, - "AddPtrOp will be handled separately"); - } - if (isScalarI1ToI8PointerBitcast(op)) - return rewriter.notifyMatchFailure( - op, "scalar i1-to-i8 pointer bitcast requires conversion"); - if (isMetaUse(op)) { - rewriter.eraseOp(op); - return success(); - } - return rewriter.notifyMatchFailure(op, "requires meta ops"); -} diff --git a/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp b/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp index e5eeac0bfa..84a6bdf70b 100644 --- a/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp +++ b/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp @@ -21,7 +21,7 @@ */ #include "TritonToUnstructure/UnstructureConversionPass.h" -#include "TritonToLinalg/MaskAnalysis.h" +#include "TritonMemoryAccess/LoadStoreMaskAnalysis.h" #include "TritonToStructured/CannonicalizerConverter.h" #include "TritonToUnstructure/IndirectAtomicUtils.h" #include "Utils/Utils.h" diff --git a/third_party/ascend/triton_ascend.cc b/third_party/ascend/triton_ascend.cc index 4588c80cae..9d71e58f9d 100644 --- a/third_party/ascend/triton_ascend.cc +++ b/third_party/ascend/triton_ascend.cc @@ -20,7 +20,7 @@ #include "ascend/include/TritonToHFusion/Passes.h" #include "ascend/include/TritonToHIVM/Passes.h" #include "ascend/include/TritonToLLVM/Passes.h" -#include "ascend/include/TritonToLinalg/Passes.h" +#include "incubated/Conversion/TritonToLinalgIncubated/TritonToLinalgIncubatedPass.h" #include "ascend/include/TritonToStructured/Passes.h" #include "ascend/include/TritonToUnstructure/Passes.h" @@ -104,7 +104,7 @@ void init_triton_ascend_passes_ttir(py::module &&m) { [](mlir::PassManager &pm, bool globalKernel, bool namedOps, bool enableNd2nzOnVector, bool enableSelectAnalysis, bool compileOn91095, const std::string &compileMode) { - pm.addPass(mlir::triton::createTritonToLinalgPass( + pm.addPass(mlir::triton::Incubated::createTritonToLinalgIncubatedPass( globalKernel, namedOps, enableNd2nzOnVector, enableSelectAnalysis, compileOn91095, compileMode)); }); From 43e29f9de1810c525688c34781c5eb5165244426 Mon Sep 17 00:00:00 2001 From: zhongsanming Date: Wed, 18 Nov 2026 11:18:00 +0800 Subject: [PATCH 2/7] [Ascend][Test] Retarget TTIR->Linalg tests to the FLIR incubated pass Use --triton-to-linalg-incubated and the TritonToLinalgIncubated IR dump label, and refresh checks for the current MLIR printer (to_tensor result type elision, explicit matmul indexing_maps). --- .../loop_mask_fallback.mlir | 2 +- .../TritonToLinalg/chunk_coalescing.mlir | 2 +- .../TritonToLinalg/indirect_load_rewrite.mlir | 8 +++---- .../TritonToLinalg/sin_cos_simt_route.mlir | 4 ++-- .../strided_axis_coalescing.mlir | 2 +- .../DiscreteMaskAccess/loop_prefix_mask.mlir | 2 +- .../Inner-scope-alloc-to-tensor-clone.mlir | 10 ++++----- .../AllocMultiCache/Inner-scope-whileop.mlir | 22 +++++++++---------- .../SplitDataflow/test_fixpipe_nz2nz.mlir | 10 ++++----- .../DynamicCVPipeline/test-remove-attrs.mlir | 2 +- .../General/TritonAscendAllPass/conv1d.mlir | 2 +- .../General/TritonAscendAllPass/conv2d.mlir | 2 +- .../d007_per_axis_structured_opaque_e2e.mlir | 2 +- .../scope_pointer_downstream.mlir | 2 +- .../tensor_ptr_different_base_invalid.mlir | 2 +- .../graph-optimize-load-store-lowering.mlir | 4 ++-- .../General/TritonToLinalg/atomic_rmw.mlir | 2 +- .../TritonToLinalg/atomic_rmw_block.mlir | 2 +- .../General/TritonToLinalg/clamp_fp8.mlir | 2 +- .../TritonToLinalg/custom_macro_op.mlir | 2 +- .../custom_op_pointer_splat.mlir | 4 ++-- .../General/TritonToLinalg/custom_op_ptr.mlir | 2 +- .../General/TritonToLinalg/debug_nop.mlir | 4 ++-- .../General/TritonToLinalg/dotScaled_fp4.mlir | 2 +- .../General/TritonToLinalg/dotScaled_fp8.mlir | 2 +- .../General/TritonToLinalg/dot_fractal.mlir | 12 +++++----- .../General/TritonToLinalg/dot_int4.mlir | 2 +- ...d_one_hot_gather_after_max_with_index.mlir | 2 +- .../TritonToLinalg/fp_to_fp_scalar_rtne.mlir | 2 +- .../General/TritonToLinalg/full_row_copy.mlir | 2 +- .../General/TritonToLinalg/histogram.mlir | 6 ++--- .../TritonToLinalg/hoist_broadcast.mlir | 2 +- .../TritonToLinalg/hoist_broadcast2.mlir | 2 +- .../index_select_simd_bool.mlir | 4 ++-- .../TritonToLinalg/isin_by_search_kernel.mlir | 2 +- .../General/TritonToLinalg/mask_nop.mlir | 4 ++-- .../TritonToLinalg/multi_input_reduce.mlir | 2 +- .../General/TritonToLinalg/parse_custom.mlir | 2 +- .../General/TritonToLinalg/parse_select.mlir | 2 +- .../pointer_bitcast_byte_buffer.mlir | 4 ++-- .../TritonToLinalg/scalar_masked_load.mlir | 6 ++--- .../TritonToLinalg/scalar_pointer_select.mlir | 2 +- .../tensor_pointer_descriptor_loop.mlir | 2 +- .../test_device_print_offset_rewrite.mlir | 2 +- .../TritonToLinalg/test_implicit_permute.mlir | 2 +- .../test_inttoptr_offset_reset.mlir | 2 +- .../test_load_out_of_bound.mlir | 4 ++-- .../TritonToLinalg/test_load_with_other.mlir | 2 +- .../test_make_block_ptr_boundary_size.mlir | 4 ++-- .../TritonToLinalg/test_nd2nz_block_ptr.mlir | 2 +- .../TritonToLinalg/test_parse_atomic_rmw.mlir | 2 +- .../test_parse_descriptor_gather.mlir | 6 ++--- .../test_parse_descriptor_reduce.mlir | 2 +- .../test_parse_descriptor_scatter.mlir | 2 +- .../TritonToLinalg/test_parse_div.mlir | 2 +- .../TritonToLinalg/test_parse_fptosi.mlir | 2 +- .../test_parse_select_dense_const.mlir | 2 +- .../TritonToLinalg/test_parse_sub.mlir | 2 +- .../test_tensor_descriptor.mlir | 2 +- .../test_while_loop_index_tensor.mlir | 2 +- .../LoadBroadcastConverter.mlir | 4 ++-- .../StoreBroadcastConverter.mlir | 2 +- .../TritonToUnstructure/indirect_store.mlir | 2 +- .../mixed_axis_load_mask.mlir | 6 ++--- 64 files changed, 108 insertions(+), 108 deletions(-) diff --git a/third_party/ascend/unittest/Conversion/950PR/DiscreteMaskAccess/loop_mask_fallback.mlir b/third_party/ascend/unittest/Conversion/950PR/DiscreteMaskAccess/loop_mask_fallback.mlir index f082be9473..e7aedf8132 100644 --- a/third_party/ascend/unittest/Conversion/950PR/DiscreteMaskAccess/loop_mask_fallback.mlir +++ b/third_party/ascend/unittest/Conversion/950PR/DiscreteMaskAccess/loop_mask_fallback.mlir @@ -1,5 +1,5 @@ // RUN: triton-opt '--discrete-mask-access-conversion=compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file --verify-each %s | FileCheck %s --check-prefix=MASK -// RUN: triton-opt '--discrete-mask-access-conversion=compile-on-910-95=true compile-mode=simd_simt_template' '--triton-to-unstructure=compile-on-910-95=true compile-mode=simd_simt_template' '--triton-to-linalg=compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file --verify-each %s | FileCheck %s --check-prefix=LOWER --implicit-check-not=unrealized_conversion_cast --implicit-check-not=RuntimeLoopMask +// RUN: triton-opt '--discrete-mask-access-conversion=compile-on-910-95=true compile-mode=simd_simt_template' '--triton-to-unstructure=compile-on-910-95=true compile-mode=simd_simt_template' '--triton-to-linalg-incubated=compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file --verify-each %s | FileCheck %s --check-prefix=LOWER --implicit-check-not=unrealized_conversion_cast --implicit-check-not=RuntimeLoopMask // A suffix is outside the prefix rewrite, but is still an analyzable range. // Replacing the iter_arg by its init would incorrectly forget earlier bounds. diff --git a/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/chunk_coalescing.mlir b/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/chunk_coalescing.mlir index 3fb7a2ca1e..98a94016ae 100644 --- a/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/chunk_coalescing.mlir +++ b/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/chunk_coalescing.mlir @@ -1,5 +1,5 @@ // RUN: triton-opt %s --triton-to-unstructure='compile-on-910-95=true compile-mode=simd_simt_template' \ -// RUN: --triton-to-linalg='compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file \ +// RUN: --triton-to-linalg-incubated='compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file \ // RUN: | FileCheck %s // ----- diff --git a/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/indirect_load_rewrite.mlir b/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/indirect_load_rewrite.mlir index 1430910ec4..1f2bc4895b 100644 --- a/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/indirect_load_rewrite.mlir +++ b/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/indirect_load_rewrite.mlir @@ -1,9 +1,9 @@ // RUN: triton-opt %s --triton-to-unstructure='compile-on-910-95=true compile-mode=simd_simt_template' \ -// RUN: --triton-to-linalg='compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file \ +// RUN: --triton-to-linalg-incubated='compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file \ // RUN: | FileCheck %s // RUN: triton-opt %s --mlir-print-ir-after-all \ // RUN: --triton-to-unstructure='compile-on-910-95=true compile-mode=simd_simt_template' \ -// RUN: --triton-to-linalg='compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file 2>&1 \ +// RUN: --triton-to-linalg-incubated='compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file 2>&1 \ // RUN: | FileCheck %s --check-prefix=SLS-INSPECT // The static stride-4 AddPtr below is deliberately inspected but cannot be @@ -19,7 +19,7 @@ // SLS-INSPECT: IR Dump After Canonicalizer // SLS-INSPECT: tt.func public @addptr_stride4_1d // SLS-INSPECT: tt.load {{.*}} {InspectedByStridedLoadStoreRewrite} -// SLS-INSPECT: IR Dump After TritonToLinalg +// SLS-INSPECT: IR Dump After TritonToLinalgIncubated // SLS-INSPECT: func.func @addptr_stride4_1d // SLS-INSPECT-NOT: call @triton_indirect_load // SLS-INSPECT: memref.copy @@ -30,7 +30,7 @@ // SLS-INSPECT: IR Dump After {{.*}}LayoutMemoryCompatibilityPass{{.*}} // SLS-INSPECT: tt.func public @addptr_stride4_masked_single_tile // SLS-INSPECT: ascend.indirect_load {{.*}} {RewrittenByStridedLoadStoreRewrite, isVolatile = false} -// SLS-INSPECT: IR Dump After TritonToLinalg +// SLS-INSPECT: IR Dump After TritonToLinalgIncubated // SLS-INSPECT: func.func @addptr_stride4_masked_single_tile // SLS-INSPECT: call @triton_indirect_load diff --git a/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/sin_cos_simt_route.mlir b/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/sin_cos_simt_route.mlir index bbc54f74cd..284e5a7c7f 100644 --- a/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/sin_cos_simt_route.mlir +++ b/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/sin_cos_simt_route.mlir @@ -1,5 +1,5 @@ -// RUN: triton-opt %s --triton-to-linalg='compile-on-910-95=true' --split-input-file | FileCheck %s --check-prefixes=ON-CHECK-LABEL,ON-CHECK-SAME -// RUN: triton-opt %s --triton-to-linalg='compile-on-910-95=false' --split-input-file | FileCheck %s --check-prefixes=OFF-CHECK-LABEL,OFF-CHECK-SAME +// RUN: triton-opt %s --triton-to-linalg-incubated='compile-on-910-95=true' --split-input-file | FileCheck %s --check-prefixes=ON-CHECK-LABEL,ON-CHECK-SAME +// RUN: triton-opt %s --triton-to-linalg-incubated='compile-on-910-95=false' --split-input-file | FileCheck %s --check-prefixes=OFF-CHECK-LABEL,OFF-CHECK-SAME // isSIMTOp() routes math.sin / math.cos on f16/f32 tensor inputs to the SIMT // template. Downstream justification (Ascend 950 RegBase normalize, enable-high- diff --git a/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/strided_axis_coalescing.mlir b/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/strided_axis_coalescing.mlir index ea352fa6cd..330d3cd6c0 100644 --- a/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/strided_axis_coalescing.mlir +++ b/third_party/ascend/unittest/Conversion/950PR/TritonToLinalg/strided_axis_coalescing.mlir @@ -1,5 +1,5 @@ // RUN: triton-opt %s --triton-to-unstructure='compile-on-910-95=true compile-mode=simd_simt_template' \ -// RUN: --triton-to-linalg='compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file \ +// RUN: --triton-to-linalg-incubated='compile-on-910-95=true compile-mode=simd_simt_template' --split-input-file \ // RUN: | FileCheck %s // ----- diff --git a/third_party/ascend/unittest/Conversion/General/DiscreteMaskAccess/loop_prefix_mask.mlir b/third_party/ascend/unittest/Conversion/General/DiscreteMaskAccess/loop_prefix_mask.mlir index f94d6dc1f9..9631bec9b5 100644 --- a/third_party/ascend/unittest/Conversion/General/DiscreteMaskAccess/loop_prefix_mask.mlir +++ b/third_party/ascend/unittest/Conversion/General/DiscreteMaskAccess/loop_prefix_mask.mlir @@ -1,5 +1,5 @@ // RUN: triton-opt --discrete-mask-access-conversion --split-input-file --verify-each %s | FileCheck %s --check-prefix=PREFIX -// RUN: triton-opt --triton-control-flow-opt --discrete-mask-access-conversion --triton-to-unstructure --triton-to-linalg --split-input-file --verify-each %s | FileCheck %s --check-prefix=LINALG --implicit-check-not=unrealized_conversion_cast --implicit-check-not=indirect_load +// RUN: triton-opt --triton-control-flow-opt --discrete-mask-access-conversion --triton-to-unstructure --triton-to-linalg-incubated --split-input-file --verify-each %s | FileCheck %s --check-prefix=LINALG --implicit-check-not=unrealized_conversion_cast --implicit-check-not=indirect_load // Growing limits must not reopen lanes closed by a preceding iteration. // PREFIX-LABEL: tt.func public @prefix_history diff --git a/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/AllocMultiCache/Inner-scope-alloc-to-tensor-clone.mlir b/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/AllocMultiCache/Inner-scope-alloc-to-tensor-clone.mlir index 64db68eb9a..cff17603c7 100644 --- a/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/AllocMultiCache/Inner-scope-alloc-to-tensor-clone.mlir +++ b/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/AllocMultiCache/Inner-scope-alloc-to-tensor-clone.mlir @@ -25,10 +25,10 @@ // CHECK-LABEL: func.func @test_alloc_to_tensor_single_consumer // Original alloc + to_tensor stay in producer block (orphaned). // CHECK: %[[ORIG_ALLOC:.*]] = memref.alloc() {ssbuffer.block_id = 9 : i32} : memref -// CHECK: %[[ORIG_TT:.*]] = bufferization.to_tensor %[[ORIG_ALLOC]] restrict writable {ssbuffer.block_id = 9 : i32} : memref to tensor +// CHECK: %[[ORIG_TT:.*]] = bufferization.to_tensor %[[ORIG_ALLOC]] restrict writable {ssbuffer.block_id = 9 : i32} : memref // Cloned alloc + to_tensor appear in consumer block, tagged with block 10. // CHECK: %[[CLONE_ALLOC:.*]] = memref.alloc() {ssbuffer.block_id = 10 : i32} : memref -// CHECK: %[[CLONE_TT:.*]] = bufferization.to_tensor %[[CLONE_ALLOC]] restrict writable {ssbuffer.block_id = 10 : i32} : memref to tensor +// CHECK: %[[CLONE_TT:.*]] = bufferization.to_tensor %[[CLONE_ALLOC]] restrict writable {ssbuffer.block_id = 10 : i32} : memref // linalg.fill's outs is rewired to the cloned to_tensor, NOT the original. // CHECK: linalg.fill {ssbuffer.block_id = 10 : i32} ins(%{{.+}} : f32) outs(%[[CLONE_TT]] : tensor) -> tensor // alloc + to_tensor must NOT go through the multi-buffer path: no UB @@ -39,15 +39,15 @@ // CHECK-LABEL: func.func @test_alloc_to_tensor_multi_consumer // Original alloc + to_tensor stay in producer block (orphaned). // CHECK: %[[B_ORIG_ALLOC:.*]] = memref.alloc() {ssbuffer.block_id = 9 : i32} : memref -// CHECK: %[[B_ORIG_TT:.*]] = bufferization.to_tensor %[[B_ORIG_ALLOC]] restrict writable {ssbuffer.block_id = 9 : i32} : memref to tensor +// CHECK: %[[B_ORIG_TT:.*]] = bufferization.to_tensor %[[B_ORIG_ALLOC]] restrict writable {ssbuffer.block_id = 9 : i32} : memref // Clone for block_id = 10 consumer. // CHECK: %[[B_CLONE_ALLOC_10:.*]] = memref.alloc() {ssbuffer.block_id = 10 : i32} : memref -// CHECK: %[[B_CLONE_TT_10:.*]] = bufferization.to_tensor %[[B_CLONE_ALLOC_10]] restrict writable {ssbuffer.block_id = 10 : i32} : memref to tensor +// CHECK: %[[B_CLONE_TT_10:.*]] = bufferization.to_tensor %[[B_CLONE_ALLOC_10]] restrict writable {ssbuffer.block_id = 10 : i32} : memref // block_id = 10 fill uses the block 10 clone, NOT the original. // CHECK: linalg.fill {ssbuffer.block_id = 10 : i32} ins(%{{.+}} : f32) outs(%[[B_CLONE_TT_10]] : tensor) -> tensor // Clone for block_id = 11 consumer. // CHECK: %[[B_CLONE_ALLOC_11:.*]] = memref.alloc() {ssbuffer.block_id = 11 : i32} : memref -// CHECK: %[[B_CLONE_TT_11:.*]] = bufferization.to_tensor %[[B_CLONE_ALLOC_11]] restrict writable {ssbuffer.block_id = 11 : i32} : memref to tensor +// CHECK: %[[B_CLONE_TT_11:.*]] = bufferization.to_tensor %[[B_CLONE_ALLOC_11]] restrict writable {ssbuffer.block_id = 11 : i32} : memref // block_id = 11 fill uses the block 11 clone, NOT the original or block 10 clone. // CHECK: linalg.fill {ssbuffer.block_id = 11 : i32} ins(%{{.+}} : f32) outs(%[[B_CLONE_TT_11]] : tensor) -> tensor diff --git a/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/AllocMultiCache/Inner-scope-whileop.mlir b/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/AllocMultiCache/Inner-scope-whileop.mlir index 82fc6b0898..51b9429889 100644 --- a/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/AllocMultiCache/Inner-scope-whileop.mlir +++ b/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/AllocMultiCache/Inner-scope-whileop.mlir @@ -19,10 +19,10 @@ // CHECK: ^bb0(%{{.*}}: tensor<128xf32>, %{{.*}}: i32): // Original alloc + to_tensor stay in producer block (orphaned, will be DCE'd). // CHECK-DAG: memref.alloc() {ssbuffer.block_id = 7 : i32} : memref<128xf32> -// CHECK-DAG: bufferization.to_tensor {{.*}}{ssbuffer.block_id = 7 : i32} : memref<128xf32> to tensor<128xf32> +// CHECK-DAG: bufferization.to_tensor {{.*}}{ssbuffer.block_id = 7 : i32} : memref<128xf32> // Cloned alloc + to_tensor appear in consumer block (block_id = 10). // CHECK-DAG: memref.alloc() {ssbuffer.block_id = 10 : i32} : memref<128xf32> -// CHECK-DAG: bufferization.to_tensor {{.*}}{ssbuffer.block_id = 10 : i32} : memref<128xf32> to tensor<128xf32> +// CHECK-DAG: bufferization.to_tensor {{.*}}{ssbuffer.block_id = 10 : i32} : memref<128xf32> // Consumer uses the cloned to_tensor at block_id = 10. // CHECK: arith.addf {{.*}}{ssbuffer.block_id = 10 : i32} : tensor<128xf32> // NO UB alloc, NO hivm.copy chain — dep was cloned, not multi-buffered. @@ -52,10 +52,10 @@ // CHECK: ^bb0(%{{.*}}: tensor<128xf32>, %{{.*}}: i32, %{{.*}}: i32): // Original alloc + to_tensor stay in producer block (orphaned). // CHECK-DAG: memref.alloc() {ssbuffer.block_id = 7 : i32} : memref<128xf32> -// CHECK-DAG: bufferization.to_tensor {{.*}}{ssbuffer.block_id = 7 : i32} : memref<128xf32> to tensor<128xf32> +// CHECK-DAG: bufferization.to_tensor {{.*}}{ssbuffer.block_id = 7 : i32} : memref<128xf32> // Cloned alloc + to_tensor appear in consumer block (block_id = 10). // CHECK-DAG: memref.alloc() {ssbuffer.block_id = 10 : i32} : memref<128xf32> -// CHECK-DAG: bufferization.to_tensor {{.*}}{ssbuffer.block_id = 10 : i32} : memref<128xf32> to tensor<128xf32> +// CHECK-DAG: bufferization.to_tensor {{.*}}{ssbuffer.block_id = 10 : i32} : memref<128xf32> // Consumer uses the cloned to_tensor at block_id = 10. // CHECK: arith.addf {{.*}}{ssbuffer.block_id = 10 : i32} : tensor<128xf32> // NO UB alloc, NO scf.if dispatch, NO hivm.copy — dep was cloned, not @@ -79,14 +79,14 @@ // CHECK-LABEL: func.func @test_while_mainloop_attr_on_op // Original alloc + to_tensor stay in producer block (orphaned). // CHECK: %[[C_ORIG_ALLOC:.*]] = memref.alloc() {ssbuffer.block_id = 7 : i32} : memref<64xf16> -// CHECK: %[[C_ORIG_TT:.*]] = bufferization.to_tensor %[[C_ORIG_ALLOC]] {ssbuffer.block_id = 7 : i32} : memref<64xf16> to tensor<64xf16> +// CHECK: %[[C_ORIG_TT:.*]] = bufferization.to_tensor %[[C_ORIG_ALLOC]] {ssbuffer.block_id = 7 : i32} : memref<64xf16> // Cloned alloc + to_tensor appear in BOTH consumer blocks (block_id = 11 in // the if-branch, block_id = 12 in the else-branch). // CHECK: %[[C_CLONE_ALLOC_11:.*]] = memref.alloc() {ssbuffer.block_id = 11 : i32} : memref<64xf16> -// CHECK: %[[C_CLONE_TT_11:.*]] = bufferization.to_tensor %[[C_CLONE_ALLOC_11]] {ssbuffer.block_id = 11 : i32} : memref<64xf16> to tensor<64xf16> +// CHECK: %[[C_CLONE_TT_11:.*]] = bufferization.to_tensor %[[C_CLONE_ALLOC_11]] {ssbuffer.block_id = 11 : i32} : memref<64xf16> // CHECK: arith.addf %[[C_CLONE_TT_11]], %[[C_CLONE_TT_11]] {ssbuffer.block_id = 11 : i32} : tensor<64xf16> // CHECK: %[[C_CLONE_ALLOC_12:.*]] = memref.alloc() {ssbuffer.block_id = 12 : i32} : memref<64xf16> -// CHECK: %[[C_CLONE_TT_12:.*]] = bufferization.to_tensor %[[C_CLONE_ALLOC_12]] {ssbuffer.block_id = 12 : i32} : memref<64xf16> to tensor<64xf16> +// CHECK: %[[C_CLONE_TT_12:.*]] = bufferization.to_tensor %[[C_CLONE_ALLOC_12]] {ssbuffer.block_id = 12 : i32} : memref<64xf16> // CHECK: arith.mulf %[[C_CLONE_TT_12]], %[[C_CLONE_TT_12]] {ssbuffer.block_id = 12 : i32} : tensor<64xf16> // NO UB alloc, NO consumer-side to_tensor readback — dep was cloned. // CHECK-NOT: memref.alloc() : memref<64xf16, #hivm.address_space> @@ -112,12 +112,12 @@ // CHECK-LABEL: func.func @test_while_outermost_id_priority // Original alloc + to_tensor stay at producer block (block_id = 8). // CHECK: %[[D_ORIG_ALLOC:.*]] = memref.alloc() {ssbuffer.block_id = 8 : i32} : memref<32xf32> -// CHECK: %[[D_ORIG_TT:.*]] = bufferization.to_tensor %[[D_ORIG_ALLOC]] {ssbuffer.block_id = 8 : i32} : memref<32xf32> to tensor<32xf32> +// CHECK: %[[D_ORIG_TT:.*]] = bufferization.to_tensor %[[D_ORIG_ALLOC]] {ssbuffer.block_id = 8 : i32} : memref<32xf32> // Cloned alloc + to_tensor appear at consumer block (block_id = 12). If the // outermost-id priority bug were present, no clone would be emitted (the dep // would be misclassified as same-block and silently skipped). // CHECK: %[[D_CLONE_ALLOC:.*]] = memref.alloc() {ssbuffer.block_id = 12 : i32} : memref<32xf32> -// CHECK: %[[D_CLONE_TT:.*]] = bufferization.to_tensor %[[D_CLONE_ALLOC]] {ssbuffer.block_id = 12 : i32} : memref<32xf32> to tensor<32xf32> +// CHECK: %[[D_CLONE_TT:.*]] = bufferization.to_tensor %[[D_CLONE_ALLOC]] {ssbuffer.block_id = 12 : i32} : memref<32xf32> // CHECK: arith.addf %[[D_CLONE_TT]], %[[D_CLONE_TT]] {ssbuffer.block_id = 12 : i32} : tensor<32xf32> // NO UB alloc, NO copy chain. // CHECK-NOT: memref.alloc() : memref<32xf32, #hivm.address_space> @@ -172,10 +172,10 @@ // CHECK-LABEL: func.func @test_while_counter_relocation // Original alloc + to_tensor stay at producer block (block_id = 5, orphaned). // CHECK: %[[G_ORIG_ALLOC:.*]] = memref.alloc() {ssbuffer.block_id = 5 : i32} : memref<16xf32> -// CHECK: %[[G_ORIG_TT:.*]] = bufferization.to_tensor %[[G_ORIG_ALLOC]] {ssbuffer.block_id = 5 : i32} : memref<16xf32> to tensor<16xf32> +// CHECK: %[[G_ORIG_TT:.*]] = bufferization.to_tensor %[[G_ORIG_ALLOC]] {ssbuffer.block_id = 5 : i32} : memref<16xf32> // Cloned alloc + to_tensor at consumer block (block_id = 50). // CHECK: %[[G_CLONE_ALLOC:.*]] = memref.alloc() {ssbuffer.block_id = 50 : i32} : memref<16xf32> -// CHECK: %[[G_CLONE_TT:.*]] = bufferization.to_tensor %[[G_CLONE_ALLOC]] {ssbuffer.block_id = 50 : i32} : memref<16xf32> to tensor<16xf32> +// CHECK: %[[G_CLONE_TT:.*]] = bufferization.to_tensor %[[G_CLONE_ALLOC]] {ssbuffer.block_id = 50 : i32} : memref<16xf32> // CHECK: arith.addf %[[G_CLONE_TT]], %[[G_CLONE_TT]] {ssbuffer.block_id = 50 : i32} : tensor<16xf32> // NO UB alloc, NO copy chain. // CHECK-NOT: memref.alloc() : memref<16xf32, #hivm.address_space> diff --git a/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/SplitDataflow/test_fixpipe_nz2nz.mlir b/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/SplitDataflow/test_fixpipe_nz2nz.mlir index c6223ca77a..a07582047d 100644 --- a/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/SplitDataflow/test_fixpipe_nz2nz.mlir +++ b/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/SplitDataflow/test_fixpipe_nz2nz.mlir @@ -10,7 +10,7 @@ // CHECK: hivm.hir.fixpipe {channel_split = true, ssbuffer.block_id = 3 : i32, ssbuffer.core_type = "CUBE", ssbuffer.intraDeps = [0 : i32, 1 : i32]} ins({{%.*}} : tensor<32x32xf32>) outs({{%.*}} : memref<32x32xf32, #hivm.address_space>) // CHECK: memref.memory_space_cast {{%.*}} {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE", ssbuffer.intraDeps = [0 : i32, 0 : i32]} : memref<32x32xf32, #hivm.address_space> to memref<32x32xf32> -// CHECK: bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf32> to tensor<32x32xf32> +// CHECK: bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf32> // CHECK-NOT: ssbuffer.main_loop // CHECK: return @@ -45,7 +45,7 @@ module attributes {hacc.target = #hacc.target<"Ascend950PR_9579">} { // CHECK: hivm.hir.fixpipe {pre_quant = #hivm.fixpipe_pre_quant_mode, ssbuffer.block_id = 3 : i32, ssbuffer.core_type = "CUBE", ssbuffer.intraDeps = [0 : i32, 1 : i32]} ins({{%.*}} : tensor<32x32xf32>) outs({{%.*}} : memref<32x32xf16, #hivm.address_space>) // CHECK: memref.memory_space_cast {{%.*}} {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE", ssbuffer.intraDeps = [0 : i32, 0 : i32]} : memref<32x32xf16, #hivm.address_space> to memref<32x32xf16> -// CHECK: bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf16> to tensor<32x32xf16> +// CHECK: bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf16> // CHECK-NOT: ssbuffer.main_loop // CHECK-NOT: arith.truncf @@ -83,7 +83,7 @@ module attributes {hacc.target = #hacc.target<"Ascend950PR_9579">} { // CHECK-NOT: channel_split // CHECK: memref.memory_space_cast {{%.*}} {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE", ssbuffer.intraDeps = [0 : i32, 0 : i32]} : memref<32x32xf16, #hivm.address_space> to memref<32x32xf16> -// CHECK: bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf16> to tensor<32x32xf16> +// CHECK: bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf16> // CHECK: return @@ -118,7 +118,7 @@ module attributes {hacc.target = #hacc.target<"Ascend950PR_9579">} { // CHECK: hivm.hir.fixpipe {channel_split = true, ssbuffer.block_id = 3 : i32, ssbuffer.core_type = "CUBE", ssbuffer.intraDeps = [0 : i32, 1 : i32]} ins([[MM1:%.*]] : tensor<32x32xf32>) outs({{%.*}} : memref<32x32xf32, #hivm.address_space>) // CHECK: memref.memory_space_cast {{%.*}} {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE", ssbuffer.intraDeps = [0 : i32, 0 : i32]} : memref<32x32xf32, #hivm.address_space> to memref<32x32xf32> -// CHECK: [[TO_TENSOR:%.*]] = bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf32> to tensor<32x32xf32> +// CHECK: [[TO_TENSOR:%.*]] = bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf32> // Verify: the consumer matmul uses [[TO_TENSOR]] as input and [[MM1]] as init // CHECK: linalg.matmul {{.*}} ins([[TO_TENSOR]], {{%.*}} : tensor<32x32xf32>, tensor<32x32xf32>) outs([[MM1]] : tensor<32x32xf32>) @@ -155,7 +155,7 @@ module attributes {hacc.target = #hacc.target<"Ascend950PR_9579">} { // CHECK: hivm.hir.fixpipe {channel_split = true, ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} ins([[MM1:%.*]] : tensor<32x32xf32>) outs({{%.*}} : memref<32x32xf32, #hivm.address_space>) // CHECK: memref.memory_space_cast {{%.*}} {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf32, #hivm.address_space> to memref<32x32xf32> -// CHECK: [[TO_TENSOR:%.*]] = bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf32> to tensor<32x32xf32> +// CHECK: [[TO_TENSOR:%.*]] = bufferization.to_tensor {{%.*}} restrict writable {ssbuffer.block_id = 5 : i32, ssbuffer.core_type = "CUBE"} : memref<32x32xf32> // Verify: the consumer matmul uses [[TO_TENSOR]] as input and [[MM1]] as init // CHECK: linalg.matmul {{.*}} ins([[TO_TENSOR]], {{%.*}} : tensor<32x32xf32>, tensor<32x32xf32>) outs([[MM1]] : tensor<32x32xf32>) diff --git a/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/test-remove-attrs.mlir b/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/test-remove-attrs.mlir index d457212e72..5d4abe5b78 100644 --- a/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/test-remove-attrs.mlir +++ b/third_party/ascend/unittest/Conversion/General/DynamicCVPipeline/test-remove-attrs.mlir @@ -8,7 +8,7 @@ module { // CHECK-NOT: ssbuffer.block_id %memref = memref.alloc() {ssbuffer.core_type = "CUBE", ssbuffer.block_id = 1 : i32} : memref<1024x1024xf32> - // CHECK: bufferization.to_tensor %{{.*}} : memref<1024x1024xf32> to tensor<1024x1024xf32> + // CHECK: bufferization.to_tensor %{{.*}} : memref<1024x1024xf32> // CHECK-NOT: ssbuffer.core_type %tensor = bufferization.to_tensor %memref {ssbuffer.core_type = "CUBE"} : memref<1024x1024xf32> to tensor<1024x1024xf32> return diff --git a/third_party/ascend/unittest/Conversion/General/TritonAscendAllPass/conv1d.mlir b/third_party/ascend/unittest/Conversion/General/TritonAscendAllPass/conv1d.mlir index ffe0134e14..79f6e44cdc 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonAscendAllPass/conv1d.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonAscendAllPass/conv1d.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-structured '--discrete-mask-access-conversion=compile-on-910-95=False compile-mode=simd' '--triton-to-unstructure=compile-on-910-95=False compile-mode=simd' --triton-to-hivm --triton-to-hfusion --triton-to-llvm --bubble-up-operation --triton-to-structured '--triton-to-linalg=compile-mode=simd' --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-structured '--discrete-mask-access-conversion=compile-on-910-95=False compile-mode=simd' '--triton-to-unstructure=compile-on-910-95=False compile-mode=simd' --triton-to-hivm --triton-to-hfusion --triton-to-llvm --bubble-up-operation --triton-to-structured '--triton-to-linalg-incubated=compile-mode=simd' --split-input-file %s | FileCheck %s module attributes {hacc.target = #hacc.target<"Ascend910B2">} { tt.func public @triton_conv1d_2d_kernel(%input_ptr: !tt.ptr {tt.divisibility = 16 : i32}, %weight_ptr: !tt.ptr {tt.divisibility = 16 : i32}, %bias_ptr: !tt.ptr {tt.divisibility = 16 : i32}, %output_ptr: !tt.ptr {tt.divisibility = 16 : i32}) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonAscendAllPass/conv2d.mlir b/third_party/ascend/unittest/Conversion/General/TritonAscendAllPass/conv2d.mlir index 2cf3f1c9f7..bb6013428f 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonAscendAllPass/conv2d.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonAscendAllPass/conv2d.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-structured '--discrete-mask-access-conversion=compile-on-910-95=False compile-mode=simd' '--triton-to-unstructure=compile-on-910-95=False compile-mode=simd' --triton-to-hivm --triton-to-hfusion --triton-to-llvm --bubble-up-operation --triton-to-structured '--triton-to-linalg=compile-mode=simd' --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-structured '--discrete-mask-access-conversion=compile-on-910-95=False compile-mode=simd' '--triton-to-unstructure=compile-on-910-95=False compile-mode=simd' --triton-to-hivm --triton-to-hfusion --triton-to-llvm --bubble-up-operation --triton-to-structured '--triton-to-linalg-incubated=compile-mode=simd' --split-input-file %s | FileCheck %s module attributes {hacc.target = #hacc.target<"Ascend910B2">} { tt.func public @triton_conv2d_3d_kernel(%input_ptr: !tt.ptr {tt.divisibility = 16 : i32}, %weight_ptr: !tt.ptr {tt.divisibility = 16 : i32}, %output_ptr: !tt.ptr {tt.divisibility = 16 : i32}) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/d007_per_axis_structured_opaque_e2e.mlir b/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/d007_per_axis_structured_opaque_e2e.mlir index 2b6dd3ab00..8cef5031df 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/d007_per_axis_structured_opaque_e2e.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/d007_per_axis_structured_opaque_e2e.mlir @@ -1,5 +1,5 @@ // RUN: triton-opt --triton-control-flow-opt %s -verify-each | FileCheck %s --check-prefix=CFO -// RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure --triton-to-linalg %s -verify-each | FileCheck %s --check-prefix=E2E --implicit-check-not='!tt.ptr' --implicit-check-not=unrealized_conversion_cast --implicit-check-not=PointerDescriptorBoundary --implicit-check-not=PointerDescriptorRebuild --implicit-check-not=PointerDescriptorOffsetForm --implicit-check-not=PointerDescriptorStructuredAxes +// RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure --triton-to-linalg-incubated %s -verify-each | FileCheck %s --check-prefix=E2E --implicit-check-not='!tt.ptr' --implicit-check-not=unrealized_conversion_cast --implicit-check-not=PointerDescriptorBoundary --implicit-check-not=PointerDescriptorRebuild --implicit-check-not=PointerDescriptorOffsetForm --implicit-check-not=PointerDescriptorStructuredAxes module attributes {hacc.target = #hacc.target<"Ascend910B2">} { // A rank-1 affine range plus a uniform scalar offset remains one diff --git a/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/scope_pointer_downstream.mlir b/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/scope_pointer_downstream.mlir index c36a5a0f3b..b882acae83 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/scope_pointer_downstream.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/scope_pointer_downstream.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure --triton-to-linalg --split-input-file %s -verify-each | FileCheck %s --check-prefix=E2E --implicit-check-not='!tt.ptr' --implicit-check-not=unrealized_conversion_cast --implicit-check-not=PointerDescriptorBoundary --implicit-check-not=PointerDescriptorRebuild --implicit-check-not=PointerDescriptorOffsetForm --implicit-check-not=PointerDescriptorStructuredAxes +// RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure --triton-to-linalg-incubated --split-input-file %s -verify-each | FileCheck %s --check-prefix=E2E --implicit-check-not='!tt.ptr' --implicit-check-not=unrealized_conversion_cast --implicit-check-not=PointerDescriptorBoundary --implicit-check-not=PointerDescriptorRebuild --implicit-check-not=PointerDescriptorOffsetForm --implicit-check-not=PointerDescriptorStructuredAxes module attributes {hacc.target = #hacc.target<"Ascend910B2">} { tt.func public @scope_block_ptr_load_store(%base: !tt.ptr, %output: !tt.ptr) { diff --git a/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/tensor_ptr_different_base_invalid.mlir b/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/tensor_ptr_different_base_invalid.mlir index 6348111db2..2cdda19a94 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/tensor_ptr_different_base_invalid.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonControlFlowOpt/tensor_ptr_different_base_invalid.mlir @@ -1,7 +1,7 @@ // RUN: triton-opt --triton-control-flow-opt %s -verify-each | FileCheck %s --check-prefix=CFO // RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure %s -verify-each | FileCheck %s --check-prefix=T2U --implicit-check-not='tensor.extract {{.*}} : tensor<4x!tt.ptr>' // RUN: triton-opt --triton-control-flow-opt '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd_simt_template' %s -verify-each | FileCheck %s --check-prefix=T2U --implicit-check-not='tensor.extract {{.*}} : tensor<4x!tt.ptr>' -// RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure --triton-to-linalg %s -verify-each | FileCheck %s --check-prefix=LINALG --implicit-check-not='!tt.ptr' --implicit-check-not=unrealized_conversion_cast --implicit-check-not=PointerDescriptorBoundary --implicit-check-not=PointerDescriptorRebuild --implicit-check-not=PointerDescriptorOffsetForm +// RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure --triton-to-linalg-incubated %s -verify-each | FileCheck %s --check-prefix=LINALG --implicit-check-not='!tt.ptr' --implicit-check-not=unrealized_conversion_cast --implicit-check-not=PointerDescriptorBoundary --implicit-check-not=PointerDescriptorRebuild --implicit-check-not=PointerDescriptorOffsetForm module attributes {hacc.target = #hacc.target<"Ascend910B2">} { tt.func public @if_tensor_ptr_different_base( diff --git a/third_party/ascend/unittest/Conversion/General/TritonToGraph/graph-optimize-load-store-lowering.mlir b/third_party/ascend/unittest/Conversion/General/TritonToGraph/graph-optimize-load-store-lowering.mlir index bce6549ed7..70ec3fc99a 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToGraph/graph-optimize-load-store-lowering.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToGraph/graph-optimize-load-store-lowering.mlir @@ -1,6 +1,6 @@ // RUN: triton-opt %s --verify-each '-graph-optimize=rule-mask=1' -o - | FileCheck %s --check-prefix=GRAPH -// RUN: triton-opt %s --verify-each '-graph-optimize=rule-mask=1' -triton-to-linalg -o /dev/null -// RUN: triton-opt %s --verify-each '-graph-optimize=rule-mask=1' -triton-to-linalg -o - | FileCheck %s --check-prefix=LOWER +// RUN: triton-opt %s --verify-each '-graph-optimize=rule-mask=1' -triton-to-linalg-incubated -o /dev/null +// RUN: triton-opt %s --verify-each '-graph-optimize=rule-mask=1' -triton-to-linalg-incubated -o - | FileCheck %s --check-prefix=LOWER // Keep a lowering regression on the unified, type-changing layout rewrite. // The original access is tensor<2x3>; the graph pass must emit the equivalent diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/atomic_rmw.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/atomic_rmw.mlir index f3f36e5dcc..efdda99e8c 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/atomic_rmw.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/atomic_rmw.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg="named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated="named-ops=True" --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @matmul_atomic_add // CHECK-NOT: GenericAtomicRMW // CHECK: tensor.extract_slice diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/atomic_rmw_block.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/atomic_rmw_block.mlir index 9eaa677edb..684a6699a4 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/atomic_rmw_block.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/atomic_rmw_block.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt -allow-unregistered-dialect --triton-to-linalg="named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt -allow-unregistered-dialect --triton-to-linalg-incubated="named-ops=True" --split-input-file %s | FileCheck %s module attributes {hacc.target = #hacc.target<"Ascend910B2">} { tt.func public @moe_align_block_size_stage4(%arg0: !tt.ptr {tt.divisibility = 16 : i32} , %arg1: !tt.ptr {tt.divisibility = 16 : i32} , %arg2: !tt.ptr {tt.divisibility = 16 : i32} , %arg3: !tt.ptr {tt.divisibility = 16 : i32} , %arg4: !tt.ptr {tt.divisibility = 16 : i32} , %arg5: i32) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/clamp_fp8.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/clamp_fp8.mlir index b258c9bbd3..b30efd892e 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/clamp_fp8.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/clamp_fp8.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt '--discrete-mask-access-conversion=compile-on-910-95=True compile-mode=simd' --triton-to-annotation '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd' --triton-to-hivm --triton-to-hfusion --triton-to-llvm --bubble-up-operation '--triton-to-linalg=global-kernel=false named-ops=True enable-nd2nz-on-vector=False compile-on-910-95=True compile-mode=simd' --split-input-file %s | FileCheck %s +// RUN: triton-opt '--discrete-mask-access-conversion=compile-on-910-95=True compile-mode=simd' --triton-to-annotation '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd' --triton-to-hivm --triton-to-hfusion --triton-to-llvm --bubble-up-operation '--triton-to-linalg-incubated=global-kernel=false named-ops=True enable-nd2nz-on-vector=False compile-on-910-95=True compile-mode=simd' --split-input-file %s | FileCheck %s module { tt.func public @tt_clamp_3d(%arg0: !tt.ptr {tt.divisibility = 16 : i32}, %arg1: !tt.ptr {tt.divisibility = 16 : i32}, %arg2: !tt.ptr {tt.divisibility = 16 : i32}, %arg3: !tt.ptr {tt.divisibility = 16 : i32}) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_macro_op.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_macro_op.mlir index de181f49b0..f2b868182c 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_macro_op.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_macro_op.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // Test hivm.hir.custom_macro survives triton-to-linalg with sync attrs preserved. // CHECK-LABEL: func.func @test_custom_macro_op diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_op_pointer_splat.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_op_pointer_splat.mlir index 49e891d58a..117e93b5a5 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_op_pointer_splat.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_op_pointer_splat.mlir @@ -1,5 +1,5 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s -// RUN: triton-opt --canonicalize --triton-reorder-broadcast --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s +// RUN: triton-opt --canonicalize --triton-reorder-broadcast --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // Start at the TD -> HIVM boundary: no distributed dialect is needed in TA. // A custom result used only by a load must not make its pointer input a diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_op_ptr.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_op_ptr.mlir index f0709459b0..da1ee6fa4a 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_op_ptr.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/custom_op_ptr.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // Test hivm.hir.custom with distributed type producing tensor results // The custom op result type is converted to memref, and load uses the remapped result diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/debug_nop.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/debug_nop.mlir index 4330fd9e4c..79af7ec44e 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/debug_nop.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/debug_nop.mlir @@ -2,8 +2,8 @@ // TRITON_DEBUG=1 and LLVM_EXTRACT_DI_LOCAL_VARIABLES=1 is set, with the original source location preserved // on each NOP. Without TRITON_DEBUG and LLVM_EXTRACT_DI_LOCAL_VARIABLES, no inline asm is emitted. // -// RUN: triton-opt --triton-to-linalg --mlir-print-debuginfo --split-input-file %s | FileCheck %s --check-prefix=NODEBUG -// RUN: env TRITON_DEBUG=1 LLVM_EXTRACT_DI_LOCAL_VARIABLES=1 triton-opt --triton-to-linalg --mlir-print-debuginfo --split-input-file %s | FileCheck %s --check-prefix=DEBUG +// RUN: triton-opt --triton-to-linalg-incubated --mlir-print-debuginfo --split-input-file %s | FileCheck %s --check-prefix=NODEBUG +// RUN: env TRITON_DEBUG=1 LLVM_EXTRACT_DI_LOCAL_VARIABLES=1 triton-opt --triton-to-linalg-incubated --mlir-print-debuginfo --split-input-file %s | FileCheck %s --check-prefix=DEBUG #loc = loc("test.py":6:0) #loc1 = loc("test.py":11:24) diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dotScaled_fp4.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dotScaled_fp4.mlir index a7852f24da..fb755d8881 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dotScaled_fp4.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dotScaled_fp4.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --pass-pipeline="builtin.module(triton-to-unstructure{compile-on-910-95=true compile-mode=simd},triton-to-linalg{compile-on-910-95=true enable-nd2nz-on-vector=false global-kernel=false named-ops=true compile-mode=simd})" --split-input-file %s | FileCheck %s +// RUN: triton-opt --pass-pipeline="builtin.module(triton-to-unstructure{compile-on-910-95=true compile-mode=simd},triton-to-linalg-incubated{compile-on-910-95=true enable-nd2nz-on-vector=false global-kernel=false named-ops=true compile-mode=simd})" --split-input-file %s | FileCheck %s module attributes {hacc.target = #hacc.target<"Ascend950PR_9579">} { tt.func public @_dot_scaled_test_fp4(%a_ptr: !tt.ptr, %a_scale_ptr: !tt.ptr, %b_ptr: !tt.ptr, %b_scale_ptr: !tt.ptr, %c_ptr: !tt.ptr) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dotScaled_fp8.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dotScaled_fp8.mlir index 70d5914a8e..a5f8f764ad 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dotScaled_fp8.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dotScaled_fp8.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --pass-pipeline="builtin.module(triton-to-unstructure{compile-on-910-95=true compile-mode=simd},triton-to-linalg{compile-on-910-95=true enable-nd2nz-on-vector=false global-kernel=false named-ops=true compile-mode=simd})" --split-input-file %s | FileCheck %s +// RUN: triton-opt --pass-pipeline="builtin.module(triton-to-unstructure{compile-on-910-95=true compile-mode=simd},triton-to-linalg-incubated{compile-on-910-95=true enable-nd2nz-on-vector=false global-kernel=false named-ops=true compile-mode=simd})" --split-input-file %s | FileCheck %s module { tt.func public @dot_scale_fp8_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32}, %arg1: i32 {tt.divisibility = 16 : i32}, %arg2: !tt.ptr {tt.divisibility = 16 : i32}, %arg3: !tt.ptr {tt.divisibility = 16 : i32}, %arg4: i32 {tt.divisibility = 16 : i32}, %arg5: !tt.ptr {tt.divisibility = 16 : i32}) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dot_fractal.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dot_fractal.mlir index 9b76f595af..80f4a4297d 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dot_fractal.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dot_fractal.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg="named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated="named-ops=True" --split-input-file %s | FileCheck %s module { tt.func public @case1_a_fractal(%arg0: !tt.ptr, %arg1: !tt.ptr, %arg2: !tt.ptr) attributes {noinline = false} { @@ -65,7 +65,7 @@ module { // CHECK-SAME: mix_mode = "mix" // CHECK: hivm.hir.convert_layout %{{.*}} output_shape [160, 320] {dstLayout = #hivm.data_layout, srcLayout = #hivm.data_layout} : (tensor<20x10x16x16xf16>) -> tensor<160x320xf16> // CHECK: %[[ACC:.*]] = tensor.empty() : tensor<160x80xf32> -// CHECK: linalg.matmul {input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xf16>, tensor<320x80xf16>) outs(%[[ACC]] : tensor<160x80xf32>) -> tensor<160x80xf32> +// CHECK: linalg.matmul {{.*}}input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xf16>, tensor<320x80xf16>) outs(%[[ACC]] : tensor<160x80xf32>) -> tensor<160x80xf32> // ND result: no ND->Fractal convert afterwards. // CHECK-NOT: hivm.hir.convert_layout @@ -152,7 +152,7 @@ module { // CHECK: hivm.hir.convert_layout %{{.*}} output_shape [160, 320] {dstLayout = #hivm.data_layout, srcLayout = #hivm.data_layout} : (tensor<20x10x16x16xf16>) -> tensor<160x320xf16> // CHECK: hivm.hir.convert_layout %{{.*}} output_shape [320, 80] {dstLayout = #hivm.data_layout, srcLayout = #hivm.data_layout} : (tensor<5x20x16x16xf16>) -> tensor<320x80xf16> // CHECK: tensor.empty() : tensor<160x80xf32> -// CHECK: %[[MM:.*]] = linalg.matmul {input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xf16>, tensor<320x80xf16>) outs(%{{.*}} : tensor<160x80xf32>) -> tensor<160x80xf32> +// CHECK: %[[MM:.*]] = linalg.matmul {{.*}}input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xf16>, tensor<320x80xf16>) outs(%{{.*}} : tensor<160x80xf32>) -> tensor<160x80xf32> // fractal_c: ND [M,N] -> [N/16, M/16, 16, 16], carrying the f32 accumulator dtype. // CHECK: hivm.hir.convert_layout %[[MM]] output_shape [5, 10, 16, 16] {dstLayout = #hivm.data_layout, srcLayout = #hivm.data_layout} : (tensor<160x80xf32>) -> tensor<5x10x16x16xf32> @@ -225,7 +225,7 @@ module { // ND operands: no Fractal->ND convert is emitted before the matmul. // CHECK-NOT: hivm.hir.convert_layout // CHECK: tensor.empty() : tensor<160x80xf32> -// CHECK: %[[MM:.*]] = linalg.matmul {input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xf16>, tensor<320x80xf16>) outs(%{{.*}} : tensor<160x80xf32>) -> tensor<160x80xf32> +// CHECK: %[[MM:.*]] = linalg.matmul {{.*}}input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xf16>, tensor<320x80xf16>) outs(%{{.*}} : tensor<160x80xf32>) -> tensor<160x80xf32> // CHECK: hivm.hir.convert_layout %[[MM]] output_shape [5, 10, 16, 16] {dstLayout = #hivm.data_layout, srcLayout = #hivm.data_layout} : (tensor<160x80xf32>) -> tensor<5x10x16x16xf32> // ----- @@ -297,7 +297,7 @@ module { // CHECK-SAME: mix_mode = "mix" // CHECK: hivm.hir.convert_layout %{{.*}} output_shape [160, 320] {dstLayout = #hivm.data_layout, srcLayout = #hivm.data_layout} : (tensor<40x10x16x8xf32>) -> tensor<160x320xf32> // CHECK: tensor.empty() : tensor<160x80xf32> -// CHECK: linalg.matmul {input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xf32>, tensor<320x80xf32>) outs(%{{.*}} : tensor<160x80xf32>) -> tensor<160x80xf32> +// CHECK: linalg.matmul {{.*}}input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xf32>, tensor<320x80xf32>) outs(%{{.*}} : tensor<160x80xf32>) -> tensor<160x80xf32> // ----- @@ -393,6 +393,6 @@ module { // CHECK: hivm.hir.convert_layout %{{.*}} output_shape [320, 64] {dstLayout = #hivm.data_layout, srcLayout = #hivm.data_layout} : (tensor<2x10x32x32xi8>) -> tensor<320x64xi8> // i32 accumulator for an int8 input. // CHECK: tensor.empty() : tensor<160x64xi32> -// CHECK: %[[MM:.*]] = linalg.matmul {input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xi8>, tensor<320x64xi8>) outs(%{{.*}} : tensor<160x64xi32>) -> tensor<160x64xi32> +// CHECK: %[[MM:.*]] = linalg.matmul {{.*}}input_precision = "ieee"} ins(%{{.*}}, %{{.*}} : tensor<160x320xi8>, tensor<320x64xi8>) outs(%{{.*}} : tensor<160x64xi32>) -> tensor<160x64xi32> // fractal_c block stays [16, 16] even though the int8 input block is [16, 32]. // CHECK: hivm.hir.convert_layout %[[MM]] output_shape [4, 10, 16, 16] {dstLayout = #hivm.data_layout, srcLayout = #hivm.data_layout} : (tensor<160x64xi32>) -> tensor<4x10x16x16xi32> diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dot_int4.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dot_int4.mlir index c536393550..0e8b86176b 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dot_int4.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/dot_int4.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --pass-pipeline="builtin.module(auto-blockify{auto-blockify-size=1},triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},discrete-mask-access-conversion{compile-on-910-95=false compile-mode=simd_simt_template},triton-to-annotation,triton-to-unstructure{compile-on-910-95=false compile-mode=simd_simt_template},triton-to-hivm,triton-to-hfusion,triton-to-llvm,bubble-up-operation,triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},triton-to-linalg{compile-on-910-95=false enable-nd2nz-on-vector=false enable-select-analysis=true global-kernel=false named-ops=true compile-mode=simd_simt_template})" --split-input-file %s | FileCheck %s +// RUN: triton-opt --pass-pipeline="builtin.module(auto-blockify{auto-blockify-size=1},triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},discrete-mask-access-conversion{compile-on-910-95=false compile-mode=simd_simt_template},triton-to-annotation,triton-to-unstructure{compile-on-910-95=false compile-mode=simd_simt_template},triton-to-hivm,triton-to-hfusion,triton-to-llvm,bubble-up-operation,triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},triton-to-linalg-incubated{compile-on-910-95=false enable-nd2nz-on-vector=false enable-select-analysis=true global-kernel=false named-ops=true compile-mode=simd_simt_template})" --split-input-file %s | FileCheck %s module attributes {hacc.target = #hacc.target<"Ascend910B2">} { tt.func public @matmul_kernel(%arg0: !tt.ptr {tt.divisibility = 16 : i32}, %arg1: !tt.ptr {tt.divisibility = 16 : i32}, %arg2: !tt.ptr {tt.divisibility = 16 : i32}, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32, %arg8: i32) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/fold_one_hot_gather_after_max_with_index.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/fold_one_hot_gather_after_max_with_index.mlir index 88c3b0669c..c546e5bdae 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/fold_one_hot_gather_after_max_with_index.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/fold_one_hot_gather_after_max_with_index.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg="named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated="named-ops=True" --split-input-file %s | FileCheck %s module { func.func @fold_one_hot_gather_after_max_with_index(%logits: tensor<2x4xf32>, %indices: tensor<2x4xi32>, %mask: tensor<2x4xi1>) -> tensor<2x1xf32> { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/fp_to_fp_scalar_rtne.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/fp_to_fp_scalar_rtne.mlir index fae3e530d9..af75b0e9ae 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/fp_to_fp_scalar_rtne.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/fp_to_fp_scalar_rtne.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-hfusion --triton-to-linalg %s | FileCheck %s +// RUN: triton-opt --triton-to-hfusion --triton-to-linalg-incubated %s | FileCheck %s // CHECK-LABEL: func.func @scalar_fp32_to_fp8_rtne // CHECK-NOT: tt.fp_to_fp diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/full_row_copy.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/full_row_copy.mlir index 24154be3f2..e0a76cb8fc 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/full_row_copy.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/full_row_copy.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt "--triton-to-linalg=global-kernel=false" --split-input-file %s | FileCheck %s +// RUN: triton-opt "--triton-to-linalg-incubated=global-kernel=false" --split-input-file %s | FileCheck %s // Full rows recover a static copy; the tail keeps the exact original extent. // The condition and allocation must stay outside both row loops. diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/histogram.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/histogram.mlir index a81077e62c..d9572cc22d 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/histogram.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/histogram.mlir @@ -1,7 +1,7 @@ -// RUN: triton-opt "--triton-to-linalg=compile-on-910-95=true" --split-input-file %s | FileCheck %s +// RUN: triton-opt "--triton-to-linalg-incubated=compile-on-910-95=true" --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @test_histogram_i32 -// CHECK: %[[INPUT:.*]] = bufferization.to_tensor {{.*}} : memref<1024xi32> to tensor<1024xi32> +// CHECK: %[[INPUT:.*]] = bufferization.to_tensor {{.*}} : memref<1024xi32> // CHECK: %[[INIT:.*]] = linalg.fill ins(%{{.*}} : i32) outs(%{{.*}} : tensor<256xi32>) -> tensor<256xi32> // CHECK: %[[HIST:.*]] = hivm.hir.custom {gm_addr_args_indices = array, hivm.pipe = #hivm.pipe, hivm.tcore_type = #hivm.tcore_type, hivm.vf_mode = #hivm.vf_mode, symbol = "__builtin_histogram"} "__builtin_histogram" ins(%[[INPUT]], %{{.*}} : tensor<1024xi32>, i64) outs(%[[INIT]] : tensor<256xi32>) -> tensor<256xi32> // CHECK: bufferization.materialize_in_destination %[[HIST]] in writable {{.*}} : (tensor<256xi32>, memref<256xi32, strided<[1]>>) -> () @@ -23,7 +23,7 @@ module attributes {hacc.target = #hacc.target<"Ascend950PR_9579">} { // ----- // CHECK-LABEL: func.func @test_histogram_i16 -// CHECK: %[[INPUT:.*]] = bufferization.to_tensor {{.*}} : memref<16xi16> to tensor<16xi16> +// CHECK: %[[INPUT:.*]] = bufferization.to_tensor {{.*}} : memref<16xi16> // CHECK: %[[INIT:.*]] = linalg.fill ins(%{{.*}} : i16) outs(%{{.*}} : tensor<8xi16>) -> tensor<8xi16> // CHECK: %[[HIST:.*]] = hivm.hir.custom {gm_addr_args_indices = array, hivm.pipe = #hivm.pipe, hivm.tcore_type = #hivm.tcore_type, hivm.vf_mode = #hivm.vf_mode, symbol = "__builtin_histogram"} "__builtin_histogram" ins(%[[INPUT]], %{{.*}} : tensor<16xi16>, i64) outs(%[[INIT]] : tensor<8xi16>) -> tensor<8xi16> // CHECK: bufferization.materialize_in_destination %[[HIST]] in writable {{.*}} : (tensor<8xi16>, memref<8xi16, strided<[1]>>) -> () diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/hoist_broadcast.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/hoist_broadcast.mlir index efb3abf186..67e339fad9 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/hoist_broadcast.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/hoist_broadcast.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg="named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated="named-ops=True" --split-input-file %s | FileCheck %s module { tt.func public @copy_all_layer_kv_cache2(%arg0: !tt.ptr {tt.divisibility = 16 : i32}, %arg1: !tt.ptr {tt.divisibility = 16 : i32}, %arg2: !tt.ptr {tt.divisibility = 16 : i32}, %arg3: !tt.ptr {tt.divisibility = 16 : i32}) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/hoist_broadcast2.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/hoist_broadcast2.mlir index be3654d657..a94cd06958 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/hoist_broadcast2.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/hoist_broadcast2.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --pass-pipeline="builtin.module(auto-blockify{auto-blockify-size=1},triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},discrete-mask-access-conversion{compile-on-910-95=true compile-mode=simd},triton-to-annotation,triton-to-unstructure{compile-on-910-95=true force-scalarize-mode=false compile-mode=simd},triton-to-hivm,triton-to-hfusion,triton-to-llvm,bubble-up-operation{enable-aggressive-mode=true},triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},triton-to-linalg{compile-on-910-95=true enable-nd2nz-on-vector=false enable-select-analysis=true global-kernel=false named-ops=true compile-mode=simd})" --split-input-file %s | FileCheck %s +// RUN: triton-opt --pass-pipeline="builtin.module(auto-blockify{auto-blockify-size=1},triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},discrete-mask-access-conversion{compile-on-910-95=true compile-mode=simd},triton-to-annotation,triton-to-unstructure{compile-on-910-95=true force-scalarize-mode=false compile-mode=simd},triton-to-hivm,triton-to-hfusion,triton-to-llvm,bubble-up-operation{enable-aggressive-mode=true},triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},triton-to-linalg-incubated{compile-on-910-95=true enable-nd2nz-on-vector=false enable-select-analysis=true global-kernel=false named-ops=true compile-mode=simd})" --split-input-file %s | FileCheck %s module { tt.func public @_attn_bwd(%arg0: !tt.ptr {tt.divisibility = 16 : i32}, diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/index_select_simd_bool.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/index_select_simd_bool.mlir index 7f714e3bcc..4cc1ff5fc6 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/index_select_simd_bool.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/index_select_simd_bool.mlir @@ -1,9 +1,9 @@ -// RUN: triton-opt --triton-to-linalg="named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated="named-ops=True" --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @index_select_simd_bool // CHECK: memref.reinterpret_cast {{.*}} : memref to memref to tensor<2x8xi8> +// CHECK: bufferization.to_tensor {{.*}} {index_select_simd, was_bool_to_int8 = true} : memref<2x8xi8> module attributes {hacc.target = #hacc.target<"Ascend950PR_9579">} { tt.func public @index_select_simd_bool(%src: !tt.ptr, %dst: !tt.ptr) attributes {noinline = false} { %c0 = arith.constant 0 : index diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/isin_by_search_kernel.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/isin_by_search_kernel.mlir index 62e6cf19cb..25af919aee 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/isin_by_search_kernel.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/isin_by_search_kernel.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --discrete-mask-access-conversion --triton-to-unstructure --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --discrete-mask-access-conversion --triton-to-unstructure --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @isin_by_search_kernel // The one-element loop mask controls the load itself, including in SIMD mode. diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/mask_nop.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/mask_nop.mlir index 869b29a93c..61504d116a 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/mask_nop.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/mask_nop.mlir @@ -1,6 +1,6 @@ -// RUN: triton-opt --triton-to-linalg --mlir-print-debuginfo --split-input-file %s \ +// RUN: triton-opt --triton-to-linalg-incubated --mlir-print-debuginfo --split-input-file %s \ // RUN: | FileCheck %s --check-prefix=NODEBUG -// RUN: env LLVM_EXTRACT_DI_LOCAL_VARIABLES=1 triton-opt --triton-to-linalg \ +// RUN: env LLVM_EXTRACT_DI_LOCAL_VARIABLES=1 triton-opt --triton-to-linalg-incubated \ // RUN: --mlir-print-debuginfo --split-input-file %s | FileCheck %s --check-prefix=DEBUG #loc = loc("mask.py":1:0) diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/multi_input_reduce.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/multi_input_reduce.mlir index ebc7c90ea4..3b8d53db98 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/multi_input_reduce.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/multi_input_reduce.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated %s | FileCheck %s // CHECK-LABEL: func.func @three_input_reduce // CHECK-NOT: linalg.reduce diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/parse_custom.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/parse_custom.mlir index a9ef1c7cdb..5e793a70da 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/parse_custom.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/parse_custom.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-unstructure --triton-to-linalg="named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-unstructure --triton-to-linalg-incubated="named-ops=True" --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @parse_custom // CHECK: %4 = hivm.hir.custom // CHECK: %reinterpret_cast = memref.reinterpret_cast %4 to offset: [0], sizes: [32], strides: [1] : memref to memref<32xf16, strided<[1]>> diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/parse_select.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/parse_select.mlir index f844ca90d5..91bc9dc643 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/parse_select.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/parse_select.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s -verify-each 2>&1 | FileCheck %s --check-prefix=NOERR +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s -verify-each 2>&1 | FileCheck %s --check-prefix=NOERR // NOERR-NOT: parseSelect currently supports all-ones shape unless cond=i1 with dense constants // CHECK-LABEL: func.func public @triton_for_if_load diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/pointer_bitcast_byte_buffer.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/pointer_bitcast_byte_buffer.mlir index e20053d2af..cdb42949e4 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/pointer_bitcast_byte_buffer.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/pointer_bitcast_byte_buffer.mlir @@ -1,7 +1,7 @@ -// RUN: triton-opt --triton-to-unstructure --triton-to-linalg="named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-unstructure --triton-to-linalg-incubated="named-ops=True" --split-input-file %s | FileCheck %s // RUN: triton-opt --triton-to-unstructure --split-input-file %s | FileCheck %s --check-prefix=UNSTRUCTURE // RUN: triton-opt '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd_simt_template' --split-input-file %s | FileCheck %s --check-prefix=SIMT -// RUN: triton-opt '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd_simt_template' '--triton-to-linalg=compile-on-910-95=True compile-mode=simd_simt_template' --split-input-file %s | FileCheck %s --check-prefix=SIMT-LINALG +// RUN: triton-opt '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd_simt_template' '--triton-to-linalg-incubated=compile-on-910-95=True compile-mode=simd_simt_template' --split-input-file %s | FileCheck %s --check-prefix=SIMT-LINALG // CHECK-LABEL: func.func @widen_tensor_offset // CHECK: scf.for diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/scalar_masked_load.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/scalar_masked_load.mlir index bfbbaa796e..2e8a2f16ed 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/scalar_masked_load.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/scalar_masked_load.mlir @@ -1,6 +1,6 @@ -// RUN: triton-opt %s --triton-to-linalg -verify-each | FileCheck %s --implicit-check-not='tensor<' --implicit-check-not='arith.select' -// RUN: triton-opt %s --triton-to-linalg='named-ops=true' -verify-each | FileCheck %s --implicit-check-not='tensor<' --implicit-check-not='arith.select' -// RUN: sed 's/Ascend910B2/Ascend950DT_9582/' %s | triton-opt --triton-to-linalg='compile-on-910-95=true' -verify-each | FileCheck %s --implicit-check-not='tensor<' --implicit-check-not='arith.select' +// RUN: triton-opt %s --triton-to-linalg-incubated -verify-each | FileCheck %s --implicit-check-not='tensor<' --implicit-check-not='arith.select' +// RUN: triton-opt %s --triton-to-linalg-incubated='named-ops=true' -verify-each | FileCheck %s --implicit-check-not='tensor<' --implicit-check-not='arith.select' +// RUN: sed 's/Ascend910B2/Ascend950DT_9582/' %s | triton-opt --triton-to-linalg-incubated='compile-on-910-95=true' -verify-each | FileCheck %s --implicit-check-not='tensor<' --implicit-check-not='arith.select' // Scalar masked loads must keep their result scalar and only read memory in // the active branch. In particular, selecting the result of an unconditional diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/scalar_pointer_select.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/scalar_pointer_select.mlir index 6d8ac31dfd..c151ad55e8 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/scalar_pointer_select.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/scalar_pointer_select.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg %s -verify-each | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated %s -verify-each | FileCheck %s module attributes {hacc.target = #hacc.target<"Ascend910B2">} { tt.func public @scalar_pointer_select(%lhs: !tt.ptr, %rhs: !tt.ptr, %condition: i1) -> i64 { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/tensor_pointer_descriptor_loop.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/tensor_pointer_descriptor_loop.mlir index 46f4c7d81e..b81d59243e 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/tensor_pointer_descriptor_loop.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/tensor_pointer_descriptor_loop.mlir @@ -1,6 +1,6 @@ // RUN: triton-opt --triton-control-flow-opt %s -verify-each | FileCheck %s --check-prefix=CFO // RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure %s -verify-each | FileCheck %s --check-prefix=T2U -// RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure --bubble-up-operation --triton-to-linalg %s -verify-each | FileCheck %s --check-prefix=LINALG +// RUN: triton-opt --triton-control-flow-opt --triton-to-unstructure --bubble-up-operation --triton-to-linalg-incubated %s -verify-each | FileCheck %s --check-prefix=LINALG module attributes {hacc.target = #hacc.target<"Ascend910B2">} { tt.func public @tensor_pointer_descriptor_loop( diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_device_print_offset_rewrite.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_device_print_offset_rewrite.mlir index 6c7ec7efa1..6cb77f002d 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_device_print_offset_rewrite.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_device_print_offset_rewrite.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --pass-pipeline="builtin.module(auto-blockify{auto-blockify-size=1},triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},discrete-mask-access-conversion{compile-on-910-95=false compile-mode=simd_simt_template},triton-to-annotation,triton-to-unstructure{compile-on-910-95=false compile-mode=simd_simt_template},triton-to-hivm,triton-to-hfusion,triton-to-llvm,bubble-up-operation,triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},triton-to-linalg{compile-on-910-95=false enable-nd2nz-on-vector=false enable-select-analysis=true global-kernel=false named-ops=true compile-mode=simd_simt_template})" --split-input-file %s | FileCheck %s +// RUN: triton-opt --pass-pipeline="builtin.module(auto-blockify{auto-blockify-size=1},triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},discrete-mask-access-conversion{compile-on-910-95=false compile-mode=simd_simt_template},triton-to-annotation,triton-to-unstructure{compile-on-910-95=false compile-mode=simd_simt_template},triton-to-hivm,triton-to-hfusion,triton-to-llvm,bubble-up-operation,triton-to-structured{enable-mask-fallback-conversion=false optimize-dynamic-offset=false},triton-to-linalg-incubated{compile-on-910-95=false enable-nd2nz-on-vector=false enable-select-analysis=true global-kernel=false named-ops=true compile-mode=simd_simt_template})" --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @kernel_B8_dynamic_offset // CHECK: %[[PIDOFF:.*]] = arith.muli diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_implicit_permute.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_implicit_permute.mlir index e7593291d6..50a28e82ab 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_implicit_permute.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_implicit_permute.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt '--triton-to-linalg=global-kernel=False named-ops=True enable-nd2nz-on-vector=False enable-select-analysis=True compile-on-910-95=True' --split-input-file %s | FileCheck %s +// RUN: triton-opt '--triton-to-linalg-incubated=global-kernel=False named-ops=True enable-nd2nz-on-vector=False enable-select-analysis=True compile-on-910-95=True' --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @addptr_implicit_perm_load_2d( // CHECK: %[[IN_STRIDE0:[A-Za-z0-9_]+]] = arith.index_cast %arg6 : i32 to index diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_inttoptr_offset_reset.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_inttoptr_offset_reset.mlir index e1a752fe90..756c237749 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_inttoptr_offset_reset.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_inttoptr_offset_reset.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt "--triton-to-linalg=global-kernel=false named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt "--triton-to-linalg-incubated=global-kernel=false named-ops=True" --split-input-file %s | FileCheck %s // Test the positive static offset representation selected when a scalar // pointer reaches the layout-sensitive memref.copy boundary. diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_load_out_of_bound.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_load_out_of_bound.mlir index 05b526ed98..a16aa19c63 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_load_out_of_bound.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_load_out_of_bound.mlir @@ -1,10 +1,10 @@ -// RUN: triton-opt "--triton-to-linalg=global-kernel=false named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt "--triton-to-linalg-incubated=global-kernel=false named-ops=True" --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @_triton_mrope_forward // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32 // CHECK: %[[ALLOC_5:.*]] = memref.alloc() : memref<64xf32> // CHECK: linalg.fill ins(%[[CST]] : f32) outs(%[[ALLOC_5]] : memref<64xf32>) -// CHECK: bufferization.to_tensor %[[ALLOC_5]] restrict writable : memref<64xf32> to tensor<64xf32> +// CHECK: bufferization.to_tensor %[[ALLOC_5]] restrict writable : memref<64xf32> module attributes {hacc.target = #hacc.target<"Ascend910_9382">} { tt.func public @_triton_mrope_forward(%q_ptr: !tt.ptr {tt.divisibility = 16 : i32}, %k_ptr: !tt.ptr {tt.divisibility = 16 : i32}, %cos: !tt.ptr {tt.divisibility = 16 : i32}, %sin: !tt.ptr {tt.divisibility = 16 : i32}, %num_tokens: i32 {tt.divisibility = 16 : i32}) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_load_with_other.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_load_with_other.mlir index ca52e22123..ddcb21c3b4 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_load_with_other.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_load_with_other.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --discrete-mask-access-conversion "--triton-to-linalg=global-kernel=false named-ops=True" --split-input-file %s | FileCheck %s +// RUN: triton-opt --discrete-mask-access-conversion "--triton-to-linalg-incubated=global-kernel=false named-ops=True" --split-input-file %s | FileCheck %s // Case 1: other = uitofp(mask) -> select(mask, loaded, zeros) // CHECK-LABEL: func.func @load_other_mask diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_make_block_ptr_boundary_size.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_make_block_ptr_boundary_size.mlir index 81de97711f..e8bb580056 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_make_block_ptr_boundary_size.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_make_block_ptr_boundary_size.mlir @@ -1,5 +1,5 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s -// RUN: triton-opt --triton-to-linalg --split-input-file %s 2>&1 | FileCheck %s --check-prefix=WARN +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s 2>&1 | FileCheck %s --check-prefix=WARN // Unit tests for ConverterUtils::getBoundarySizes() (PR #1838, // "[TritonToLinalg](fix) boundary size Error for make_block_ptr"). diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_nd2nz_block_ptr.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_nd2nz_block_ptr.mlir index 8edd3bf152..bb88059015 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_nd2nz_block_ptr.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_nd2nz_block_ptr.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg=enable-nd2nz-on-vector=true %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated=enable-nd2nz-on-vector=true %s | FileCheck %s // CHECK-LABEL: func.func @test_nd2nz_block_ptr diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_atomic_rmw.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_atomic_rmw.mlir index dbe1f2636a..d90833f72b 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_atomic_rmw.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_atomic_rmw.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @test_parse_atomic_rmw // CHECK: %[[C8_I32:.*]] = arith.constant 8 : i32 diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_gather.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_gather.mlir index 6dbacfe07a..6b6b3d8896 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_gather.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_gather.mlir @@ -1,18 +1,18 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @tensor_descriptor_gather_rows // CHECK-NOT: tt.descriptor_gather // CHECK: %[[IDX_VIEW:.*]] = memref.reinterpret_cast %{{.*}} to offset: [0], sizes: [32], strides: [1] // CHECK: %[[IDX_ALLOC:.*]] = memref.alloc() : memref<32xi32> // CHECK: memref.copy %[[IDX_VIEW]], %[[IDX_ALLOC]] -// CHECK: %[[XOFFSETS:.*]] = bufferization.to_tensor %[[IDX_ALLOC]] restrict writable : memref<32xi32> to tensor<32xi32> +// CHECK: %[[XOFFSETS:.*]] = bufferization.to_tensor %[[IDX_ALLOC]] restrict writable : memref<32xi32> // CHECK: %[[OUT_ALLOC:.*]] = memref.alloc() : memref<32x32xf32> // CHECK: scf.for %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} // CHECK: %[[XOFFSET:.*]] = tensor.extract %[[XOFFSETS]][%{{.*}}] : tensor<32xi32> // CHECK: %[[DESC_VIEW:.*]] = memref.reinterpret_cast %{{.*}} to offset: [%{{.*}}], sizes: [1, 32], strides: [128, 1] // CHECK: %[[OUT_SUBVIEW:.*]] = memref.subview %[[OUT_ALLOC]][%{{.*}}, 0] [1, 32] [1, 1] // CHECK: memref.copy %{{.*}}, %{{.*}} -// CHECK: %[[OUT_TENSOR:.*]] = bufferization.to_tensor %[[OUT_ALLOC]] restrict writable : memref<32x32xf32> to tensor<32x32xf32> +// CHECK: %[[OUT_TENSOR:.*]] = bufferization.to_tensor %[[OUT_ALLOC]] restrict writable : memref<32x32xf32> // CHECK: bufferization.materialize_in_destination %[[OUT_TENSOR]] in writable %{{.*}} : (tensor<32x32xf32>, memref<32x32xf32, strided<[32, 1]>>) -> () module attributes {hacc.target = #hacc.target<"Ascend910B2">} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_reduce.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_reduce.mlir index e0b3aa62ba..c46a1c3ee8 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_reduce.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_reduce.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @kernel // CHECK: %[[SUBVIEW:.*]] = memref.subview %[[RC:.*]][%[[VAL_36:.*]], %[[VAL_39:.*]]] [%[[VAL_38:.*]], %[[VAL_41:.*]]] [1, 1] : memref<2x16xi32, strided<[?, 1], offset: ?>> to memref> diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_scatter.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_scatter.mlir index 725604afd4..de960127c0 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_scatter.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_descriptor_scatter.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @tensor_descriptor_scatter_rows_kernel // CHECK-NOT: tt.descriptor_scatter diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_div.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_div.mlir index adb5243fff..875f5fb913 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_div.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_div.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg=named-ops=false --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated=named-ops=false --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @test_parse_div // CHECK: %[[SUB:.*]] = linalg.generic diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_fptosi.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_fptosi.mlir index 448cd52ee2..3fe6d10cab 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_fptosi.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_fptosi.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg=named-ops=false --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated=named-ops=false --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @test_parse_fptosi // CHECK: %[[GENERIC:.*]] = linalg.generic diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_select_dense_const.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_select_dense_const.mlir index ecb5ba4efb..53da17d419 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_select_dense_const.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_select_dense_const.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @test_parse_select_dense_const // CHECK: %[[C32_I32:.*]] = arith.constant 32 : i32 diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_sub.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_sub.mlir index b9d1086358..1cffb0687e 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_sub.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_parse_sub.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // CHECK-LABEL: func.func @sub_offset_kernel diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_tensor_descriptor.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_tensor_descriptor.mlir index 70cd13a374..c04276507c 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_tensor_descriptor.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_tensor_descriptor.mlir @@ -1,5 +1,5 @@ // RUN: triton-opt --triton-rewrite-tensor-descriptor-to-pointer --split-input-file %s | FileCheck %s --check-prefix=CHECK -// RUN: triton-opt --triton-rewrite-tensor-descriptor-to-pointer --canonicalize "--triton-to-linalg=global-kernel=false named-ops=True" --split-input-file %s | FileCheck %s --check-prefix=CHECK-LINALG +// RUN: triton-opt --triton-rewrite-tensor-descriptor-to-pointer --canonicalize "--triton-to-linalg-incubated=global-kernel=false named-ops=True" --split-input-file %s | FileCheck %s --check-prefix=CHECK-LINALG // ----- diff --git a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_while_loop_index_tensor.mlir b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_while_loop_index_tensor.mlir index 097c0cf466..a3714143f5 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_while_loop_index_tensor.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToLinalg/test_while_loop_index_tensor.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt --triton-to-linalg %s | FileCheck %s +// RUN: triton-opt --triton-to-linalg-incubated %s | FileCheck %s // CHECK-LABEL: func.func @test_while_loop_index_tensor // CHECK: %[[C8:.*]] = arith.constant 8 : index diff --git a/third_party/ascend/unittest/Conversion/General/TritonToStructured/LoadBroadcastConverter.mlir b/third_party/ascend/unittest/Conversion/General/TritonToStructured/LoadBroadcastConverter.mlir index bdfb85ff16..15258f1a04 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToStructured/LoadBroadcastConverter.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToStructured/LoadBroadcastConverter.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt '--triton-to-structured=enable-mask-fallback-conversion=false optimize-dynamic-offset=true' --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt '--triton-to-structured=enable-mask-fallback-conversion=false optimize-dynamic-offset=true' --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // Tests for LoadBroadcastConverter: // Case 1: ptr broadcast src shape matches mask broadcast src shape → @@ -140,5 +140,5 @@ module { } // CHECK-LABEL: func.func @kernel_load_same_axis_broadcast -// CHECK: bufferization.to_tensor %{{.*}} restrict writable : memref<1x64xf32> to tensor<1x64xf32> +// CHECK: bufferization.to_tensor %{{.*}} restrict writable : memref<1x64xf32> // CHECK: linalg.broadcast ins(%{{.*}} : tensor<64xf32>) outs(%{{.*}} : tensor<256x64xf32>) dimensions = [0] diff --git a/third_party/ascend/unittest/Conversion/General/TritonToStructured/StoreBroadcastConverter.mlir b/third_party/ascend/unittest/Conversion/General/TritonToStructured/StoreBroadcastConverter.mlir index a8e03dcf0d..240bf7bd8c 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToStructured/StoreBroadcastConverter.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToStructured/StoreBroadcastConverter.mlir @@ -1,4 +1,4 @@ -// RUN: triton-opt '--triton-to-structured=enable-mask-fallback-conversion=false optimize-dynamic-offset=true' --triton-to-linalg --split-input-file %s | FileCheck %s +// RUN: triton-opt '--triton-to-structured=enable-mask-fallback-conversion=false optimize-dynamic-offset=true' --triton-to-linalg-incubated --split-input-file %s | FileCheck %s // Tests for StoreBroadcastConverter: when store.ptr is defined by a // tt.broadcast, every pointer along the broadcast axis points to the same diff --git a/third_party/ascend/unittest/Conversion/General/TritonToUnstructure/indirect_store.mlir b/third_party/ascend/unittest/Conversion/General/TritonToUnstructure/indirect_store.mlir index ef1d679898..098d33f740 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToUnstructure/indirect_store.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToUnstructure/indirect_store.mlir @@ -1,5 +1,5 @@ // RUN: triton-opt --triton-to-structured '--discrete-mask-access-conversion=compile-on-910-95=True compile-mode=simd_simt_template' '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd_simt_template' %s --split-input-file | FileCheck %s -// RUN: triton-opt '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd_simt_template' '--triton-to-linalg=compile-on-910-95=True compile-mode=simd_simt_template' --split-input-file %s | FileCheck %s --check-prefix=LINALG +// RUN: triton-opt '--triton-to-unstructure=compile-on-910-95=True compile-mode=simd_simt_template' '--triton-to-linalg-incubated=compile-on-910-95=True compile-mode=simd_simt_template' --split-input-file %s | FileCheck %s --check-prefix=LINALG // tt.store -> ascend.indirect_store tt.func public @triton_indirect_store_kernel(%arg0: !tt.ptr, %arg1: !tt.ptr, %arg2: !tt.ptr, %arg3: i32) attributes {noinline = false} { diff --git a/third_party/ascend/unittest/Conversion/General/TritonToUnstructure/mixed_axis_load_mask.mlir b/third_party/ascend/unittest/Conversion/General/TritonToUnstructure/mixed_axis_load_mask.mlir index a35eda5c64..13a182cb2c 100644 --- a/third_party/ascend/unittest/Conversion/General/TritonToUnstructure/mixed_axis_load_mask.mlir +++ b/third_party/ascend/unittest/Conversion/General/TritonToUnstructure/mixed_axis_load_mask.mlir @@ -1,7 +1,7 @@ // RUN: triton-opt %s --triton-to-unstructure --verify-each | FileCheck %s --check-prefix=T2U -// RUN: triton-opt %s --triton-to-unstructure --bubble-up-operation --triton-to-linalg --verify-each | FileCheck %s --check-prefix=LOWER -// RUN: triton-opt %s --triton-to-unstructure='compile-on-910-95=true compile-mode=simd_simt_template' --bubble-up-operation --triton-to-linalg='compile-on-910-95=true compile-mode=simd_simt_template' --verify-each | FileCheck %s --check-prefix=LOWER -// RUN: sed 's/%mask, %other/%mask/' %s | triton-opt --triton-to-unstructure --bubble-up-operation --triton-to-linalg --verify-each | FileCheck %s --check-prefix=LOWER +// RUN: triton-opt %s --triton-to-unstructure --bubble-up-operation --triton-to-linalg-incubated --verify-each | FileCheck %s --check-prefix=LOWER +// RUN: triton-opt %s --triton-to-unstructure='compile-on-910-95=true compile-mode=simd_simt_template' --bubble-up-operation --triton-to-linalg-incubated='compile-on-910-95=true compile-mode=simd_simt_template' --verify-each | FileCheck %s --check-prefix=LOWER +// RUN: sed 's/%mask, %other/%mask/' %s | triton-opt --triton-to-unstructure --bubble-up-operation --triton-to-linalg-incubated --verify-each | FileCheck %s --check-prefix=LOWER // RUN: sed 's/cmpi slt, %x/cmpi eq, %x/' %s | triton-opt --triton-to-unstructure --verify-each | FileCheck %s --check-prefix=KEEP // RUN: sed 's/cmpi slt, %x/cmpi sge, %x/' %s | triton-opt --triton-to-unstructure --verify-each | FileCheck %s --check-prefix=KEEP From e572c01fe54d8657333fb5f263705b0a6ac3dfc0 Mon Sep 17 00:00:00 2001 From: zhongsanming Date: Wed, 18 Nov 2026 11:44:50 +0800 Subject: [PATCH 3/7] [Ascend] Consume FLIR TritonMemoryAccess Remove the duplicate Ascend TritonMemoryAccess implementation and point all Ascend consumers at incubated/TritonMemoryAccess from FLIR. --- third_party/ascend/AscendNPU-IR | 1 + third_party/ascend/CMakeLists.txt | 4 +- .../LoadStoreMaskAnalysis.h | 219 ---- .../TritonMemoryAccess/MemoryAccessTags.h | 41 - .../TritonMemoryAccess/OpFoldResultUtils.h | 95 -- .../StridedLoadStoreRewrite.h | 2 +- .../include/Utils/InterleaveOptimization.h | 2 +- third_party/ascend/include/Utils/Utils.h | 2 +- third_party/ascend/lib/CMakeLists.txt | 5 +- .../CMakeLists.txt | 2 +- .../lib/TritonMemoryAccess/CMakeLists.txt | 25 - .../lib/TritonMemoryAccess/MaskAnalysis.cpp | 1063 ----------------- .../TritonMemoryAccess/OpFoldResultUtils.cpp | 463 ------- .../ascend/lib/TritonToGraph/CMakeLists.txt | 2 +- .../StridedLoadStoreRewrite.cpp | 4 +- .../Rules/IndependentAxisTensorizeRule.cpp | 2 +- .../Rules/PersistentTaskStripMiningRule.cpp | 2 +- .../lib/TritonToUnstructure/CMakeLists.txt | 2 +- .../UnstructureConversionPass.cpp | 2 +- third_party/ascend/lib/Utils/CMakeLists.txt | 4 +- 20 files changed, 19 insertions(+), 1923 deletions(-) create mode 120000 third_party/ascend/AscendNPU-IR delete mode 100644 third_party/ascend/include/TritonMemoryAccess/LoadStoreMaskAnalysis.h delete mode 100644 third_party/ascend/include/TritonMemoryAccess/MemoryAccessTags.h delete mode 100644 third_party/ascend/include/TritonMemoryAccess/OpFoldResultUtils.h delete mode 100644 third_party/ascend/lib/TritonMemoryAccess/CMakeLists.txt delete mode 100644 third_party/ascend/lib/TritonMemoryAccess/MaskAnalysis.cpp delete mode 100644 third_party/ascend/lib/TritonMemoryAccess/OpFoldResultUtils.cpp diff --git a/third_party/ascend/AscendNPU-IR b/third_party/ascend/AscendNPU-IR new file mode 120000 index 0000000000..214d7f7805 --- /dev/null +++ b/third_party/ascend/AscendNPU-IR @@ -0,0 +1 @@ +/home/serephus/dev/work/flagos/AscendNPU-IR \ No newline at end of file diff --git a/third_party/ascend/CMakeLists.txt b/third_party/ascend/CMakeLists.txt index 49a621a83a..3c81c9f628 100644 --- a/third_party/ascend/CMakeLists.txt +++ b/third_party/ascend/CMakeLists.txt @@ -25,6 +25,9 @@ include_directories(${ASCENDNPU_IR_BINARY_DIR}/bishengir/include) # Tablegen'd f # Ascend libraries so DiscreteMaskAccessConversion can link it; its Phase-A # Ascend dependencies are attached after add_subdirectory(lib) below. if(FLAGTREE_BACKEND STREQUAL "ascend") + add_subdirectory( + ${PROJECT_SOURCE_DIR}/third_party/flir/lib/TritonMemoryAccess + ${PROJECT_BINARY_DIR}/third_party/flir/lib/TritonMemoryAccess) add_subdirectory( ${PROJECT_SOURCE_DIR}/third_party/flir/include/incubated/Conversion/TritonToLinalgIncubated ${PROJECT_BINARY_DIR}/third_party/flir/include/incubated/Conversion/TritonToLinalgIncubated) @@ -45,7 +48,6 @@ if(FLAGTREE_BACKEND STREQUAL "ascend") BiShengIRScopeDialect TritonAscendIR TritonStructuredIR - TritonMemoryAccessAnalysis TritonToStructured TritonToUnstructure TritonToGraph diff --git a/third_party/ascend/include/TritonMemoryAccess/LoadStoreMaskAnalysis.h b/third_party/ascend/include/TritonMemoryAccess/LoadStoreMaskAnalysis.h deleted file mode 100644 index fa53449b52..0000000000 --- a/third_party/ascend/include/TritonMemoryAccess/LoadStoreMaskAnalysis.h +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * Copyright (c) Microsoft Corporation, Meta Platforms. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_MEMORY_ACCESS_LOAD_STORE_MASK_ANALYSIS_H -#define TRITON_MEMORY_ACCESS_LOAD_STORE_MASK_ANALYSIS_H - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include -#include - -namespace mlir { - -// this class helps build Operations -class OpBuilder; - -namespace triton { -// use to decode the pattern in a mask used for load and store - -enum class MaskPosition { Head, Tail, Middle, Unknown }; - -class MaskState { -public: - OpFoldResult start; - OpFoldResult end; - SmallVector dims; - SmallVector offsets; - OpFoldResult scalar; - - int64_t getRank() const { - assert(dims.size() == offsets.size() && "dims and offsets rank mismatch!"); - return dims.size(); - } - - MaskPosition getMaskPosition(llvm::ArrayRef &tensorShape) { - if (getRank() != tensorShape.size()) { - return MaskPosition::Unknown; - } - - bool isHead = true; - int dynIndex = -1; - - for (int i = 0; i < getRank(); ++i) { - auto offsetVal = mlir::getConstantIntValue(offsets[i]); - if (!offsetVal.has_value() || offsetVal.value() != 0) { - isHead = false; - if (dynIndex == -1) { - dynIndex = i; - } else { // temporarily support only one dyn dim - return MaskPosition::Unknown; - } - } - } - - if (isHead) { - return MaskPosition::Head; - } - - for (int i = 0; i < getRank(); ++i) { - auto dimVal = mlir::getConstantIntValue(dims[i]); - if (i == dynIndex) { - continue; - } - if (!dimVal.has_value() || dimVal.value() != tensorShape[i]) { - return MaskPosition::Unknown; - } - } - return MaskPosition::Middle; - } - - bool isEmpty() const { return getRank() == 0 && !scalar && !start && !end; } - - bool isMask() const { - return !start && !end && !scalar && dims.size() != 0 && offsets.size() != 0; - } - - bool isMemrefSubviewValid(Value source, OpBuilder &builder) const; - - // parse value recursively - LogicalResult parse(Value operand, const Location &loc, OpBuilder &builder); - - tensor::ExtractSliceOp getExtractSlice(Value source, const Location &loc, - OpBuilder &builder) const; - - tensor::ExtractSliceOp getExtractSlice(Value source, const Location &loc, - OpBuilder &builder, - SmallVector offsets, - SmallVector dims) const; - - tensor::InsertSliceOp getInsertSlice(Value source, Value dest, - const Location &loc, - OpBuilder &builder) const; - - tensor::InsertSliceOp getInsertSlice(Value source, Value dest, - const Location &loc, OpBuilder &builder, - SmallVector offsets, - SmallVector dims) const; - - memref::SubViewOp getSubview(Value source, const Location &loc, - OpBuilder &builder) const; - - void eraseInsertedOps(Operation *rawOp, PatternRewriter &rewriter); - -private: - LogicalResult addStateScalar(const MaskState &state, - const OpFoldResult scalar, const Location &loc, - OpBuilder &builder); - - LogicalResult addStates(const MaskState &lhsState, const MaskState &rhsState, - const Location &loc, OpBuilder &builder); - - LogicalResult subStateScalar(const MaskState &state, - const OpFoldResult scalar, const Location &loc, - OpBuilder &builder); - - // Only supports (range - scalar). - LogicalResult subStates(const MaskState &lhsState, const MaskState &rhsState, - const Location &loc, OpBuilder &builder); - - LogicalResult divStateScalar(const MaskState &state, - const OpFoldResult scalar, const Location &loc, - OpBuilder &builder); - - LogicalResult divStates(const MaskState &lhsState, const MaskState &rhsState, - const Location &loc, OpBuilder &builder); - - // Helper function to handle operator `and` both mask state - LogicalResult minStates(const MaskState &lhsState, const MaskState &rhsState, - const Location &loc, OpBuilder &builder); - - OpFoldResult clampToNonNegativeIndex(const OpFoldResult value, - const Location &loc, OpBuilder &builder, - bool clampDynamic = false) const; - - // Helper functions to parse values to populate MaskState - - LogicalResult parseConstant(arith::ConstantOp constOp, const Location &loc, - OpBuilder &builder); - - // Operand is an integer scalar - LogicalResult parseIntScalar(Value scalar, const Location &loc, - OpBuilder &builder); - - // TODO - LogicalResult parseAdd(arith::AddIOp addOp, const Location &loc, - OpBuilder &builder); - - // Operand is the result of subi (range - scalar). - LogicalResult parseSub(arith::SubIOp subOp, const Location &loc, - OpBuilder &builder); - - // operand is the result of divsi - LogicalResult parseDiv(arith::DivSIOp divOp, const Location &loc, - OpBuilder &builder); - - // Operand is the result of andi - LogicalResult parseAnd(arith::AndIOp andOp, const Location &loc, - OpBuilder &builder); - - // Operand is the result of cmpi, necessary method to fuse scalar, start and - // end into dims and offset - LogicalResult parseCmp(arith::CmpIOp cmpOp, const Location &loc, - OpBuilder &builder); - - // Operand is the result of select - LogicalResult parseSel(arith::SelectOp selOp, const Location &loc, - OpBuilder &builder); - - // Operand is the result of make_range - LogicalResult parseMakeRange(triton::MakeRangeOp rangeOp, const Location &loc, - OpBuilder &builder); - - // Operand is the result of broadcast - LogicalResult parseBroadcast(triton::BroadcastOp broadcastOp, - const Location &loc, OpBuilder &builder); - - // Operand is the result of splat - LogicalResult parseSplat(triton::SplatOp splatOp, const Location &loc, - OpBuilder &builder); - - // Operand is the result of expand_dims - LogicalResult parseExpandDims(triton::ExpandDimsOp expandDimsOp, - const Location &loc, OpBuilder &builder); - - // Operand is the result of insert - LogicalResult parseInsert(tensor::InsertOp insertOp, const Location &loc, - OpBuilder &builder); -}; - -std::optional runMaskAnalysis(Operation *op, OpBuilder &builder); - -} // namespace triton - -} // namespace mlir - -#endif // TRITON_MEMORY_ACCESS_LOAD_STORE_MASK_ANALYSIS_H diff --git a/third_party/ascend/include/TritonMemoryAccess/MemoryAccessTags.h b/third_party/ascend/include/TritonMemoryAccess/MemoryAccessTags.h deleted file mode 100644 index 6e47b1bf22..0000000000 --- a/third_party/ascend/include/TritonMemoryAccess/MemoryAccessTags.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_MEMORY_ACCESS_TAGS_H -#define TRITON_MEMORY_ACCESS_TAGS_H - -namespace mlir::triton::memory_access { - -inline constexpr const char *ImplicitPermuteHandledTAG = - "ImplicitPermuteHandled"; -inline constexpr const char *InspectedByStridedLoadStoreRewriteTAG = - "InspectedByStridedLoadStoreRewrite"; -inline constexpr const char *RewrittenByStridedLoadStoreRewriteTAG = - "RewrittenByStridedLoadStoreRewrite"; -inline constexpr const char *IATRuntimeExtentUnsignedMaskTAG = - "IATRuntimeExtentUnsignedMask"; -inline constexpr const char *PTSMRuntimeExtentUnsignedMaskTAG = - "PTSMRuntimeExtentUnsignedMask"; - -} // namespace mlir::triton::memory_access - -#endif // TRITON_MEMORY_ACCESS_TAGS_H diff --git a/third_party/ascend/include/TritonMemoryAccess/OpFoldResultUtils.h b/third_party/ascend/include/TritonMemoryAccess/OpFoldResultUtils.h deleted file mode 100644 index 3cb16a2128..0000000000 --- a/third_party/ascend/include/TritonMemoryAccess/OpFoldResultUtils.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_MEMORY_ACCESS_OP_FOLD_RESULT_UTILS_H -#define TRITON_MEMORY_ACCESS_OP_FOLD_RESULT_UTILS_H - -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Value.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallBitVector.h" -#include "llvm/Support/LogicalResult.h" - -#include - -namespace mlir { - -class OpBuilder; - -enum class IntegerExtensionKind { - Signed, - Unsigned, -}; - -FailureOr -castIntegerLike(OpBuilder &builder, Location loc, Value value, Type targetType, - IntegerExtensionKind extension = IntegerExtensionKind::Signed); - -std::optional getConstantOfAttr(const OpFoldResult &arg); - -/// Without an explicit result type, preserve equal types and widen signless -/// integers. Mixed index/integer operands require an explicit result type. -OpFoldResult addOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &builder, - Type resultType = {}); - -OpFoldResult subOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &builder); - -/// Uses the same result-type rules as addOpFoldResult. -OpFoldResult mulOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &builder, - Type resultType = {}); - -OpFoldResult divOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &builder); - -OpFoldResult remOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &builder); - -OpFoldResult minOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &builder); - -OpFoldResult maxOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &builder); - -std::optional getIntAttr(const OpFoldResult ofr); - -Value materializeValue(OpBuilder &builder, Location loc, OpFoldResult ofr); - -bool isZero(const OpFoldResult ofr); - -bool isOne(const OpFoldResult ofr); - -Value convertToIndexIfNeeded(Value input, const Location &loc, - OpBuilder &builder); - -RankedTensorType getExtractSlicedType(llvm::ArrayRef shape, - const llvm::SmallBitVector &droppedDims, - Type elemType); - -OpFoldResult getOpFoldResultOfLayoutInfo(Value value, OpBuilder &builder); - -} // namespace mlir - -#endif // TRITON_MEMORY_ACCESS_OP_FOLD_RESULT_UTILS_H diff --git a/third_party/ascend/include/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h b/third_party/ascend/include/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h index bbbb63cb16..2b145a3e59 100644 --- a/third_party/ascend/include/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h +++ b/third_party/ascend/include/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h @@ -23,7 +23,7 @@ #ifndef TRITON_TO_GRAPH_LEGACY_MEMORY_ACCESS_STRIDED_LOAD_STORE_REWRITE_H #define TRITON_TO_GRAPH_LEGACY_MEMORY_ACCESS_STRIDED_LOAD_STORE_REWRITE_H -#include "TritonMemoryAccess/MemoryAccessTags.h" +#include "incubated/TritonMemoryAccess/MemoryAccessTags.h" #include "mlir/IR/PatternMatch.h" #include "triton/Dialect/Triton/IR/Dialect.h" diff --git a/third_party/ascend/include/Utils/InterleaveOptimization.h b/third_party/ascend/include/Utils/InterleaveOptimization.h index 63478c1252..e5f5f888fd 100644 --- a/third_party/ascend/include/Utils/InterleaveOptimization.h +++ b/third_party/ascend/include/Utils/InterleaveOptimization.h @@ -22,7 +22,7 @@ #pragma once -#include "TritonMemoryAccess/LoadStoreMaskAnalysis.h" +#include "incubated/TritonMemoryAccess/LoadStoreMaskAnalysis.h" #include "ascend/include/Utils/Utils.h" #include "triton/Dialect/Triton/IR/Dialect.h" diff --git a/third_party/ascend/include/Utils/Utils.h b/third_party/ascend/include/Utils/Utils.h index c04abb575c..e74f67bec8 100644 --- a/third_party/ascend/include/Utils/Utils.h +++ b/third_party/ascend/include/Utils/Utils.h @@ -23,7 +23,7 @@ #ifndef TRITONNPU_UTILS_UTILS_H #define TRITONNPU_UTILS_UTILS_H -#include "TritonMemoryAccess/OpFoldResultUtils.h" +#include "incubated/TritonMemoryAccess/OpFoldResultUtils.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/SCF.h" diff --git a/third_party/ascend/lib/CMakeLists.txt b/third_party/ascend/lib/CMakeLists.txt index f17cb15c6d..19db07bd20 100644 --- a/third_party/ascend/lib/CMakeLists.txt +++ b/third_party/ascend/lib/CMakeLists.txt @@ -1,6 +1,5 @@ add_subdirectory(AutoBlockify) add_subdirectory(Dialect) -add_subdirectory(TritonMemoryAccess) add_subdirectory(Utils) add_subdirectory(TritonToStructured) add_subdirectory(TritonToGraph) @@ -26,8 +25,8 @@ if(TRITON_ENABLE_COVERAGE_HITEST) TritonToStructured TritonToUnstructure MLIRTritonNPUUtils # from Utils - TritonMemoryAccessCore - TritonMemoryAccessAnalysis + FlirTritonMemoryAccessCore + FlirTritonMemoryAccessAnalysis TritonAscendIR # from Dialect/TritonAscend/IR TritonStructuredIR # from Dialect/TritonStructured/IR AutoBlockify diff --git a/third_party/ascend/lib/DiscreteMaskAccessConversion/CMakeLists.txt b/third_party/ascend/lib/DiscreteMaskAccessConversion/CMakeLists.txt index 9e45c0ed29..e218001bc5 100644 --- a/third_party/ascend/lib/DiscreteMaskAccessConversion/CMakeLists.txt +++ b/third_party/ascend/lib/DiscreteMaskAccessConversion/CMakeLists.txt @@ -13,5 +13,5 @@ add_triton_library(DiscreteMaskAccessConversion MLIRSupport TritonIR TritonToLinalgIncubated - TritonMemoryAccessAnalysis + FlirTritonMemoryAccessAnalysis ) diff --git a/third_party/ascend/lib/TritonMemoryAccess/CMakeLists.txt b/third_party/ascend/lib/TritonMemoryAccess/CMakeLists.txt deleted file mode 100644 index 73d789bec2..0000000000 --- a/third_party/ascend/lib/TritonMemoryAccess/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -add_triton_library(TritonMemoryAccessCore - OpFoldResultUtils.cpp - - LINK_LIBS PUBLIC - MLIRArithDialect - MLIRDialectUtils - MLIRIR - MLIRSupport -) - -add_triton_library(TritonMemoryAccessAnalysis - MaskAnalysis.cpp - - LINK_LIBS PUBLIC - MLIRArithDialect - MLIRDialectUtils - MLIRIR - MLIRMemRefDialect - MLIRSCFDialect - MLIRSupport - MLIRTensorDialect - MLIRTransforms - TritonIR - TritonMemoryAccessCore -) diff --git a/third_party/ascend/lib/TritonMemoryAccess/MaskAnalysis.cpp b/third_party/ascend/lib/TritonMemoryAccess/MaskAnalysis.cpp deleted file mode 100644 index 8b635c99e3..0000000000 --- a/third_party/ascend/lib/TritonMemoryAccess/MaskAnalysis.cpp +++ /dev/null @@ -1,1063 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "TritonMemoryAccess/LoadStoreMaskAnalysis.h" -#include "TritonMemoryAccess/MemoryAccessTags.h" -#include "TritonMemoryAccess/OpFoldResultUtils.h" - -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Operation.h" -#include "mlir/Support/LLVM.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Debug.h" -#include -#include - -#define DEBUG_TYPE "mask-analysis" - -namespace mlir { - -namespace triton { - -namespace { - -template -std::optional runMaskAnalysisImpl(MemAccOpTy op, - OpBuilder &builder) { - auto mask = op.getMask(); - if (!mask) { - return std::nullopt; - } - - PatternRewriter::InsertionGuard insertGuard(builder); - builder.setInsertionPoint(op); - - MaskState mstate; - if (mstate.parse(mask, op.getLoc(), builder).failed()) { - return std::nullopt; - } - return mstate; -} - -bool isZeroMaskConstant(const OpFoldResult &value) { - // Keep the actual mlir::isZeroInteger implementation used by the source - // baseline: a zero IntegerAttr or a Value matched as a constant integer. - if (auto constant = getConstantIntValue(value)) - return *constant == 0; - return false; -} - -static Value unwrapRuntimeExtentUnsignedOperand(Value operand, - const Location &loc, - OpBuilder &builder) { - if (auto extend = operand.getDefiningOp()) - return extend.getIn(); - - auto splat = operand.getDefiningOp(); - if (!splat) - return Value(); - auto extend = splat.getSrc().getDefiningOp(); - if (!extend) - return Value(); - - auto resultType = dyn_cast(splat.getType()); - if (!resultType || !isa(extend.getIn().getType())) - return Value(); - auto narrowedType = - RankedTensorType::get(resultType.getShape(), extend.getIn().getType(), - resultType.getEncoding()); - return builder.create(loc, narrowedType, extend.getIn()); -} - -// Follow an i1 tensor to its init only when all corresponding loop edges -// forward the same value. A while may permute condition/result slots, so do -// not assume its before and after argument numbers are interchangeable. -Value getInvariantLoopMaskInit(BlockArgument argument) { - Operation *parent = argument.getOwner()->getParentOp(); - if (auto loop = dyn_cast(parent)) { - if (argument == loop.getInductionVar()) - return {}; - unsigned slot = argument.getArgNumber() - 1; - auto yield = cast(loop.getBody()->getTerminator()); - return yield.getOperand(slot) == argument ? loop.getInitArgs()[slot] - : Value(); - } - if (auto loop = dyn_cast(parent)) { - auto condition = loop.getConditionOp(); - auto yield = loop.getYieldOp(); - BlockArgument before, after; - if (argument.getOwner() == &loop.getBefore().front()) { - before = argument; - after = dyn_cast(yield.getOperand(before.getArgNumber())); - if (!after || after.getOwner() != &loop.getAfter().front() || - condition.getArgs()[after.getArgNumber()] != before) - return {}; - } else { - after = argument; - before = - dyn_cast(condition.getArgs()[after.getArgNumber()]); - if (!before || before.getOwner() != &loop.getBefore().front() || - yield.getOperand(before.getArgNumber()) != after) - return {}; - } - return loop.getInits()[before.getArgNumber()]; - } - return {}; -} - -} // namespace - -OpFoldResult MaskState::clampToNonNegativeIndex(const OpFoldResult value, - const Location &loc, - OpBuilder &builder, - bool clampDynamic) const { - if (auto cst = getConstantIntValue(value)) { - return builder.getIndexAttr(std::max(0, *cst)); - } - - // Only minStates needs the dynamic clamp (disjoint intersection -> negative - // extent); the cmp paths are already non-negative, and clamping them - // regressed the atomic max/min UT. - if (clampDynamic) { - return maxOpFoldResult(value, builder.getIndexAttr(0), loc, builder); - } - return value; -} - -LogicalResult MaskState::parse(Value operand, const Location &loc, - OpBuilder &builder) { - Type operandType = operand.getType(); - if (isa(operandType) || operandType.isIndex()) { - return parseIntScalar(operand, loc, builder); - } - - if (auto blockArgument = dyn_cast(operand)) { - auto parentOp = blockArgument.getOwner()->getParentOp(); - if (auto loopOp = dyn_cast(parentOp)) { - auto type = dyn_cast(operand.getType()); - if (type && type.getElementType().isInteger(1)) { - Value init = getInvariantLoopMaskInit(blockArgument); - return init ? parse(init, loc, builder) : failure(); - } - OpOperand *initArgOperand = loopOp.getTiedLoopInit(blockArgument); - if (initArgOperand) { - if (!isa(operand.getType())) - return failure(); - - // Parse the init value to get the base range structure - if (failed(parse(initArgOperand->get(), loc, builder))) - return failure(); - - // Only scf.for loops are handled - auto forOp = dyn_cast(parentOp); - if (!forOp) - return failure(); - - unsigned slot = blockArgument.getArgNumber() - 1; - if (slot >= forOp.getYieldedValues().size()) - return failure(); - Value yielded = forOp.getYieldedValues()[slot]; - if (yielded == operand) { - // iter_arg is unchanged across iterations: the init value is the - // current value, so the parsed state needs no adjustment. - return success(); - } - - // Detect yield == iter_arg + const_tensor (or const_tensor + iter_arg). - auto addOp = yielded.getDefiningOp(); - if (!addOp) - return failure(); - Value incrementValue; - if (addOp.getLhs() == operand) - incrementValue = addOp.getRhs(); - else if (addOp.getRhs() == operand) - incrementValue = addOp.getLhs(); - else - return failure(); - - // The increment must be a splat integer constant tensor or a scalar - // integer constant (i.e. iteration-independent). - auto constOp = incrementValue.getDefiningOp(); - if (!constOp) - return failure(); - int64_t increment = 0; - if (auto denseAttr = dyn_cast(constOp.getValue())) { - if (!denseAttr.isSplat() || - !isa(denseAttr.getElementType())) - return failure(); - increment = denseAttr.getSplatValue().getInt(); - } else if (auto intAttr = dyn_cast(constOp.getValue())) { - increment = intAttr.getInt(); - } else { - return failure(); - } - - // The iteration count n = (iv - lb) / step relates the induction - // variable to the per-iteration increment: current = init + n*delta. - // lb/step may be dynamic; cast to index and compute at runtime. - FailureOr ivIndex = castIntegerLike( - builder, loc, forOp.getInductionVar(), builder.getIndexType()); - if (failed(ivIndex)) - return failure(); - - Value iterCount = *ivIndex; - - // iterCount = iv - lb (handle dynamic or constant lb) - auto lb = getConstantIntValue(forOp.getLowerBound()); - if (!lb || *lb != 0) { - FailureOr lbIndex = castIntegerLike( - builder, loc, forOp.getLowerBound(), builder.getIndexType()); - if (failed(lbIndex)) - return failure(); - iterCount = builder.create(loc, iterCount, *lbIndex); - } - - // iterCount = (iv - lb) / step (handle dynamic or constant step) - auto stepCst = getConstantIntValue(forOp.getStep()); - if (stepCst && *stepCst == 0) - return failure(); - if (!stepCst || *stepCst != 1) { - FailureOr stepIndex = castIntegerLike( - builder, loc, forOp.getStep(), builder.getIndexType()); - if (failed(stepIndex)) - return failure(); - iterCount = - builder.create(loc, iterCount, *stepIndex); - } - - OpFoldResult offset = - mulOpFoldResult(iterCount, builder.getIndexAttr(increment), loc, - builder, builder.getIndexType()); - if (!offset) - return failure(); - - if (this->start && this->end) { - this->start = addOpFoldResult(this->start, offset, loc, builder, - builder.getIndexType()); - this->end = addOpFoldResult(this->end, offset, loc, builder, - builder.getIndexType()); - if (!this->start || !this->end) - return failure(); - } else if (this->scalar) { - this->scalar = addOpFoldResult(this->scalar, offset, loc, builder, - builder.getIndexType()); - if (!this->scalar) - return failure(); - } else { - return failure(); - } - return success(); - } - } - } - - auto definingOp = operand.getDefiningOp(); - if (!definingOp) - return failure(); - - LLVM_DEBUG({ - llvm::dbgs() << "[MaskState]==> parse op\n" - << *definingOp << "\n[MaskState]<==\n"; - }); - return TypeSwitch(definingOp) - .Case( - [&](auto op) { return this->parseConstant(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseAdd(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseSub(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseAnd(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseCmp(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseMakeRange(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseBroadcast(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseSplat(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseExpandDims(op, loc, builder); }) - .Case( - [&](auto op) { return this->parse(op.getIn(), loc, builder); }) - .Case( - [&](auto op) { return this->parseDiv(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseSel(op, loc, builder); }) - .Case( - [&](auto op) { return this->parseInsert(op, loc, builder); }) - .Default([&](Operation *op) { return failure(); }); -} - -// extractSlice -tensor::ExtractSliceOp MaskState::getExtractSlice(Value source, - const Location &loc, - OpBuilder &builder) const { - auto sourceRType = cast(source.getType()); - SmallVector strides(getRank(), builder.getIndexAttr(1)); - - auto dstRType = tensor::ExtractSliceOp::inferResultType(sourceRType, offsets, - dims, strides); - return builder.create(loc, dstRType, source, offsets, - dims, strides); -} - -tensor::ExtractSliceOp MaskState::getExtractSlice( - Value source, const Location &loc, OpBuilder &builder, - SmallVector offsets, SmallVector dims) const { - auto sourceRType = cast(source.getType()); - SmallVector strides(getRank(), builder.getIndexAttr(1)); - - auto dstRType = tensor::ExtractSliceOp::inferResultType(sourceRType, offsets, - dims, strides); - return builder.create(loc, dstRType, source, offsets, - dims, strides); -} - -// insertSlice -tensor::InsertSliceOp MaskState::getInsertSlice(Value source, Value dest, - const Location &loc, - OpBuilder &builder) const { - SmallVector strides(getRank(), builder.getIndexAttr(1)); - return builder.create(loc, source, dest, offsets, dims, - strides); -} - -tensor::InsertSliceOp -MaskState::getInsertSlice(Value source, Value dest, const Location &loc, - OpBuilder &builder, SmallVector offsets, - SmallVector dims) const { - SmallVector strides(getRank(), builder.getIndexAttr(1)); - return builder.create(loc, source, dest, offsets, dims, - strides); -} - -memref::SubViewOp MaskState::getSubview(Value source, const Location &loc, - OpBuilder &builder) const { - auto sourceType = cast(source.getType()); - int64_t rank = sourceType.getRank(); - SmallVector strides(rank, builder.getIndexAttr(1)); - - SmallVector fixedOffsets(offsets.begin(), offsets.end()); - SmallVector fixedDims(dims.begin(), dims.end()); - fixedOffsets.resize(rank, builder.getIndexAttr(0)); - fixedDims.resize(rank, builder.getIndexAttr(1)); - - auto dstType = memref::SubViewOp::inferResultType(sourceType, fixedOffsets, - fixedDims, strides); - return builder.create( - loc, cast(dstType), source, fixedOffsets, fixedDims, strides); -} - -// In llvm later version, for each dimension, assert that: -// 0 <= offset < dim_size -// 0 <= offset + (size - 1) *stride < dim_size -// To adatpt to this change, add this verification to avoid llvm assert error -// And currently, in the function calling scenario, the stride coefficient is -// always 1 -bool MaskState::isMemrefSubviewValid(Value source, OpBuilder &builder) const { - auto sourceType = cast(source.getType()); - int64_t rank = sourceType.getRank(); - - SmallVector fixedOffsets(offsets.begin(), offsets.end()); - SmallVector fixedDims(dims.begin(), dims.end()); - fixedOffsets.resize(rank, builder.getIndexAttr(0)); - fixedDims.resize(rank, builder.getIndexAttr(1)); - - for (int64_t i = 0; i < rank; ++i) { - int64_t sourceSize = sourceType.getDimSize(i); - if (ShapedType::isDynamic(sourceSize)) - continue; - std::optional offsetVal = - mlir::getConstantIntValue(fixedOffsets[i]); - std::optional dimVal = mlir::getConstantIntValue(fixedDims[i]); - if (offsetVal.has_value() && dimVal.has_value()) { - if (offsetVal.value() >= sourceSize || offsetVal.value() < 0) { - LLVM_DEBUG({ - llvm::dbgs() << "MemrefSubview offset check faied at dim " << i - << "sourceSize :" << sourceSize - << "offsetVal :" << offsetVal << "\n"; - }); - return false; - } - - int64_t computedEnd = offsetVal.value() + dimVal.value(); - if (computedEnd > sourceSize) { - LLVM_DEBUG({ - llvm::dbgs() << "MemrefSubview offset end check faied at dim " << i - << "dimVal :" << dimVal << "offsetVal :" << offsetVal - << "\n"; - }); - return false; - } - } - } - return true; -} - -static memref::SubViewOp createSubview(Value src, const Location &loc, - OpBuilder &builder, - ArrayRef offsets, - ArrayRef sizes, - ArrayRef strides) { - auto srcType = cast(src.getType()); - auto dstType = - memref::SubViewOp::inferResultType(srcType, offsets, sizes, strides); - return builder.create(loc, cast(dstType), src, - offsets, sizes, strides); -} - -LogicalResult MaskState::addStateScalar(const MaskState &state, - const OpFoldResult scalar, - const Location &loc, - OpBuilder &builder) { - start = addOpFoldResult(state.start, scalar, loc, builder, - builder.getIndexType()); - end = - addOpFoldResult(state.end, scalar, loc, builder, builder.getIndexType()); - dims = state.dims; - offsets = state.offsets; - - bool allDimsOne = llvm::all_of(state.dims, [](OpFoldResult dim) { - return getConstantIntValue(dim).value() == std::optional(1); - }); - if (allDimsOne) { - this->scalar = this->start; - } - - return success(); -} - -LogicalResult MaskState::addStates(const MaskState &lhsState, - const MaskState &rhsState, - const Location &loc, OpBuilder &builder) { - if (lhsState.scalar && rhsState.scalar) { - InFlightDiagnostic diag = - emitWarning(loc) - << "Unexpected case where both lhs and rhs are scalars"; - return failure(); - } - if (!lhsState.scalar && !rhsState.scalar) { - InFlightDiagnostic diag = - emitWarning(loc) - << "Unsupported scenario where neither lhs nor rhs is a scalar"; - return failure(); - } - - if (lhsState.scalar) { - return addStateScalar(rhsState, lhsState.scalar, loc, builder); - } else { - return addStateScalar(lhsState, rhsState.scalar, loc, builder); - } -} - -LogicalResult MaskState::subStateScalar(const MaskState &state, - const OpFoldResult scalar, - const Location &loc, - OpBuilder &builder) { - start = subOpFoldResult(state.start, scalar, loc, builder); - end = subOpFoldResult(state.end, scalar, loc, builder); - if (!start || !end) - return failure(); - dims = state.dims; - offsets = state.offsets; - - bool allDimsOne = llvm::all_of(state.dims, [](OpFoldResult dim) { - return getConstantIntValue(dim).value() == std::optional(1); - }); - if (allDimsOne) { - this->scalar = this->start; - } - - return success(); -} - -LogicalResult MaskState::subStates(const MaskState &lhsState, - const MaskState &rhsState, - const Location &loc, OpBuilder &builder) { - // Contiguous mask indices only keep a monotonic range when subtracting a - // scalar from a range: (arange - C). scalar - range is rejected. - if (!lhsState.scalar && rhsState.scalar) - return subStateScalar(lhsState, rhsState.scalar, loc, builder); - - InFlightDiagnostic diag = - emitWarning(loc) - << "Unsupported subi for continuous mask: only (range - scalar) is " - "supported"; - return failure(); -} - -LogicalResult MaskState::divStateScalar(const MaskState &state, - const OpFoldResult scalar, - const Location &loc, - OpBuilder &builder) { - start = divOpFoldResult(state.start, scalar, loc, builder); - end = divOpFoldResult(state.end, scalar, loc, builder); - dims = state.dims; - offsets = state.offsets; - return success(); -} - -LogicalResult MaskState::divStates(const MaskState &lhsState, - const MaskState &rhsState, - const Location &loc, OpBuilder &builder) { - if (!lhsState.scalar && rhsState.scalar) { - if (isZeroMaskConstant(rhsState.scalar)) { - InFlightDiagnostic diag = - emitError(loc) - << "Unsupported scenario where rhs is zero constant in divide!"; - return failure(); - } - - return divStateScalar(lhsState, rhsState.scalar, loc, builder); - } - - InFlightDiagnostic diag = emitWarning(loc) - << "Supported scenario where only rhs is a scalar"; - return failure(); -} - -LogicalResult MaskState::minStates(const MaskState &lhsState, - const MaskState &rhsState, - const Location &loc, OpBuilder &builder) { - if (lhsState.getRank() != rhsState.getRank()) { - InFlightDiagnostic diag = - emitError(loc) - << "Unexpected case where lhs and rhs have different ranks"; - return failure(); - } - - for (uint32_t i = 0; i < lhsState.getRank(); i++) { - auto lhsOffset = lhsState.offsets[i]; - auto rhsOffset = rhsState.offsets[i]; - auto newOffset = maxOpFoldResult(lhsOffset, rhsOffset, loc, builder); - auto lhsDim = lhsState.dims[i]; - auto rhsDim = rhsState.dims[i]; - auto lhsEnd = addOpFoldResult(lhsOffset, lhsDim, loc, builder, - builder.getIndexType()); - auto rhsEnd = addOpFoldResult(rhsOffset, rhsDim, loc, builder, - builder.getIndexType()); - auto newEnd = minOpFoldResult(lhsEnd, rhsEnd, loc, builder); - auto newDim = subOpFoldResult(newEnd, newOffset, loc, builder); - auto clampedNewDim = - clampToNonNegativeIndex(newDim, loc, builder, /*clampDynamic=*/true); - - offsets.push_back(newOffset); - dims.push_back(clampedNewDim); - } - return success(); -} - -// Helper func for MaskState::parse() -LogicalResult MaskState::parseConstant(arith::ConstantOp constOp, - const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - - if (isa(constOp.getValue())) { - auto attr = cast(constOp.getValue()); - auto elementType = attr.getElementType(); - assert(attr.isSplat() && isa(elementType) && - "All elements must share a single integer constant value"); - - if (elementType.isInteger(1) && - isa(constOp.getValue().getType())) { - auto shapedType = cast(constOp.getValue().getType()); - auto shape = shapedType.getShape(); - for (size_t i = 0; i < shape.size(); i++) { - this->dims.push_back(builder.getIndexAttr(shape[i])); - this->offsets.push_back(builder.getIndexAttr(0)); - } - } else { - this->scalar = builder.getIndexAttr( - attr.getSplatValue().getValue().getSExtValue()); - } - } else { - auto value = cast(constOp.getValue()).getInt(); - this->scalar = builder.getIndexAttr(value); - } - return success(); -} - -// parseIntScalar -LogicalResult MaskState::parseIntScalar(Value scalar, const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - - this->scalar = getOpFoldResultOfLayoutInfo(scalar, builder); - return success(); -} - -LogicalResult MaskState::parseAdd(arith::AddIOp addOp, const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - MaskState lhsState; - if (failed(lhsState.parse(addOp.getLhs(), loc, builder))) { - return failure(); - } - - MaskState rhsState; - if (failed(rhsState.parse(addOp.getRhs(), loc, builder))) { - return failure(); - } - return this->addStates(lhsState, rhsState, loc, builder); -} - -LogicalResult MaskState::parseSub(arith::SubIOp subOp, const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - MaskState lhsState; - if (failed(lhsState.parse(subOp.getLhs(), loc, builder))) { - return failure(); - } - - MaskState rhsState; - if (failed(rhsState.parse(subOp.getRhs(), loc, builder))) { - return failure(); - } - return this->subStates(lhsState, rhsState, loc, builder); -} - -LogicalResult MaskState::parseDiv(arith::DivSIOp divOp, const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - return failure(); // temporarily disable parseDiv - MaskState lhsState; - if (failed(lhsState.parse(divOp.getLhs(), loc, builder))) { - return failure(); - } - - MaskState rhsState; - if (failed(rhsState.parse(divOp.getRhs(), loc, builder))) { - return failure(); - } - return this->divStates(lhsState, rhsState, loc, builder); -} - -LogicalResult MaskState::parseAnd(arith::AndIOp andOp, const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - MaskState lhsState; - if (failed(lhsState.parse(andOp.getLhs(), loc, builder)) || - !lhsState.isMask()) { - return failure(); - } - - MaskState rhsState; - if (failed(rhsState.parse(andOp.getRhs(), loc, builder)) || - !rhsState.isMask()) { - return failure(); - } - - if (!lhsState.isMask() && !rhsState.isMask()) { - return failure(); - } - - // Only support both lhs and rhs satisfy `isMask` condition - return this->minStates(lhsState, rhsState, loc, builder); -} - -LogicalResult MaskState::parseSel(arith::SelectOp selOp, const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - auto trueValue = selOp.getTrueValue(); - auto falseValue = selOp.getFalseValue(); - - MaskState condState; - auto condition = selOp.getCondition(); - auto cmpOp = condition.getDefiningOp(); - if (!cmpOp || failed(condState.parse(condition, loc, builder))) { - return failure(); - } - - MaskState trueState; - if (failed(trueState.parse(trueValue, loc, builder)) || !trueState.scalar) { - return failure(); - } - - MaskState falseState; - if (failed(falseState.parse(falseValue, loc, builder)) || - !falseState.scalar) { - return failure(); - } - - auto trueScalar = - dyn_cast_if_present(dyn_cast(trueState.scalar)); - auto falseScalar = - dyn_cast_if_present(dyn_cast(falseState.scalar)); - - if (trueScalar && falseScalar) { - if (trueScalar.getInt() == 1 && falseScalar.getInt() == 0) { - start = condState.start; - end = condState.end; - dims = condState.dims; - offsets = condState.offsets; - return success(); - } - } - - return failure(); -} - -LogicalResult MaskState::parseCmp(arith::CmpIOp cmpOp, const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - auto predicate = cmpOp.getPredicate(); - bool isProvenRuntimeExtentUnsignedMask = - cmpOp->hasAttr( - mlir::triton::memory_access::IATRuntimeExtentUnsignedMaskTAG) || - cmpOp->hasAttr( - mlir::triton::memory_access::PTSMRuntimeExtentUnsignedMaskTAG); - bool isTaggedUnsignedUpperBound = isProvenRuntimeExtentUnsignedMask && - (predicate == arith::CmpIPredicate::ult || - predicate == arith::CmpIPredicate::ule); - if (predicate != arith::CmpIPredicate::slt && - predicate != arith::CmpIPredicate::sle && - predicate != arith::CmpIPredicate::sge && - predicate != arith::CmpIPredicate::eq && - predicate != arith::CmpIPredicate::ne && !isTaggedUnsignedUpperBound) { - LLVM_DEBUG({ llvm::dbgs() << "Unsupported cmpi predicate\n"; }); - return failure(); - } - - MaskState lhsState; - MaskState rhsState; - auto lhs = cmpOp.getLhs(); - auto rhs = cmpOp.getRhs(); - - if (isTaggedUnsignedUpperBound) { - lhs = unwrapRuntimeExtentUnsignedOperand(lhs, loc, builder); - rhs = unwrapRuntimeExtentUnsignedOperand(rhs, loc, builder); - if (!lhs || !rhs) - return failure(); - predicate = predicate == arith::CmpIPredicate::ult - ? arith::CmpIPredicate::slt - : arith::CmpIPredicate::sle; - } - - if (predicate == arith::CmpIPredicate::ne) { - auto selOp = lhs.getDefiningOp(); - auto constantOp = rhs.getDefiningOp(); - if (!selOp || !constantOp) { - return failure(); - } - } - - if (failed(lhsState.parse(lhs, loc, builder))) { - return failure(); - } - - if (failed(rhsState.parse(rhs, loc, builder))) { - return failure(); - } - - if (!(!lhsState.scalar && rhsState.scalar)) { - InFlightDiagnostic diag = emitWarning(loc) - << "[MaskState] Unsupported cmpi scenario"; - return failure(); - } - - int32_t cmpDim = -1; - for (int32_t i = 0; i < lhsState.getRank(); i++) { - auto constDimLength = getConstantIntValue(lhsState.dims[i]); - if (!constDimLength || constDimLength.value() != 1) { - if (cmpDim != -1) { - InFlightDiagnostic diag = emitWarning(loc) - << "Unsupported cmpi with more than one " - "dimension with size larger than 1"; - return failure(); - } - cmpDim = i; - } - } - - assert(cmpDim != -1 && - "Unexpected case where no dimension has size larger than 1"); - - this->offsets = lhsState.offsets; - this->dims = lhsState.dims; - switch (predicate) { - case arith::CmpIPredicate::slt: { - auto realBound = - maxOpFoldResult(lhsState.start, rhsState.scalar, loc, builder); - auto newEnd = minOpFoldResult(lhsState.end, realBound, loc, builder); - auto newDim = subOpFoldResult(newEnd, lhsState.start, loc, builder); - auto clampedNewDim = clampToNonNegativeIndex(newDim, loc, builder); - - this->dims[cmpDim] = clampedNewDim; - break; - } - case arith::CmpIPredicate::sle: { - // lhs <= rhs <=> lhs < rhs + 1 - auto rhsPlusOne = addOpFoldResult(rhsState.scalar, builder.getIndexAttr(1), - loc, builder, builder.getIndexType()); - auto realBound = maxOpFoldResult(lhsState.start, rhsPlusOne, loc, builder); - auto newEnd = minOpFoldResult(lhsState.end, realBound, loc, builder); - auto newDim = subOpFoldResult(newEnd, lhsState.start, loc, builder); - auto clampedNewDim = clampToNonNegativeIndex(newDim, loc, builder); - - this->dims[cmpDim] = clampedNewDim; - break; - } - case arith::CmpIPredicate::sge: { - auto realBound = - maxOpFoldResult(lhsState.start, rhsState.scalar, loc, builder); - auto newStart = minOpFoldResult(lhsState.end, realBound, loc, builder); - auto newOffset = subOpFoldResult(newStart, lhsState.start, loc, builder); - auto newDim = subOpFoldResult(lhsState.end, newStart, loc, builder); - auto clampedNewDim = clampToNonNegativeIndex(newDim, loc, builder); - - this->offsets[cmpDim] = newOffset; - this->dims[cmpDim] = clampedNewDim; - break; - } - case arith::CmpIPredicate::eq: { - auto newOffset = - subOpFoldResult(rhsState.scalar, lhsState.start, loc, builder); - auto newDim = builder.getIndexAttr(1); - - this->offsets[cmpDim] = newOffset; - this->dims[cmpDim] = newDim; - break; - } - case arith::CmpIPredicate::ne: { - // only support lhs != 0 - auto rhsScalar = dyn_cast(cast(rhsState.scalar)); - if (!rhsScalar || rhsScalar.getInt() != 0) { - return failure(); - } - - start = lhsState.start; - end = lhsState.end; - break; - } - default: - return failure(); - } - return success(); -} - -LogicalResult MaskState::parseMakeRange(triton::MakeRangeOp rangeOp, - const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - auto shape = cast(rangeOp.getType()).getShape(); - auto start = rangeOp.getStart(); - auto end = rangeOp.getEnd(); - auto stride = (end - start + shape[0] - 1) / shape[0]; - - if (stride != 1) { - InFlightDiagnostic diag = - emitWarning(loc) - << "stride must be 1 for make_range whose result is used " - "as load or store masks"; - return failure(); - } - - this->start = builder.getIndexAttr(start); - this->end = builder.getIndexAttr(end); - this->dims.push_back(builder.getIndexAttr(shape[0])); - this->offsets.push_back(builder.getIndexAttr(0)); - return success(); -} - -LogicalResult MaskState::parseBroadcast(triton::BroadcastOp broadcastOp, - const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - auto src = broadcastOp.getSrc(); - auto dst = broadcastOp.getResult(); - assert(isa(src.getType()) && - "input to tt.broadcast should be a tensor"); - - auto srcShape = cast(src.getType()).getShape(); - auto dstShape = cast(dst.getType()).getShape(); - assert(srcShape.size() == dstShape.size() && - "rank of source and destination should match"); - - if (failed(parse(src, loc, builder))) { - return failure(); - } - for (size_t i = 0; i < srcShape.size(); i++) { - if (srcShape[i] == dstShape[i]) - continue; - else if (srcShape[i] < dstShape[i]) - this->dims[i] = builder.getIndexAttr(dstShape[i]); - else - llvm_unreachable("unexpected dimensions used in broadcast"); - } - return success(); -} - -LogicalResult MaskState::parseSplat(triton::SplatOp splatOp, - const Location &loc, OpBuilder &builder) { - assert(this->isEmpty()); - - auto src = splatOp.getSrc(); - auto dst = splatOp.getResult(); - auto dstShape = cast(dst.getType()).getShape(); - - if (!isa(src.getType())) { - InFlightDiagnostic diag = - emitWarning(loc) - << "splat source must be an integer scalar for load/store masks"; - return failure(); - } - - if (failed(this->parse(src, loc, builder))) - return failure(); - - auto splatAsMask = [&](Operation *userOp) -> bool { - return TypeSwitch(userOp) - .Case([&](arith::AndIOp andOp) { return true; }) - .Case([&](arith::SelectOp selectOp) { - return selectOp.getCondition() == dst; - }) - .Case( - [&](triton::LoadOp loadOp) { return loadOp.getMask() == dst; }) - .Case( - [&](triton::StoreOp storeOp) { return storeOp.getMask() == dst; }) - .Case([&](triton::AtomicRMWOp atomicOp) { - return atomicOp.getMask() == dst; - }) - .Default([&](Operation *op) { return false; }); - }; - - if (src.getType().isInteger(1) && !splatOp->use_empty() && - llvm::all_of(splatOp->getUsers(), splatAsMask)) { - for (auto s : dstShape) { - auto currentDim = mulOpFoldResult(builder.getIndexAttr(s), this->scalar, - loc, builder, builder.getIndexType()); - this->dims.push_back(currentDim); - this->offsets.push_back(builder.getIndexAttr(0)); - } - - this->scalar = nullptr; - return success(); - } - - for (auto s : dstShape) { - this->dims.push_back(builder.getIndexAttr(s)); - this->offsets.push_back(builder.getIndexAttr(0)); - } - return success(); -} - -LogicalResult MaskState::parseExpandDims(triton::ExpandDimsOp expandDimsOp, - const Location &loc, - OpBuilder &builder) { - assert(this->isEmpty()); - - if (failed(this->parse(expandDimsOp.getSrc(), loc, builder))) { - return failure(); - } - - auto dstShape = - cast(expandDimsOp.getResult().getType()).getShape(); - auto axis = expandDimsOp.getAxis(); - assert(dstShape[axis] == 1 && - "Expect changed dimention to be 1 in expand_dims"); - this->dims.insert(this->dims.begin() + axis, builder.getIndexAttr(1)); - this->offsets.insert(this->offsets.begin() + axis, builder.getIndexAttr(0)); - - return success(); -} - -LogicalResult MaskState::parseInsert(tensor::InsertOp insertOp, - const Location &loc, OpBuilder &builder) { - if (auto tensorType = dyn_cast(insertOp.getType())) { - if (llvm::all_of(tensorType.getShape(), - [](int64_t dim) { return dim == 1; })) { - SmallVector indices( - tensorType.getRank(), builder.create(loc, 0)); - auto scalarlizeTensor = - builder.create(loc, insertOp, indices).getResult(); - this->offsets = SmallVector(tensorType.getRank(), - builder.getIndexAttr(0)); - this->dims = SmallVector(tensorType.getRank(), - builder.getIndexAttr(1)); - this->scalar = getOpFoldResultOfLayoutInfo(scalarlizeTensor, builder); - return success(); - } - } - return failure(); -} - -void MaskState::eraseInsertedOps(Operation *rawOp, PatternRewriter &rewriter) { - auto moduleOp = rawOp->getParentOfType(); - SmallVector worklist; - moduleOp->walk([&](Operation *op) { - if (isOpTriviallyDead(op) && op->use_empty()) { - worklist.push_back(op); - } - }); - while (!worklist.empty()) { - Operation *op = worklist.pop_back_val(); - if (!isOpTriviallyDead(op) || !op->use_empty()) { - continue; - } - if (!op->getBlock()) { - continue; - } - SmallVector operandDefOps; - for (Value value : op->getOperands()) { - if (auto defOp = value.getDefiningOp()) { - if (defOp->getBlock()) { - operandDefOps.push_back(defOp); - } - } - } - LLVM_DEBUG({ - llvm::dbgs() << "[MaskState]==> inserted op: \n" - << *op << "\n[MaskState]<== is removed\n"; - }); - - rewriter.eraseOp(op); - for (auto defOp : operandDefOps) { - worklist.push_back(defOp); - } - } -} - -std::optional runMaskAnalysis(Operation *op, OpBuilder &builder) { - if (auto loadOp = dyn_cast(op)) { - return runMaskAnalysisImpl(loadOp, builder); - } - if (auto storeOp = dyn_cast(op)) { - return runMaskAnalysisImpl(storeOp, builder); - } - if (auto atomicRMWOp = dyn_cast(op)) { - return runMaskAnalysisImpl(atomicRMWOp, builder); - } - return std::nullopt; -} - -} // namespace triton - -} // namespace mlir diff --git a/third_party/ascend/lib/TritonMemoryAccess/OpFoldResultUtils.cpp b/third_party/ascend/lib/TritonMemoryAccess/OpFoldResultUtils.cpp deleted file mode 100644 index 3c9a1e9d9c..0000000000 --- a/third_party/ascend/lib/TritonMemoryAccess/OpFoldResultUtils.cpp +++ /dev/null @@ -1,463 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "TritonMemoryAccess/OpFoldResultUtils.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinTypeInterfaces.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Diagnostics.h" -#include "mlir/IR/Value.h" - -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/Support/ErrorHandling.h" - -#include -#include -#include - -namespace mlir { -namespace { - -Value createConstIndexValueOp(const Location &loc, OpBuilder &builder, - int64_t value) { - return builder.create(loc, builder.getIndexAttr(value)) - .getResult(); -} - -FailureOr materializeIndexOperand(const OpFoldResult &operand, - std::optional constant, - Location loc, OpBuilder &builder) { - if (constant) - return createConstIndexValueOp(loc, builder, *constant); - auto value = dyn_cast(operand); - if (!value) - return failure(); - return castIntegerLike(builder, loc, value, builder.getIndexType()); -} - -Type getOpFoldResultType(const OpFoldResult &operand) { - if (auto value = dyn_cast(operand)) - return value.getType(); - if (auto attribute = dyn_cast(operand)) { - if (auto typedAttribute = dyn_cast(attribute)) - return typedAttribute.getType(); - } - return Type(); -} - -bool isSupportedScalarIntegerType(Type type) { - return type && (type.isIndex() || type.isSignlessInteger()); -} - -FailureOr resolveIntegerResultType(const OpFoldResult &lhs, - const OpFoldResult &rhs, - Type requestedType) { - Type lhsType = getOpFoldResultType(lhs); - Type rhsType = getOpFoldResultType(rhs); - if (!isSupportedScalarIntegerType(lhsType) || - !isSupportedScalarIntegerType(rhsType) || - (requestedType && !isSupportedScalarIntegerType(requestedType))) - return failure(); - - if (requestedType) - return requestedType; - if (lhsType == rhsType) - return lhsType; - - // Index has no fixed IR-level width, so it does not participate in default - // widening against ordinary integers. - auto lhsInteger = dyn_cast(lhsType); - auto rhsInteger = dyn_cast(rhsType); - if (!lhsInteger || !rhsInteger || - lhsInteger.getWidth() == rhsInteger.getWidth()) - return failure(); - return lhsInteger.getWidth() > rhsInteger.getWidth() ? lhsType : rhsType; -} - -FailureOr castIntegerFoldResult(const OpFoldResult &operand, - Type targetType, Location loc, - OpBuilder &builder) { - if (std::optional constant = getConstantOfAttr(operand)) - return OpFoldResult(builder.getIntegerAttr(targetType, *constant)); - - auto value = dyn_cast(operand); - if (!value) - return failure(); - FailureOr converted = castIntegerLike(builder, loc, value, targetType); - if (failed(converted)) - return failure(); - return OpFoldResult(*converted); -} - -FailureOr materializeIntegerOperand(const OpFoldResult &operand, - std::optional constant, - Type targetType, Location loc, - OpBuilder &builder) { - if (constant) - return builder - .create( - loc, builder.getIntegerAttr(targetType, *constant)) - .getResult(); - auto value = dyn_cast(operand); - if (!value) - return failure(); - return castIntegerLike(builder, loc, value, targetType); -} - -} // namespace - -std::optional getConstantOfAttr(const OpFoldResult &arg) { - if (isa(arg)) - return getConstantIntValue(arg); - return std::nullopt; -} - -FailureOr castIntegerLike(OpBuilder &builder, Location loc, Value value, - Type targetType, - IntegerExtensionKind extension) { - if (!value || !targetType) - return failure(); - - Type sourceType = value.getType(); - if (sourceType == targetType) - return value; - - if ((sourceType.isIndex() && isa(targetType)) || - (isa(sourceType) && targetType.isIndex())) { - if (extension == IntegerExtensionKind::Unsigned) - return builder.create(loc, targetType, value) - .getResult(); - return builder.create(loc, targetType, value) - .getResult(); - } - - auto extendOrTruncate = [&](unsigned sourceWidth, - unsigned targetWidth) -> FailureOr { - if (sourceWidth < targetWidth) { - if (extension == IntegerExtensionKind::Unsigned) - return builder.create(loc, targetType, value) - .getResult(); - return builder.create(loc, targetType, value).getResult(); - } - if (sourceWidth > targetWidth) - return builder.create(loc, targetType, value) - .getResult(); - return failure(); - }; - - auto sourceInteger = dyn_cast(sourceType); - auto targetInteger = dyn_cast(targetType); - if (sourceInteger && targetInteger) - return extendOrTruncate(sourceInteger.getWidth(), targetInteger.getWidth()); - - auto sourceTensor = dyn_cast(sourceType); - auto targetTensor = dyn_cast(targetType); - if (!sourceTensor || !targetTensor || - sourceTensor.getShape() != targetTensor.getShape() || - sourceTensor.getEncoding() != targetTensor.getEncoding()) - return failure(); - - auto sourceElement = dyn_cast(sourceTensor.getElementType()); - auto targetElement = dyn_cast(targetTensor.getElementType()); - if (!sourceElement || !targetElement) - return failure(); - - return extendOrTruncate(sourceElement.getWidth(), targetElement.getWidth()); -} - -OpFoldResult addOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &b, - Type resultType) { - FailureOr resolvedType = resolveIntegerResultType(lhs, rhs, resultType); - if (failed(resolvedType)) - return OpFoldResult(); - - auto lhsInt = getConstantOfAttr(lhs); - auto rhsInt = getConstantOfAttr(rhs); - - if (lhsInt && rhsInt) - return b.getIntegerAttr(*resolvedType, *lhsInt + *rhsInt); - - if (!lhsInt && rhsInt && *rhsInt == 0) { - FailureOr result = - castIntegerFoldResult(lhs, *resolvedType, loc, b); - return succeeded(result) ? *result : OpFoldResult(); - } - if (!rhsInt && lhsInt && *lhsInt == 0) { - FailureOr result = - castIntegerFoldResult(rhs, *resolvedType, loc, b); - return succeeded(result) ? *result : OpFoldResult(); - } - - FailureOr lhsValue = - materializeIntegerOperand(lhs, lhsInt, *resolvedType, loc, b); - FailureOr rhsValue = - materializeIntegerOperand(rhs, rhsInt, *resolvedType, loc, b); - if (failed(lhsValue) || failed(rhsValue)) - return OpFoldResult(); - - return b.create(loc, *lhsValue, *rhsValue).getResult(); -} - -OpFoldResult subOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &b) { - auto lhsInt = getConstantOfAttr(lhs); - auto rhsInt = getConstantOfAttr(rhs); - - if (lhsInt && rhsInt) - return b.getIndexAttr(lhsInt.value() - rhsInt.value()); - - if (!lhsInt && rhsInt && rhsInt.value() == 0) - return lhs; - - FailureOr lhsValue = materializeIndexOperand(lhs, lhsInt, loc, b); - FailureOr rhsValue = materializeIndexOperand(rhs, rhsInt, loc, b); - if (failed(lhsValue) || failed(rhsValue)) - return OpFoldResult(); - - return b.create(loc, *lhsValue, *rhsValue).getResult(); -} - -OpFoldResult mulOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &b, - Type resultType) { - FailureOr resolvedType = resolveIntegerResultType(lhs, rhs, resultType); - if (failed(resolvedType)) - return OpFoldResult(); - - auto lhsInt = getConstantOfAttr(lhs); - auto rhsInt = getConstantOfAttr(rhs); - - if (lhsInt && rhsInt) - return b.getIntegerAttr(*resolvedType, *lhsInt * *rhsInt); - - if (lhsInt) { - if (*lhsInt == 0) - return b.getIntegerAttr(*resolvedType, 0); - if (*lhsInt == 1) { - FailureOr result = - castIntegerFoldResult(rhs, *resolvedType, loc, b); - return succeeded(result) ? *result : OpFoldResult(); - } - } - if (rhsInt) { - if (*rhsInt == 0) - return b.getIntegerAttr(*resolvedType, 0); - if (*rhsInt == 1) { - FailureOr result = - castIntegerFoldResult(lhs, *resolvedType, loc, b); - return succeeded(result) ? *result : OpFoldResult(); - } - } - - FailureOr lhsValue = - materializeIntegerOperand(lhs, lhsInt, *resolvedType, loc, b); - FailureOr rhsValue = - materializeIntegerOperand(rhs, rhsInt, *resolvedType, loc, b); - if (failed(lhsValue) || failed(rhsValue)) - return OpFoldResult(); - - return b.create(loc, *lhsValue, *rhsValue).getResult(); -} - -OpFoldResult divOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &b) { - auto lhsInt = getConstantOfAttr(lhs); - auto rhsInt = getConstantOfAttr(rhs); - - if (rhsInt && rhsInt.value() == 0) { - emitError(loc) << "cannot div 0!"; - return OpFoldResult(); - } - - if (lhsInt && rhsInt) - return b.getIndexAttr(lhsInt.value() / rhsInt.value()); - - if (lhsInt) { - if (lhsInt.value() == 0) - return lhs; - } - - if (rhsInt) { - if (rhsInt.value() == 1) - return lhs; - } - - FailureOr lhsValue = materializeIndexOperand(lhs, lhsInt, loc, b); - FailureOr rhsValue = materializeIndexOperand(rhs, rhsInt, loc, b); - if (failed(lhsValue) || failed(rhsValue)) - return OpFoldResult(); - - return b.create(loc, *lhsValue, *rhsValue).getResult(); -} - -OpFoldResult remOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &b) { - auto lhsInt = getConstantOfAttr(lhs); - auto rhsInt = getConstantOfAttr(rhs); - - if (rhsInt && rhsInt.value() == 0) { - emitError(loc) << "cannot remainder by 0!"; - return OpFoldResult(); - } - - if (lhsInt && rhsInt) - return b.getIndexAttr(lhsInt.value() % rhsInt.value()); - - if (lhsInt) { - if (lhsInt.value() == 0) - return lhs; - } - - FailureOr lhsValue = materializeIndexOperand(lhs, lhsInt, loc, b); - FailureOr rhsValue = materializeIndexOperand(rhs, rhsInt, loc, b); - if (failed(lhsValue) || failed(rhsValue)) - return OpFoldResult(); - - return b.create(loc, *lhsValue, *rhsValue).getResult(); -} - -OpFoldResult minOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &b) { - auto lhsInt = getConstantOfAttr(lhs); - auto rhsInt = getConstantOfAttr(rhs); - if (lhsInt && rhsInt) - return b.getIndexAttr(std::min(lhsInt.value(), rhsInt.value())); - - FailureOr lhsValue = materializeIndexOperand(lhs, lhsInt, loc, b); - FailureOr rhsValue = materializeIndexOperand(rhs, rhsInt, loc, b); - if (failed(lhsValue) || failed(rhsValue)) - return OpFoldResult(); - - return b.create(loc, *lhsValue, *rhsValue).getResult(); -} - -OpFoldResult maxOpFoldResult(const OpFoldResult &lhs, const OpFoldResult &rhs, - const Location &loc, OpBuilder &b) { - auto lhsInt = getConstantOfAttr(lhs); - auto rhsInt = getConstantOfAttr(rhs); - if (lhsInt && rhsInt) - return b.getIndexAttr(std::max(lhsInt.value(), rhsInt.value())); - - FailureOr lhsValue = materializeIndexOperand(lhs, lhsInt, loc, b); - FailureOr rhsValue = materializeIndexOperand(rhs, rhsInt, loc, b); - if (failed(lhsValue) || failed(rhsValue)) - return OpFoldResult(); - - return b.create(loc, *lhsValue, *rhsValue).getResult(); -} -std::optional getIntAttr(const OpFoldResult ofr) { - Attribute attr; - if (auto val = dyn_cast(ofr)) { - if (!val.getDefiningOp()) - return std::nullopt; - attr = cast(val.getDefiningOp().getValue()); - } else { - attr = dyn_cast(ofr); - } - if (attr && isa(attr)) - return dyn_cast(attr).getInt(); - return std::nullopt; -} - -Value materializeValue(OpBuilder &builder, Location loc, OpFoldResult ofr) { - if (auto val = ofr.dyn_cast()) { - return val; - } - - auto intVal = getIntAttr(ofr); - if (intVal.has_value()) { - return builder.create( - loc, builder.getI32IntegerAttr(intVal.value())); - } - assert(intVal.has_value()); - return Value(); - - // return builder.create( - // loc, dyn_cast(attr).getInt()); -} - -bool isZero(const OpFoldResult ofr) { - auto staticOfr = getIntAttr(ofr); - return staticOfr.has_value() && staticOfr.value() == 0; -} - -bool isOne(const OpFoldResult ofr) { - auto staticOfr = getIntAttr(ofr); - return staticOfr.has_value() && staticOfr.value() == 1; -} - -Value convertToIndexIfNeeded(Value input, const Location &loc, OpBuilder &b) { - auto inputType = input.getType(); - if (auto intType = dyn_cast(inputType)) { - if (intType.isInteger(32) || intType.isInteger(64)) { - return b.create(loc, b.getIndexType(), input); - } - } - return input; -} - -RankedTensorType getExtractSlicedType(ArrayRef shape, - const llvm::SmallBitVector &droppedDims, - Type elemType) { - SmallVector targetShape; - for (auto [idx, dimOfr] : llvm::enumerate(shape)) { - if (!droppedDims[idx]) { - if (auto dim = getConstantIntValue(dimOfr)) { - targetShape.push_back(dim.value()); - } else { - targetShape.push_back(ShapedType::kDynamic); - } - } - } - return RankedTensorType::get(targetShape, elemType); -} - -// Fold layout constant info to attr, otherwise convert to index type value. -OpFoldResult getOpFoldResultOfLayoutInfo(Value value, OpBuilder &builder) { - OpFoldResult constantFold = getAsOpFoldResult(value); - if (llvm::isa(constantFold)) { - assert(isa(cast(constantFold))); - return constantFold; - } - - Type sourceType = value.getType(); - if (!sourceType.isIndex() && !isa(sourceType)) - llvm_unreachable("Illegal data type when parse block data layout info"); - - IntegerExtensionKind extension = sourceType.isInteger(/*width=*/1) - ? IntegerExtensionKind::Unsigned - : IntegerExtensionKind::Signed; - FailureOr converted = castIntegerLike( - builder, value.getLoc(), value, builder.getIndexType(), extension); - if (failed(converted)) - llvm_unreachable("Failed to convert block data layout info to index"); - return *converted; -} - -} // namespace mlir diff --git a/third_party/ascend/lib/TritonToGraph/CMakeLists.txt b/third_party/ascend/lib/TritonToGraph/CMakeLists.txt index d2af77355d..1b1866ad22 100644 --- a/third_party/ascend/lib/TritonToGraph/CMakeLists.txt +++ b/third_party/ascend/lib/TritonToGraph/CMakeLists.txt @@ -59,7 +59,7 @@ add_triton_library(TritonToGraph MLIRTritonNPUUtils TritonAscendIR TritonIR - TritonMemoryAccessAnalysis + FlirTritonMemoryAccessAnalysis TritonToStructured TritonTransforms ) diff --git a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp index ba735b3668..fc7495fefb 100644 --- a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp +++ b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp @@ -21,8 +21,8 @@ */ #include "TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h" -#include "TritonMemoryAccess/LoadStoreMaskAnalysis.h" -#include "TritonMemoryAccess/MemoryAccessTags.h" +#include "incubated/TritonMemoryAccess/LoadStoreMaskAnalysis.h" +#include "incubated/TritonMemoryAccess/MemoryAccessTags.h" #include "TritonToStructured/PtrAnalysis.h" #include "Utils/Utils.h" diff --git a/third_party/ascend/lib/TritonToGraph/Rules/IndependentAxisTensorizeRule.cpp b/third_party/ascend/lib/TritonToGraph/Rules/IndependentAxisTensorizeRule.cpp index 56462c4322..fc2f4742d6 100644 --- a/third_party/ascend/lib/TritonToGraph/Rules/IndependentAxisTensorizeRule.cpp +++ b/third_party/ascend/lib/TritonToGraph/Rules/IndependentAxisTensorizeRule.cpp @@ -20,7 +20,7 @@ * THE SOFTWARE. */ -#include "TritonMemoryAccess/MemoryAccessTags.h" +#include "incubated/TritonMemoryAccess/MemoryAccessTags.h" #include "TritonToGraph/EntryArgPointerAliasAnalysis.h" #include "TritonToGraph/GraphOptimizationRule.h" #include "TritonToGraph/IndependentRowReductionAnalysis.h" diff --git a/third_party/ascend/lib/TritonToGraph/Rules/PersistentTaskStripMiningRule.cpp b/third_party/ascend/lib/TritonToGraph/Rules/PersistentTaskStripMiningRule.cpp index 7ec81a7ab8..c7ac02b0c5 100644 --- a/third_party/ascend/lib/TritonToGraph/Rules/PersistentTaskStripMiningRule.cpp +++ b/third_party/ascend/lib/TritonToGraph/Rules/PersistentTaskStripMiningRule.cpp @@ -20,7 +20,7 @@ * THE SOFTWARE. */ -#include "TritonMemoryAccess/MemoryAccessTags.h" +#include "incubated/TritonMemoryAccess/MemoryAccessTags.h" #include "TritonToGraph/EntryArgPointerAliasAnalysis.h" #include "TritonToGraph/GraphOptimizationRule.h" #include "TritonToGraph/IndependentRowReductionAnalysis.h" diff --git a/third_party/ascend/lib/TritonToUnstructure/CMakeLists.txt b/third_party/ascend/lib/TritonToUnstructure/CMakeLists.txt index e167c44619..ba5fb621c7 100644 --- a/third_party/ascend/lib/TritonToUnstructure/CMakeLists.txt +++ b/third_party/ascend/lib/TritonToUnstructure/CMakeLists.txt @@ -18,6 +18,6 @@ add_triton_library(TritonToUnstructure MLIRSupport TritonIR TritonAnalysis - TritonMemoryAccessAnalysis + FlirTritonMemoryAccessAnalysis MLIRSCFTransforms ) diff --git a/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp b/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp index 84a6bdf70b..fda2a91498 100644 --- a/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp +++ b/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp @@ -21,7 +21,7 @@ */ #include "TritonToUnstructure/UnstructureConversionPass.h" -#include "TritonMemoryAccess/LoadStoreMaskAnalysis.h" +#include "incubated/TritonMemoryAccess/LoadStoreMaskAnalysis.h" #include "TritonToStructured/CannonicalizerConverter.h" #include "TritonToUnstructure/IndirectAtomicUtils.h" #include "Utils/Utils.h" diff --git a/third_party/ascend/lib/Utils/CMakeLists.txt b/third_party/ascend/lib/Utils/CMakeLists.txt index 2421454029..db3f60034b 100644 --- a/third_party/ascend/lib/Utils/CMakeLists.txt +++ b/third_party/ascend/lib/Utils/CMakeLists.txt @@ -7,6 +7,6 @@ add_triton_library(MLIRTritonNPUUtils MLIRIR MLIRLLVMDialect TritonIR - TritonMemoryAccessAnalysis - TritonMemoryAccessCore + FlirTritonMemoryAccessAnalysis + FlirTritonMemoryAccessCore ) From 61ec4963e410516f6472a45461a75d8b11458d86 Mon Sep 17 00:00:00 2001 From: zhongsanming Date: Wed, 18 Nov 2026 11:52:14 +0800 Subject: [PATCH 4/7] [Ascend] Consume FLIR TritonToGraph legacy helpers Remove the duplicated layout-memory/coalescing implementation and GraphOptimization metadata from Ascend, and link the FLIR incubated TritonToGraph target from TritonToGraph and TritonToLinalgIncubated. --- third_party/ascend/CMakeLists.txt | 2 +- .../include/TritonToGraph/GraphOptimization.h | 335 ------------------ .../TritonToGraph/GraphOptimizationRule.h | 2 +- .../TritonToGraph/LayoutMemoryOptimization.h | 52 --- .../ascend/include/TritonToGraph/Passes.h | 2 +- third_party/ascend/lib/CMakeLists.txt | 3 + .../ascend/lib/TritonToGraph/CMakeLists.txt | 6 +- .../LayoutMemoryCompatibilityPass.cpp | 144 -------- .../LegacyMemoryAccess/ChunkCoalescing.cpp | 4 +- .../LegacyMemoryAccess/RowCoalescing.cpp | 4 +- .../StridedAxisCoalescing.cpp | 4 +- .../StridedLoadStoreRewrite.cpp | 4 +- .../TritonToGraph/Rules/RowCoalescingRule.cpp | 2 +- third_party/ascend/triton_ascend.cc | 4 +- 14 files changed, 18 insertions(+), 550 deletions(-) delete mode 100644 third_party/ascend/include/TritonToGraph/GraphOptimization.h delete mode 100644 third_party/ascend/include/TritonToGraph/LayoutMemoryOptimization.h delete mode 100644 third_party/ascend/lib/TritonToGraph/LayoutMemoryCompatibilityPass.cpp diff --git a/third_party/ascend/CMakeLists.txt b/third_party/ascend/CMakeLists.txt index 3c81c9f628..6e97006527 100644 --- a/third_party/ascend/CMakeLists.txt +++ b/third_party/ascend/CMakeLists.txt @@ -50,7 +50,7 @@ if(FLAGTREE_BACKEND STREQUAL "ascend") TritonStructuredIR TritonToStructured TritonToUnstructure - TritonToGraph + FlirTritonToGraphLegacy TritonControlFlowOpt MLIRTritonNPUUtils ) diff --git a/third_party/ascend/include/TritonToGraph/GraphOptimization.h b/third_party/ascend/include/TritonToGraph/GraphOptimization.h deleted file mode 100644 index 1748bc1642..0000000000 --- a/third_party/ascend/include/TritonToGraph/GraphOptimization.h +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_TO_GRAPH_GRAPH_OPTIMIZATION_H -#define TRITON_TO_GRAPH_GRAPH_OPTIMIZATION_H - -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" - -#include -#include -#include -#include -#include - -namespace mlir { -namespace triton { -namespace cfg { - -// The identity of every rule is one bit of the pass rule mask, so the mask -using GraphOptimizationRuleMask = uint32_t; - -enum class GraphOptimizationRuleId : GraphOptimizationRuleMask { - LoadStoreTranspose = 1, - TransposePointwiseReorder = 2, - StoreCoalescing = 4, - // RowCoalescing includes a shared-rhs dot pattern and a pure-SIMT-only - // elementwise pattern. It is intentionally - // scheduled once after the normal per-function phases because its launch - // contract must not be suppressed by their rewrite budget. - RowCoalescing = 8, - // DiagonalMaskRemoval rewrites compute logic rather than memory access, so - // it is architecture and template independent. - DiagonalMaskRemoval = 128, - // ConvertModuloToMask linearizes wrapped tile addresses. It runs before the - // memory-access rules so they see the linear form. - ConvertModuloToMask = 256, - // The following identities are owned by the layout/memory compatibility - // passes. They deliberately are not GraphOptimizationRule candidates and - // are not added to GraphOptimizePass's per-function phase loop. - StridedAxisCoalescing = 16, - ChunkCoalescing = 32, - StridedLoadStoreRewrite = 64, - IndependentAxisTensorize = 512, - PersistentTaskStripMining = 2048, - ResidentLoadForwarding = 4096, - IntermediatePrecisionBoundaryElision = 8192, - StoreCoveragePlanning = 16384, - ContiguousBlockAccessFormation = 32768, -}; - -enum class GraphOptimizationRulePhase : uint8_t { - DiagonalMaskRemoval, - ConvertModuloToMask, - ProgramMapping, - PersistentTaskMapping, - LoadStoreTranspose, - TransposePointwiseReorder, - ResidentLoadForwarding, - IntermediatePrecisionBoundaryElision, - StoreCoveragePlanning, - StoreCoalescing, - ContiguousBlockAccessFormation, - RowCoalescing, - Compatibility, -}; - -constexpr const char * -getGraphOptimizationRuleName(GraphOptimizationRuleId rule) { - switch (rule) { - case GraphOptimizationRuleId::LoadStoreTranspose: - return "LoadStoreTranspose"; - case GraphOptimizationRuleId::TransposePointwiseReorder: - return "TransposePointwiseReorder"; - case GraphOptimizationRuleId::StoreCoalescing: - return "StoreCoalescing"; - case GraphOptimizationRuleId::RowCoalescing: - return "RowCoalescing"; - case GraphOptimizationRuleId::DiagonalMaskRemoval: - return "DiagonalMaskRemoval"; - case GraphOptimizationRuleId::ConvertModuloToMask: - return "ConvertModuloToMask"; - case GraphOptimizationRuleId::StridedAxisCoalescing: - return "StridedAxisCoalescing"; - case GraphOptimizationRuleId::ChunkCoalescing: - return "ChunkCoalescing"; - case GraphOptimizationRuleId::StridedLoadStoreRewrite: - return "StridedLoadStoreRewrite"; - case GraphOptimizationRuleId::IndependentAxisTensorize: - return "IndependentAxisTensorizeRule"; - case GraphOptimizationRuleId::PersistentTaskStripMining: - return "PersistentTaskStripMiningRule"; - case GraphOptimizationRuleId::ResidentLoadForwarding: - return "ResidentLoadForwardingRule"; - case GraphOptimizationRuleId::IntermediatePrecisionBoundaryElision: - return "IntermediatePrecisionBoundaryElisionRule"; - case GraphOptimizationRuleId::StoreCoveragePlanning: - return "StoreCoveragePlanningRule"; - case GraphOptimizationRuleId::ContiguousBlockAccessFormation: - return "ContiguousBlockAccessFormationRule"; - } - return "Unknown"; -} - -constexpr GraphOptimizationRuleMask -getGraphOptimizationRuleMask(GraphOptimizationRuleId rule) { - return static_cast(rule); -} - -constexpr GraphOptimizationRulePhase -getGraphOptimizationRulePhase(GraphOptimizationRuleId rule) { - switch (rule) { - case GraphOptimizationRuleId::DiagonalMaskRemoval: - return GraphOptimizationRulePhase::DiagonalMaskRemoval; - case GraphOptimizationRuleId::ConvertModuloToMask: - return GraphOptimizationRulePhase::ConvertModuloToMask; - case GraphOptimizationRuleId::IndependentAxisTensorize: - return GraphOptimizationRulePhase::ProgramMapping; - case GraphOptimizationRuleId::PersistentTaskStripMining: - return GraphOptimizationRulePhase::PersistentTaskMapping; - case GraphOptimizationRuleId::LoadStoreTranspose: - return GraphOptimizationRulePhase::LoadStoreTranspose; - case GraphOptimizationRuleId::TransposePointwiseReorder: - return GraphOptimizationRulePhase::TransposePointwiseReorder; - case GraphOptimizationRuleId::ResidentLoadForwarding: - return GraphOptimizationRulePhase::ResidentLoadForwarding; - case GraphOptimizationRuleId::IntermediatePrecisionBoundaryElision: - return GraphOptimizationRulePhase::IntermediatePrecisionBoundaryElision; - case GraphOptimizationRuleId::StoreCoveragePlanning: - return GraphOptimizationRulePhase::StoreCoveragePlanning; - case GraphOptimizationRuleId::StoreCoalescing: - return GraphOptimizationRulePhase::StoreCoalescing; - case GraphOptimizationRuleId::ContiguousBlockAccessFormation: - return GraphOptimizationRulePhase::ContiguousBlockAccessFormation; - case GraphOptimizationRuleId::RowCoalescing: - return GraphOptimizationRulePhase::RowCoalescing; - case GraphOptimizationRuleId::StridedAxisCoalescing: - case GraphOptimizationRuleId::ChunkCoalescing: - case GraphOptimizationRuleId::StridedLoadStoreRewrite: - return GraphOptimizationRulePhase::Compatibility; - } - return GraphOptimizationRulePhase::Compatibility; -} - -constexpr std::array - kGraphOptimizationRuleRegistry = { - GraphOptimizationRuleId::LoadStoreTranspose, - GraphOptimizationRuleId::TransposePointwiseReorder, - GraphOptimizationRuleId::StoreCoalescing, - GraphOptimizationRuleId::RowCoalescing, - GraphOptimizationRuleId::DiagonalMaskRemoval, - GraphOptimizationRuleId::ConvertModuloToMask, - GraphOptimizationRuleId::StridedAxisCoalescing, - GraphOptimizationRuleId::ChunkCoalescing, - GraphOptimizationRuleId::StridedLoadStoreRewrite, - GraphOptimizationRuleId::IndependentAxisTensorize, - GraphOptimizationRuleId::PersistentTaskStripMining, - GraphOptimizationRuleId::ResidentLoadForwarding, - GraphOptimizationRuleId::IntermediatePrecisionBoundaryElision, - GraphOptimizationRuleId::StoreCoveragePlanning, - GraphOptimizationRuleId::ContiguousBlockAccessFormation, -}; - -constexpr bool hasUniqueSingleBitGraphOptimizationRuleIds() { - for (std::size_t index = 0; index < kGraphOptimizationRuleRegistry.size(); - ++index) { - const GraphOptimizationRuleMask mask = - getGraphOptimizationRuleMask(kGraphOptimizationRuleRegistry[index]); - if (mask == 0 || (mask & (mask - 1)) != 0) - return false; - for (std::size_t other = index + 1; - other < kGraphOptimizationRuleRegistry.size(); ++other) { - if (mask == - getGraphOptimizationRuleMask(kGraphOptimizationRuleRegistry[other])) - return false; - } - } - return true; -} - -static_assert(hasUniqueSingleBitGraphOptimizationRuleIds(), - "GraphOptimizationRuleId values must be unique single bits"); - -constexpr GraphOptimizationRuleMask kLegacyGraphOptimizationRuleMask = - getGraphOptimizationRuleMask(GraphOptimizationRuleId::LoadStoreTranspose) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::TransposePointwiseReorder) | - getGraphOptimizationRuleMask(GraphOptimizationRuleId::StoreCoalescing) | - getGraphOptimizationRuleMask(GraphOptimizationRuleId::RowCoalescing) | - getGraphOptimizationRuleMask(GraphOptimizationRuleId::DiagonalMaskRemoval) | - getGraphOptimizationRuleMask(GraphOptimizationRuleId::ConvertModuloToMask) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::StridedAxisCoalescing) | - getGraphOptimizationRuleMask(GraphOptimizationRuleId::ChunkCoalescing) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::StridedLoadStoreRewrite); - -constexpr GraphOptimizationRuleMask kKnownGraphOptimizationRuleMask = - kLegacyGraphOptimizationRuleMask | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::IndependentAxisTensorize) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::PersistentTaskStripMining) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::ResidentLoadForwarding) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::IntermediatePrecisionBoundaryElision) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::StoreCoveragePlanning) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::ContiguousBlockAccessFormation); - -constexpr GraphOptimizationRuleMask kDefaultEligibleGraphOptimizationRuleMask = - kLegacyGraphOptimizationRuleMask | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::IndependentAxisTensorize) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::PersistentTaskStripMining); - -constexpr GraphOptimizationRuleMask kFixedDefaultOffGraphOptimizationRuleMask = - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::ResidentLoadForwarding) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::IntermediatePrecisionBoundaryElision) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::StoreCoveragePlanning) | - getGraphOptimizationRuleMask( - GraphOptimizationRuleId::ContiguousBlockAccessFormation); - -constexpr GraphOptimizationRuleMask kDefaultGraphOptimizationRuleMask = - kDefaultEligibleGraphOptimizationRuleMask; - -constexpr GraphOptimizationRuleMask kAllGraphOptimizationRuleMask = - kKnownGraphOptimizationRuleMask; - -static_assert(kLegacyGraphOptimizationRuleMask == 511, - "legacy graph optimization rule mask is an ABI contract"); -static_assert(kDefaultGraphOptimizationRuleMask == - kDefaultEligibleGraphOptimizationRuleMask, - "IAT and PTSM must remain enabled by default"); -static_assert(kKnownGraphOptimizationRuleMask == - (kDefaultEligibleGraphOptimizationRuleMask | - kFixedDefaultOffGraphOptimizationRuleMask), - "known graph rule identities must retain their scope partition"); -static_assert((kDefaultEligibleGraphOptimizationRuleMask & - kFixedDefaultOffGraphOptimizationRuleMask) == 0, - "fixed-default-off rules must not become default candidates"); -static_assert((kDefaultGraphOptimizationRuleMask & - ~kDefaultEligibleGraphOptimizationRuleMask) == 0, - "default graph rules must stay within the approved candidates"); -static_assert(static_cast(getGraphOptimizationRulePhase( - GraphOptimizationRuleId::ResidentLoadForwarding)) < - static_cast(getGraphOptimizationRulePhase( - GraphOptimizationRuleId::StoreCoveragePlanning)), - "resident forwarding must run before store planning"); -static_assert(static_cast(getGraphOptimizationRulePhase( - GraphOptimizationRuleId::StoreCoveragePlanning)) < - static_cast(getGraphOptimizationRulePhase( - GraphOptimizationRuleId::ContiguousBlockAccessFormation)), - "store planning must run before access formation"); - -constexpr bool -isValidGraphOptimizationRuleMask(GraphOptimizationRuleMask ruleMask) { - constexpr GraphOptimizationRuleMask unknownRuleBits = - ~kKnownGraphOptimizationRuleMask; - return (ruleMask & unknownRuleBits) == 0; -} - -struct IndependentAxisTensorizeRuleOptions { - bool enabledForCompileMode = true; - bool iatAndPtsmEnabled = false; -}; -struct PersistentTaskStripMiningRuleOptions { - bool enabledForCompileMode = true; -}; -struct StoreCoalescingRuleOptions { - bool enabledForCompileMode = true; -}; -struct ResidentLoadForwardingRuleOptions {}; -struct IntermediatePrecisionBoundaryElisionRuleOptions {}; -struct StoreCoveragePlanningRuleOptions {}; -struct ContiguousBlockAccessFormationRuleOptions {}; - -struct GraphOptimizationOptions { - // A zero mask intentionally disables every native GraphOptimizationRule. - // Layout/memory compatibility stages retain their original fixed scheduling - // and do not use this option as a new opt-out. - GraphOptimizationRuleMask enabledRuleMask = kDefaultGraphOptimizationRuleMask; - unsigned maxRewritesPerFunction = 64; - unsigned ubCapacityBytes = 0; - unsigned mappingUBCapacityBytes = 0; - unsigned storeCoalescingUBBudgetBytes = 0; - unsigned ubSafetyPercent = 80; - unsigned reservedUBBytes = 0; - // RowCoalescing changes the launch grid and is valid only for - // compile_mode="simt_only". Keep the source selector rather than a - // second derived force flag so every consumer follows one mode contract. - std::string compileMode = "simd_simt_template"; - IndependentAxisTensorizeRuleOptions independentAxisTensorize; - PersistentTaskStripMiningRuleOptions persistentTaskStripMining; - StoreCoalescingRuleOptions storeCoalescing; - ResidentLoadForwardingRuleOptions residentLoadForwarding; - IntermediatePrecisionBoundaryElisionRuleOptions - intermediatePrecisionBoundaryElision; - StoreCoveragePlanningRuleOptions storeCoveragePlanning; - ContiguousBlockAccessFormationRuleOptions contiguousBlockAccessFormation; -}; - -std::unique_ptr> -createGraphOptimizePass(GraphOptimizationOptions options = {}); - -} // namespace cfg -} // namespace triton -} // namespace mlir - -#endif // TRITON_TO_GRAPH_GRAPH_OPTIMIZATION_H diff --git a/third_party/ascend/include/TritonToGraph/GraphOptimizationRule.h b/third_party/ascend/include/TritonToGraph/GraphOptimizationRule.h index 80f4010705..e3e7a99633 100644 --- a/third_party/ascend/include/TritonToGraph/GraphOptimizationRule.h +++ b/third_party/ascend/include/TritonToGraph/GraphOptimizationRule.h @@ -23,7 +23,7 @@ #ifndef TRITON_TO_GRAPH_GRAPH_OPTIMIZATION_RULE_H #define TRITON_TO_GRAPH_GRAPH_OPTIMIZATION_RULE_H -#include "TritonToGraph/GraphOptimization.h" +#include "incubated/TritonToGraph/GraphOptimization.h" #include "TritonToGraph/GraphOptimizationContext.h" #include "mlir/IR/PatternMatch.h" diff --git a/third_party/ascend/include/TritonToGraph/LayoutMemoryOptimization.h b/third_party/ascend/include/TritonToGraph/LayoutMemoryOptimization.h deleted file mode 100644 index f13b1689fc..0000000000 --- a/third_party/ascend/include/TritonToGraph/LayoutMemoryOptimization.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_TO_GRAPH_LAYOUT_MEMORY_OPTIMIZATION_H -#define TRITON_TO_GRAPH_LAYOUT_MEMORY_OPTIMIZATION_H - -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" - -#include -#include - -namespace mlir { -namespace triton { -namespace cfg { - -// These phases deliberately preserve the T2L insertion points around the -// legacy diagonal rewrite. They are not GraphOptimizationRule phases: the -// legacy rewrites are module-wide and some of them intentionally mutate IR -// while doing their analysis. -enum class LayoutMemoryCompatibilityPhase : uint8_t { - BeforeDiagonal, - AfterDiagonal, -}; - -std::unique_ptr> -createLayoutMemoryCompatibilityPass(LayoutMemoryCompatibilityPhase phase); - -} // namespace cfg -} // namespace triton -} // namespace mlir - -#endif // TRITON_TO_GRAPH_LAYOUT_MEMORY_OPTIMIZATION_H diff --git a/third_party/ascend/include/TritonToGraph/Passes.h b/third_party/ascend/include/TritonToGraph/Passes.h index 32c0142c6d..4c48fae35b 100644 --- a/third_party/ascend/include/TritonToGraph/Passes.h +++ b/third_party/ascend/include/TritonToGraph/Passes.h @@ -23,7 +23,7 @@ #ifndef TRITON_TO_CFG_PASSES_H #define TRITON_TO_CFG_PASSES_H -#include "GraphOptimization.h" +#include "incubated/TritonToGraph/GraphOptimization.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/Pass/Pass.h" diff --git a/third_party/ascend/lib/CMakeLists.txt b/third_party/ascend/lib/CMakeLists.txt index 19db07bd20..2d0c8ceb8a 100644 --- a/third_party/ascend/lib/CMakeLists.txt +++ b/third_party/ascend/lib/CMakeLists.txt @@ -2,6 +2,9 @@ add_subdirectory(AutoBlockify) add_subdirectory(Dialect) add_subdirectory(Utils) add_subdirectory(TritonToStructured) +add_subdirectory( + ${PROJECT_SOURCE_DIR}/third_party/flir/lib/TritonToGraph + ${PROJECT_BINARY_DIR}/third_party/flir/lib/TritonToGraph) add_subdirectory(TritonToGraph) add_subdirectory(TritonToAnnotation) add_subdirectory(TritonToHFusion) diff --git a/third_party/ascend/lib/TritonToGraph/CMakeLists.txt b/third_party/ascend/lib/TritonToGraph/CMakeLists.txt index 1b1866ad22..32587a7eca 100644 --- a/third_party/ascend/lib/TritonToGraph/CMakeLists.txt +++ b/third_party/ascend/lib/TritonToGraph/CMakeLists.txt @@ -7,7 +7,6 @@ add_triton_library(TritonToGraph GraphOptimizationContext.cpp ResourceCostModel.cpp GraphOptimizationPass.cpp - LayoutMemoryCompatibilityPass.cpp MemorySsaBuilder.cpp MergeConcatLoadBuffer.cpp PermutationAnalysis.cpp @@ -16,10 +15,6 @@ add_triton_library(TritonToGraph ProgramGridTransform.cpp MergeConcatLoadBuffer.cpp Passes.cpp - LegacyMemoryAccess/ChunkCoalescing.cpp - LegacyMemoryAccess/RowCoalescing.cpp - LegacyMemoryAccess/StridedAxisCoalescing.cpp - LegacyMemoryAccess/StridedLoadStoreRewrite.cpp Rules/ConvertModuloToMaskRule.cpp Rules/ReservedGraphOptimizationRule.cpp Rules/DiagonalMaskRemovalRule.cpp @@ -60,6 +55,7 @@ add_triton_library(TritonToGraph TritonAscendIR TritonIR FlirTritonMemoryAccessAnalysis + FlirTritonToGraphLegacy TritonToStructured TritonTransforms ) diff --git a/third_party/ascend/lib/TritonToGraph/LayoutMemoryCompatibilityPass.cpp b/third_party/ascend/lib/TritonToGraph/LayoutMemoryCompatibilityPass.cpp deleted file mode 100644 index 2071bdecff..0000000000 --- a/third_party/ascend/lib/TritonToGraph/LayoutMemoryCompatibilityPass.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "TritonToGraph/GraphOptimization.h" -#include "TritonToGraph/LayoutMemoryOptimization.h" -#include "TritonToGraph/LegacyMemoryAccess/ChunkCoalescing.h" -#include "TritonToGraph/LegacyMemoryAccess/StridedAxisCoalescing.h" -#include "TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h" - -#include "Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Math/IR/Math.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" -#include "triton/Dialect/Triton/IR/Dialect.h" -#include "llvm/Support/Debug.h" - -#include -#include -#include - -#define DEBUG_TYPE "graph-optimize" - -namespace mlir { -namespace triton { -namespace cfg { -namespace { - -static void emitAppliedGraphOptimizationRule(GraphOptimizationRuleId rule) { - LLVM_DEBUG(llvm::dbgs() << "[" DEBUG_TYPE "] applied graph optimization rule " - << static_cast(rule) << " (" - << getGraphOptimizationRuleName(rule) << ")\n"); -} - -static std::size_t countStridedLoadStoreRewrites(ModuleOp moduleOp) { - std::size_t count = 0; - moduleOp.walk([&](Operation *op) { - if (op->hasAttr( - StridedLoadStoreRewrite::RewrittenByStridedLoadStoreRewriteTAG)) - ++count; - }); - return count; -} - -class LayoutMemoryCompatibilityPass final - : public PassWrapper> { -public: - explicit LayoutMemoryCompatibilityPass(LayoutMemoryCompatibilityPhase phase) - : phase(phase) {} - - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LayoutMemoryCompatibilityPass) - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - ModuleOp moduleOp = getOperation(); - bool wasCoalesced = false; - switch (phase) { - case LayoutMemoryCompatibilityPhase::BeforeDiagonal: - // Preserve the original module-wide, no-op-on-bailout semantics. This - // phase intentionally has no cleanup pass. - wasCoalesced = moduleOp->hasAttr("hacc.coalesce_factor"); - StridedAxisCoalescing::rewriteStridedAxisCoalesce(moduleOp); - if (!wasCoalesced && moduleOp->hasAttr("hacc.coalesce_factor")) - emitAppliedGraphOptimizationRule( - GraphOptimizationRuleId::StridedAxisCoalescing); - return; - case LayoutMemoryCompatibilityPhase::AfterDiagonal: - // Keep the legacy order. Chunk sees Axis's module attr and consumes the - // static-grid hint before the greedy StridedLoadStore patterns run. - wasCoalesced = moduleOp->hasAttr("hacc.coalesce_factor"); - ChunkCoalescing::rewriteChunkCoalesce(moduleOp); - if (!wasCoalesced && moduleOp->hasAttr("hacc.coalesce_factor")) - emitAppliedGraphOptimizationRule( - GraphOptimizationRuleId::ChunkCoalescing); - break; - } - - // Count rewrite tags only while this DEBUG_TYPE is active. The tag lets us - // emit one pass-level match line even when several load/store ops rewrite. - bool emitStridedLoadStoreMatch = false; - LLVM_DEBUG(emitStridedLoadStoreMatch = true;); - const std::size_t stridedLoadStoreRewritesBefore = - emitStridedLoadStoreMatch ? countStridedLoadStoreRewrites(moduleOp) : 0; - RewritePatternSet patterns(&getContext()); - patterns.add( - patterns.getContext()); - if (failed(applyPatternsGreedily(moduleOp, std::move(patterns)))) { - signalPassFailure(); - return; - } - if (emitStridedLoadStoreMatch && countStridedLoadStoreRewrites(moduleOp) > - stridedLoadStoreRewritesBefore) - emitAppliedGraphOptimizationRule( - GraphOptimizationRuleId::StridedLoadStoreRewrite); - - // CSE/Canonicalizer deliberately remain in TritonToLinalg at the original - // call site. PtrAnalysis may create helper IR even on an SLS no-op, and - // the original T2L pipeline owns the unconditional cleanup timing. - } - -private: - LayoutMemoryCompatibilityPhase phase; -}; - -} // namespace - -std::unique_ptr> -createLayoutMemoryCompatibilityPass(LayoutMemoryCompatibilityPhase phase) { - return std::make_unique(phase); -} - -} // namespace cfg -} // namespace triton -} // namespace mlir diff --git a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/ChunkCoalescing.cpp b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/ChunkCoalescing.cpp index e9c257647d..d09d8e7282 100644 --- a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/ChunkCoalescing.cpp +++ b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/ChunkCoalescing.cpp @@ -20,9 +20,9 @@ * THE SOFTWARE. */ -#include "TritonToGraph/LegacyMemoryAccess/ChunkCoalescing.h" +#include "incubated/TritonToGraph/LegacyMemoryAccess/ChunkCoalescing.h" -#include "TritonToGraph/GraphOptimization.h" +#include "incubated/TritonToGraph/GraphOptimization.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Math/IR/Math.h" #include "mlir/IR/BuiltinAttributes.h" diff --git a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/RowCoalescing.cpp b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/RowCoalescing.cpp index 33bda61b6e..e17214c9ac 100644 --- a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/RowCoalescing.cpp +++ b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/RowCoalescing.cpp @@ -20,9 +20,9 @@ * THE SOFTWARE. */ -#include "TritonToGraph/LegacyMemoryAccess/RowCoalescing.h" +#include "incubated/TritonToGraph/LegacyMemoryAccess/RowCoalescing.h" -#include "TritonToGraph/GraphOptimization.h" +#include "incubated/TritonToGraph/GraphOptimization.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" #include "mlir/Dialect/Math/IR/Math.h" diff --git a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedAxisCoalescing.cpp b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedAxisCoalescing.cpp index a0c349c5d6..da389478c7 100644 --- a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedAxisCoalescing.cpp +++ b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedAxisCoalescing.cpp @@ -20,9 +20,9 @@ * THE SOFTWARE. */ -#include "TritonToGraph/LegacyMemoryAccess/StridedAxisCoalescing.h" +#include "incubated/TritonToGraph/LegacyMemoryAccess/StridedAxisCoalescing.h" -#include "TritonToGraph/GraphOptimization.h" +#include "incubated/TritonToGraph/GraphOptimization.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Math/IR/Math.h" #include "mlir/IR/BuiltinAttributes.h" diff --git a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp index fc7495fefb..2407745abe 100644 --- a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp +++ b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp @@ -20,14 +20,14 @@ * THE SOFTWARE. */ -#include "TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h" +#include "incubated/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.h" #include "incubated/TritonMemoryAccess/LoadStoreMaskAnalysis.h" #include "incubated/TritonMemoryAccess/MemoryAccessTags.h" #include "TritonToStructured/PtrAnalysis.h" #include "Utils/Utils.h" #include "Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "TritonToGraph/GraphOptimization.h" +#include "incubated/TritonToGraph/GraphOptimization.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Utils/StaticValueUtils.h" diff --git a/third_party/ascend/lib/TritonToGraph/Rules/RowCoalescingRule.cpp b/third_party/ascend/lib/TritonToGraph/Rules/RowCoalescingRule.cpp index 8bc1a56661..bdc8b24fbc 100644 --- a/third_party/ascend/lib/TritonToGraph/Rules/RowCoalescingRule.cpp +++ b/third_party/ascend/lib/TritonToGraph/Rules/RowCoalescingRule.cpp @@ -22,7 +22,7 @@ #include "TritonToGraph/DotRowCoalescing.h" #include "TritonToGraph/GraphOptimizationRule.h" -#include "TritonToGraph/LegacyMemoryAccess/RowCoalescing.h" +#include "incubated/TritonToGraph/LegacyMemoryAccess/RowCoalescing.h" #include "TritonToGraph/ProgramAxisDependenceAnalysis.h" #include "mlir/Dialect/Arith/IR/Arith.h" diff --git a/third_party/ascend/triton_ascend.cc b/third_party/ascend/triton_ascend.cc index 9d71e58f9d..46e8391fc0 100644 --- a/third_party/ascend/triton_ascend.cc +++ b/third_party/ascend/triton_ascend.cc @@ -14,8 +14,8 @@ #include "ascend/include/DiscreteMaskAccessConversion/Passes.h" #include "ascend/include/TritonControlFlowOpt/Passes.h" #include "ascend/include/TritonToAnnotation/Passes.h" -#include "ascend/include/TritonToGraph/GraphOptimization.h" -#include "ascend/include/TritonToGraph/LayoutMemoryOptimization.h" +#include "incubated/TritonToGraph/GraphOptimization.h" +#include "incubated/TritonToGraph/LayoutMemoryOptimization.h" #include "ascend/include/TritonToGraph/Passes.h" #include "ascend/include/TritonToHFusion/Passes.h" #include "ascend/include/TritonToHIVM/Passes.h" From e8cabff446b30614b73fde04e33ac3445844423e Mon Sep 17 00:00:00 2001 From: zhongsanming Date: Wed, 18 Nov 2026 11:55:03 +0800 Subject: [PATCH 5/7] [Ascend] Reuse FLIR pointer-descriptor attributes Drop the duplicate ControlFlowRewrite attribute definitions and include the FLIR incubated shared header. --- .../TritonControlFlowOpt/ControlFlowRewrite.h | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/third_party/ascend/include/TritonControlFlowOpt/ControlFlowRewrite.h b/third_party/ascend/include/TritonControlFlowOpt/ControlFlowRewrite.h index 97a44a2ad8..949f6eb9e4 100644 --- a/third_party/ascend/include/TritonControlFlowOpt/ControlFlowRewrite.h +++ b/third_party/ascend/include/TritonControlFlowOpt/ControlFlowRewrite.h @@ -24,6 +24,7 @@ #define TRITON_ASCEND_TRITON_CONTROL_FLOW_OPT_CONTROL_FLOW_REWRITE_H #include "TritonControlFlowOpt/ControlFlowAnalysis.h" +#include "incubated/TritonControlFlowOpt/ControlFlowAttrs.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinOps.h" @@ -36,33 +37,6 @@ namespace mlir::triton::controlflow { -/// Handoff marker for SCF loops whose pointer slots have already been expanded -/// into policy-owned descriptor components. TritonToLinalg consumes and -/// removes it instead of applying its legacy pointer-loop decomposition a -/// second time. -inline constexpr llvm::StringLiteral kPointerDescriptorBoundaryAttr = - "PointerDescriptorBoundary"; - -/// Handoff marker for a pointer rebuilt at a CFO control-flow boundary. The -/// defining operation's operands are the complete descriptor roots, including -/// invariant components omitted from the replacement SCF signature. -inline constexpr llvm::StringLiteral kPointerDescriptorRebuildAttr = - "PointerDescriptorRebuild"; - -/// Optional companion on a PointerDescriptorRebuild root. The `strided_1d` -/// form means the tensor-pointer offset operand is a proven rank-1 affine -/// expression rather than an opaque complete-offset carrier. -inline constexpr llvm::StringLiteral kPointerDescriptorOffsetFormAttr = - "PointerDescriptorOffsetForm"; -inline constexpr llvm::StringLiteral kStrided1DOffsetForm = "strided_1d"; - -/// Dense per-axis classification attached to a descriptor rebuild. A value -/// of one means the complete contribution of that logical axis is represented -/// by its scalar stride; zero means the axis is carried by the opaque complete -/// contribution tensor. -inline constexpr llvm::StringLiteral kPointerDescriptorStructuredAxesAttr = - "PointerDescriptorStructuredAxes"; - /// Policy-owned description of one value crossing a control-flow boundary. /// /// `components` contain every runtime value needed to rebuild the original From 5a73d64309137a7f67ba6a71208acbd691f01f78 Mon Sep 17 00:00:00 2001 From: zhongsanming Date: Wed, 18 Nov 2026 12:01:38 +0800 Subject: [PATCH 6/7] [Ascend] Consume FLIR NPU Utils Remove the duplicated Ascend Utils implementation and include/link the FLIR incubated Utils target from the Ascend passes. --- third_party/ascend/include/CMakeLists.txt | 1 - .../DiscreteMaskAccessConversionPass.h | 2 +- .../UnstructureConversionPass.h | 2 +- .../ascend/include/Utils/CMakeLists.txt | 0 third_party/ascend/include/Utils/DebugUtils.h | 126 -- .../include/Utils/InterleaveOptimization.h | 85 -- third_party/ascend/include/Utils/Utils.h | 292 ---- .../ascend/lib/AutoBlockify/AutoBlockify.cpp | 2 +- .../lib/AutoBlockify/RewriteOperation.cpp | 2 +- third_party/ascend/lib/AutoBlockify/Utils.cpp | 2 +- third_party/ascend/lib/CMakeLists.txt | 4 +- .../TritonAscend/IR/TritonAscendOps.cpp | 2 +- .../DiscreteMaskAccessConversionPass.cpp | 2 +- .../InterCoreTransferAndSync.cpp | 2 +- .../BlockPtrDecompose.cpp | 2 +- .../TritonControlFlowOpt/CFGStructuring.cpp | 2 +- .../ControlFlowRewrite.cpp | 2 +- .../TensorPtrDecompose.cpp | 2 +- .../TritonToGraph/GraphOptimizationPass.cpp | 2 +- .../StridedLoadStoreRewrite.cpp | 2 +- .../CannonicalizerConverter.cpp | 4 +- .../lib/TritonToStructured/MaskAnalysis.cpp | 2 +- .../lib/TritonToStructured/MemOpConverter.cpp | 4 +- .../lib/TritonToStructured/PtrAnalysis.cpp | 2 +- .../TritonToStructuredPass.cpp | 4 +- .../TritonToUnstructure/BubbleUpOperation.cpp | 2 +- .../TritonToUnstructure/OffsetAnalysis.cpp | 2 +- .../TritonToUnstructure/ReplaceArguments.cpp | 2 +- .../UnstructureConversionPass.cpp | 2 +- third_party/ascend/lib/Utils/CMakeLists.txt | 12 - third_party/ascend/lib/Utils/DebugUtils.cpp | 264 ---- .../lib/Utils/InterleaveOptimization.cpp | 715 --------- third_party/ascend/lib/Utils/Utils.cpp | 1337 ----------------- 33 files changed, 29 insertions(+), 2859 deletions(-) delete mode 100644 third_party/ascend/include/Utils/CMakeLists.txt delete mode 100644 third_party/ascend/include/Utils/DebugUtils.h delete mode 100644 third_party/ascend/include/Utils/InterleaveOptimization.h delete mode 100644 third_party/ascend/include/Utils/Utils.h delete mode 100644 third_party/ascend/lib/Utils/CMakeLists.txt delete mode 100644 third_party/ascend/lib/Utils/DebugUtils.cpp delete mode 100644 third_party/ascend/lib/Utils/InterleaveOptimization.cpp delete mode 100644 third_party/ascend/lib/Utils/Utils.cpp diff --git a/third_party/ascend/include/CMakeLists.txt b/third_party/ascend/include/CMakeLists.txt index 60f198550f..824221c893 100644 --- a/third_party/ascend/include/CMakeLists.txt +++ b/third_party/ascend/include/CMakeLists.txt @@ -4,7 +4,6 @@ add_subdirectory(TritonToHFusion) add_subdirectory(TritonToHIVM) add_subdirectory(TritonToGraph) add_subdirectory(TritonControlFlowOpt) -add_subdirectory(Utils) add_subdirectory(DiscreteMaskAccessConversion) add_subdirectory(TritonToUnstructure) add_subdirectory(TritonToLLVM) diff --git a/third_party/ascend/include/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.h b/third_party/ascend/include/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.h index 8bd9272fec..90cb0ab943 100644 --- a/third_party/ascend/include/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.h +++ b/third_party/ascend/include/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.h @@ -26,7 +26,7 @@ #include "mlir/Pass/Pass.h" #include "triton/Dialect/Triton/IR/Dialect.h" -#include "ascend/include/Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/IR/PatternMatch.h" #define GEN_PASS_DECL_DISCRETEMASKACCESSCONVERSION diff --git a/third_party/ascend/include/TritonToUnstructure/UnstructureConversionPass.h b/third_party/ascend/include/TritonToUnstructure/UnstructureConversionPass.h index 6cbb2b5eb0..cf13d61372 100644 --- a/third_party/ascend/include/TritonToUnstructure/UnstructureConversionPass.h +++ b/third_party/ascend/include/TritonToUnstructure/UnstructureConversionPass.h @@ -28,7 +28,7 @@ #include "triton/Dialect/Triton/IR/Dialect.h" #include "ascend/include/Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "ascend/include/Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/IR/PatternMatch.h" #define GEN_PASS_DECL_TRITONTOUNSTRUCTURE diff --git a/third_party/ascend/include/Utils/CMakeLists.txt b/third_party/ascend/include/Utils/CMakeLists.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/third_party/ascend/include/Utils/DebugUtils.h b/third_party/ascend/include/Utils/DebugUtils.h deleted file mode 100644 index 547774a5d3..0000000000 --- a/third_party/ascend/include/Utils/DebugUtils.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITON_ASCEND_UTILS_DEBUGUTILS_H -#define TRITON_ASCEND_UTILS_DEBUGUTILS_H - -#include -#include -#include - -namespace mlir { -class Operation; -class PatternRewriter; -} // namespace mlir - -//===----------------------------------------------------------------------===// -// msdebug helpers. All declarations live in mlir::triton::debug; the NOP -// insertion helpers are additionally exposed at global scope via using -// declarations (see the bottom of this header) so existing converter call -// sites remain unqualified. -// -// The NOP insertion helpers are runtime-gated via the -// LLVM_EXTRACT_DI_LOCAL_VARIABLES environment variable (see isDebugNopEnabled); -// they are not compiled out. Definitions in DebugUtils.cpp. -//===----------------------------------------------------------------------===// - -namespace mlir { -namespace triton { -namespace debug { - -//===----------------------------------------------------------------------===// -// NOP insertion helpers. -//===----------------------------------------------------------------------===// - -/// Unwrap CallSiteLoc (caller-preferring) and FusedLoc (last non-unknown) down -/// to a single representative location for tagging a debug NOP. -/// `depth` tracks recursion internally; callers should use the default (0). -/// Recursion is bounded by an implementation-defined limit (currently 16) to -/// guard against pathologically deep or cyclic location trees; on hitting the -/// limit the location is returned as-is. -Location unwrapFusedLocForDebug(Location loc, unsigned depth = 0); - -/// Insert a side-effecting `llvm.inline_asm "nop"` carrying `loc`, so the -/// source line stays anchored in the DWARF line table. No-op unless -/// LLVM_EXTRACT_DI_LOCAL_VARIABLES is enabled. Must be called before the op -/// carrying the location is erased. -void insertDebugNop(Location loc, PatternRewriter &rewriter); - -/// Anchor the source line of a mask operand: inserts a debug NOP at the -/// location of the op defining `mask`, so a masked load/store/atomic keeps a -/// breakable PC for the `mask = ...` line. No-op when `mask` is null or has no -/// defining op. -void insertDebugNopForMask(Value mask, PatternRewriter &rewriter); - -/// Insert one debug NOP per unique FileLineColLoc reachable from `loc`, -/// descending through CallSiteLoc (caller frame), NameLoc, and nested FusedLoc -/// wrappers. Keeps every distinct user source line fused into a single op -/// breakable. Falls back to a single anchor on `loc` when no concrete -/// FileLineColLoc is reachable. -void insertDebugNopForAllLines(Location loc, PatternRewriter &rewriter); - -//===----------------------------------------------------------------------===// -// Shared location analysis / rewrite helpers used by the debug passes -// (CanonicalizeDebugLocationsPass, DeduplicateDebugNopsPass). -// These do NOT self-gate; the passes gate. -//===----------------------------------------------------------------------===// - -/// True if `filename` is an inlined library/stdlib file (lives under a -/// site-packages or dist-packages tree). -bool isForeignFile(llvm::StringRef filename); - -/// True if `op` is one of our debug NOPs: llvm.inline_asm "nop", side -/// effecting, with no results and no operands. -bool isDebugNop(Operation *op); - -/// Resolve `loc` to the FileLineColLoc the user should see -- caller frame for -/// call sites (via unwrapFusedLocForDebug), descending NameLoc. -/// Returns a null FileLineColLoc when no user-facing file location is -/// reachable; callers MUST check `if (!result)` before use. -FileLineColLoc unwrapToUserFileLineCol(Location loc); - -/// Rewrite call-site locations whose callee resolves to a foreign (stdlib) -/// file so they collapse to their caller (user) frame. Recurses through -/// NameLoc / FusedLoc / nested call sites. `depth` tracks recursion -/// internally; callers should use the default (0). Recursion is bounded by an -/// implementation-defined limit (currently 16). -Location collapseForeignCallsites(Location loc, unsigned depth = 0); - -/// Whitelist-free replacement for tools::getBoolEnv -- reads the -/// LLVM_EXTRACT_DI_LOCAL_VARIABLES gate directly so no core-Triton -/// (GetEnv.hpp) change is required. The environment is read once and cached. -bool isDebugNopEnabled(); - -} // namespace debug -} // namespace triton -} // namespace mlir - -//===----------------------------------------------------------------------===// -// Backward-compatible global-scope aliases for the NOP insertion helpers, so -// the TritonToLinalg converters can keep calling them unqualified. -//===----------------------------------------------------------------------===// - -using mlir::triton::debug::insertDebugNop; -using mlir::triton::debug::insertDebugNopForAllLines; -using mlir::triton::debug::insertDebugNopForMask; - -#endif // TRITON_ASCEND_UTILS_DEBUGUTILS_H diff --git a/third_party/ascend/include/Utils/InterleaveOptimization.h b/third_party/ascend/include/Utils/InterleaveOptimization.h deleted file mode 100644 index e5f5f888fd..0000000000 --- a/third_party/ascend/include/Utils/InterleaveOptimization.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#pragma once - -#include "incubated/TritonMemoryAccess/LoadStoreMaskAnalysis.h" -#include "ascend/include/Utils/Utils.h" - -#include "triton/Dialect/Triton/IR/Dialect.h" - -#include "mlir/Dialect/Affine/IR/AffineOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Arith/Utils/Utils.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/GPU/IR/GPUDialect.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Linalg/Passes.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Utils/ReshapeOpsUtils.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Matchers.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Value.h" -#include "mlir/Support/LogicalResult.h" - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/MathExtras.h" - -#include -#include -#include -#include -#include - -namespace mlir { -namespace triton { - -enum class IndexMode : int { EVEN_MODE = 0, ODD_MODE = 1 }; - -LogicalResult -DeinterleaveStatusOptimization(triton::LoadOp op, - triton::LoadOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter); - -LogicalResult DeinterleaveStatusWithMaskOptimization( - triton::LoadOp op, triton::LoadOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter, MaskState &mstate, Value localMem); - -LogicalResult -InterleaveStatusOptimization(SmallVector materializeVec); - -LogicalResult -InterleaveStatusWithMaskOptimization(SmallVector materializeVec); - -} // namespace triton -} // namespace mlir diff --git a/third_party/ascend/include/Utils/Utils.h b/third_party/ascend/include/Utils/Utils.h deleted file mode 100644 index e74f67bec8..0000000000 --- a/third_party/ascend/include/Utils/Utils.h +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TRITONNPU_UTILS_UTILS_H -#define TRITONNPU_UTILS_UTILS_H - -#include "incubated/TritonMemoryAccess/OpFoldResultUtils.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Operation.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/IR/TypeRange.h" -#include "mlir/Transforms/DialectConversion.h" -#include "triton/Dialect/Triton/IR/Dialect.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/Support/LogicalResult.h" - -#include -#include -#include - -namespace mlir { -namespace ConverterUtils { - -const std::string GeneratedByMakeTensorPtrTAG = "GeneratedByMakeTensorPtr"; -const std::string discreteMaskAttrName = "DiscreteMask"; -const std::string mixCompileDiscreteMaskAttrName = "MixCompileDiscreteMask"; -const std::string explicitRouteToSIMDAttrName = "ExplicitRouteToSIMD"; -// The access must consume the actual loop mask; full-load/select is unsafe. -const std::string runtimeLoopMaskAttrName = "RuntimeLoopMask"; -const std::string discreteAttrName = "DiscreteMemAccess"; -const std::string continuousAttrName = "ContinuousMemAccess"; -const std::string customSrcPtrIndexAttrName = "SrcPtrIndex"; - -bool isaPermutedMemRefType(MemRefType); - -std::optional -getLastStrideOfReinterpretCastOp(memref::ReinterpretCastOp op); - -Value getTransposedValue(Value source, const Location loc, - ConversionPatternRewriter &rewriter, - llvm::ArrayRef order); - -SmallVector getNParallelLoopsAttrs(unsigned n); - -Value getScalarValue(Value operand, Location loc, - ConversionPatternRewriter &rewriter); - -memref::SubViewOp makeSubViewOp(Value src, - const llvm::SmallVector &offsets, - const llvm::SmallVector &sizes, - const Location &loc, - ConversionPatternRewriter &rewriter); - -tensor::ExtractSliceOp -makeExtractSliceOp(Value src, const llvm::SmallVector &offsets, - const llvm::SmallVector &sizes, - const Location &loc, ConversionPatternRewriter &rewriter); - -std::optional getFullShapeOp(Value val, - ConversionPatternRewriter &rewriter); - -SmallVector -getBoundarySizes(llvm::ArrayRef boundaryCheck, Value ptr, - const Location &loc, ConversionPatternRewriter &rewriter); - -bool requiresVolatileIndirectLoad(Value srcPtr, Operation *loadOp); - -SmallVector getBroadcastDims(RankedTensorType src, - RankedTensorType dst); - -SmallVector getUnbroadcastDims(RankedTensorType src, - RankedTensorType dst); - -} // namespace ConverterUtils - -class ConversionPatternRewriter; - -namespace triton { - -namespace ascend { - -// Keep the C++ pass contract aligned with the public Python selector. The -// Python boundary performs the target validation and canonicalization; pass -// parsing remains strict so direct pass-pipeline users cannot silently turn an -// unknown string into SIMD. -enum class CompileMode { - Simd, - SimtTemplate, - SimtOnly, -}; - -inline std::optional parseCompileMode(llvm::StringRef mode) { - if (mode == "simd") - return CompileMode::Simd; - if (mode == "simd_simt_template" || mode == "unstructured_in_simt") - return CompileMode::SimtTemplate; - if (mode == "simt_only") - return CompileMode::SimtOnly; - return std::nullopt; -} - -inline bool isSimtTemplateMode(CompileMode mode) { - return mode == CompileMode::SimtTemplate; -} - -inline bool isPureSimtMode(CompileMode mode) { - return mode == CompileMode::SimtOnly; -} - -} // namespace ascend - -enum class IndirectLoadInterfaceOpType { Undefined = 0, Load = 1, Calc = 2 }; - -// Traceback from rootOp to find the targetOp with the specified condition -mlir::Operation * -findFirstMatchingOperandDef(mlir::Operation *rootOp, - const std::function &condFn); - -void traverseBackwardUpdateOperandChainIf( - Operation *op, std::function conditionFn, - std::function stopFn, - std::function actionFn, OpBuilder &builder, - DenseSet &handledOperation); - -void traverseBackwardUpdateOperandChainIf( - Operation *rootOp, std::function conditionFn, - std::function stopFn, - std::function actionFn); - -void traverseForwardUpdateUserChainIf( - Operation *op, std::function conditionFn, - std::function stopFn, - std::function actionFn, OpBuilder &builder, - llvm::SmallPtrSet &stopOps); - -void traverseForwardUpdateUserChainIf( - Operation *rootOp, std::function conditionFn, - std::function stopFn, - std::function actionFn, - llvm::SmallPtrSet &stopOps); - -// UseAnalysis will tag operations whose results are used only as meta-data -// with "MetaUse" tag. -bool isMetaUse(Operation *op); - -bool isMixUse(Operation *op); - -IndirectLoadInterfaceOpType getIndirectLoadInterfaceOpType(Operation *op); - -bool opIsIndirectLoad(Operation *op); - -bool opIsIndirectCalc(Operation *op); - -/// Maximum expected rank for loop tiling in tensor operations. -static constexpr int kMaxTiledRank = 4; - -/// This function generates a series of `scf.for` loops for the given dimensions -/// in `loopDims`. Although the loops are created sequentially, nesting is -/// simulated by adjusting the insertion point to the body of the last created -/// loop. This allows the `bodyFunc` to be inserted into the innermost scope. -/// -/// \param rewriter The MLIR OpBuilder used to create operations. -/// \param loc The source location information for debuggability. -/// \param target The memref value whose dimensions are being looped over. -/// \param loopDims An array of dimension indices to create loops for. -/// \param bodyFunc A callable that defines the operations to insert in the -/// innermost loop. -/// It takes a SmallVector of induction variables (one per -/// loop). -/// -template -void createSimpleNestedLoops(OpBuilder &rewriter, Location loc, Value target, - ArrayRef loopDims, Func bodyFunc) { - MemRefType type = cast(target.getType()); - int rank = type.getRank(); - - Value zero = rewriter.create(loc, 0); - Value one = rewriter.create(loc, 1); - - llvm::SmallVector loops; - llvm::SmallVector ivs; - - for (int dim : loopDims) { - Value ub; - if (type.isDynamicDim(dim)) { - ub = rewriter.create(loc, target, dim).getResult(); - } else { - ub = rewriter.create(loc, type.getDimSize(dim)); - } - - auto forOp = rewriter.create(loc, zero, ub, one); - rewriter.setInsertionPointToStart(forOp.getBody()); - loops.push_back(forOp); - ivs.push_back(forOp.getInductionVar()); - } - - bodyFunc(ivs); - - if (!loops.empty()) { - rewriter.setInsertionPointAfter(loops.front()); - } -} - -scf::ForOp createNestedLoops( - OpBuilder &builder, Location loc, unsigned currentDim, unsigned totalDims, - ValueRange LBs, ValueRange UBs, ValueRange steps, SmallVector &ivs, - ValueRange initArgs, - function_ref &, ValueRange)> - bodyBuilder); - -ModuleOp getModuleOpFromOperation(Operation *op); - -bool isTensorPtrType(Type type); - -namespace ascend { - -// Fractal (zN) block geometry shared by ascend.dot's type inference/verifier -// (DotOp::inferReturnTypes) and its lowering (DotConverter). The inner block is -// [kFractalBlock, b] with b = kBytesPerFractalCol / elemBytes, i.e. f16/bf16 -> -// 16, f32 -> 8, int8 -> 32; fractal_c is the L0C accumulator fractal, always -// [kFractalBlock, kFractalBlock]. -inline constexpr int64_t kFractalBlock = 16; -inline constexpr int64_t kBytesPerFractalCol = 32; -inline constexpr unsigned kBitsPerByte = 8; -// Cube accumulator width for integer inputs (int8 -> i32). -inline constexpr unsigned kDotAccIntWidth = 32; - -} // namespace ascend - -} // namespace triton - -class OpBuilder; - -/// Returns true when both ranges have identical ordered type signatures. -bool haveSameTypes(TypeRange lhs, TypeRange rhs); - -enum class ReduceWithIndexType { MAX, MIN, None }; -enum class TieBreakType { LEFT, RIGHT, None }; - -struct ReduceWithIndexParams { - ReduceWithIndexType withIndexType = ReduceWithIndexType::None; - TieBreakType tieBreakType = TieBreakType::None; - bool isUnsignedSrc; -}; - -llvm::FailureOr -getReduceWithIndexParams(triton::ReduceOp op); - -void addReduceWithIndexAttr(ReduceWithIndexParams params, - ConversionPatternRewriter &rewriter, - linalg::ReduceOp reduceOp); - -enum class TypelessValue { Undefined = 0, Zero = 1, Min = 2, Max = 3 }; - -FailureOr specializeTypelessValueToAttr(TypelessValue, Type, - OpBuilder &); - -FailureOr specializeTypelessValueToConstant(TypelessValue, Type, - Location, OpBuilder &); - -bool checkStructureAnnotated(Operation *op, RewriterBase &rewriter); - -bool isDistributedTypeCustomOp(Operation *op); -} // namespace mlir - -#endif // TRITONNPU_UTILS_UTILS_H diff --git a/third_party/ascend/lib/AutoBlockify/AutoBlockify.cpp b/third_party/ascend/lib/AutoBlockify/AutoBlockify.cpp index 009b34dee9..8a90a60331 100644 --- a/third_party/ascend/lib/AutoBlockify/AutoBlockify.cpp +++ b/third_party/ascend/lib/AutoBlockify/AutoBlockify.cpp @@ -23,7 +23,7 @@ #include "AutoBlockify/AutoBlockify.h" #include "AutoBlockify/Utils.h" #include "Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" #include "mlir/Pass/PassManager.h" diff --git a/third_party/ascend/lib/AutoBlockify/RewriteOperation.cpp b/third_party/ascend/lib/AutoBlockify/RewriteOperation.cpp index 2a1847149b..5ec508fe6f 100644 --- a/third_party/ascend/lib/AutoBlockify/RewriteOperation.cpp +++ b/third_party/ascend/lib/AutoBlockify/RewriteOperation.cpp @@ -22,7 +22,7 @@ #include "AutoBlockify/AutoBlockify.h" #include "AutoBlockify/Utils.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "llvm/Support/Debug.h" diff --git a/third_party/ascend/lib/AutoBlockify/Utils.cpp b/third_party/ascend/lib/AutoBlockify/Utils.cpp index ee3b81badd..a4052192ab 100644 --- a/third_party/ascend/lib/AutoBlockify/Utils.cpp +++ b/third_party/ascend/lib/AutoBlockify/Utils.cpp @@ -21,7 +21,7 @@ */ #include "AutoBlockify/Utils.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "llvm/Support/Debug.h" diff --git a/third_party/ascend/lib/CMakeLists.txt b/third_party/ascend/lib/CMakeLists.txt index 2d0c8ceb8a..cc405e0904 100644 --- a/third_party/ascend/lib/CMakeLists.txt +++ b/third_party/ascend/lib/CMakeLists.txt @@ -1,6 +1,8 @@ +add_subdirectory( + ${PROJECT_SOURCE_DIR}/third_party/flir/lib/TritonNPUUtils + ${PROJECT_BINARY_DIR}/third_party/flir/lib/TritonNPUUtils) add_subdirectory(AutoBlockify) add_subdirectory(Dialect) -add_subdirectory(Utils) add_subdirectory(TritonToStructured) add_subdirectory( ${PROJECT_SOURCE_DIR}/third_party/flir/lib/TritonToGraph diff --git a/third_party/ascend/lib/Dialect/TritonAscend/IR/TritonAscendOps.cpp b/third_party/ascend/lib/Dialect/TritonAscend/IR/TritonAscendOps.cpp index 86e512c9db..606ba457fd 100644 --- a/third_party/ascend/lib/Dialect/TritonAscend/IR/TritonAscendOps.cpp +++ b/third_party/ascend/lib/Dialect/TritonAscend/IR/TritonAscendOps.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// #include "ascend/include/Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "ascend/include/Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/Dialect/SPIRV/IR/TargetAndABI.h" #include "mlir/Dialect/Utils/StaticValueUtils.h" #include "triton/Tools/Sys/GetEnv.hpp" diff --git a/third_party/ascend/lib/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.cpp b/third_party/ascend/lib/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.cpp index 33655cfac6..f842b57c23 100644 --- a/third_party/ascend/lib/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.cpp +++ b/third_party/ascend/lib/DiscreteMaskAccessConversion/DiscreteMaskAccessConversionPass.cpp @@ -20,7 +20,7 @@ * THE SOFTWARE. */ -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "ascend/include/DiscreteMaskAccessConversion/Passes.h" #include "TritonControlFlowOpt/ControlFlowRewrite.h" diff --git a/third_party/ascend/lib/DynamicCVPipeline/SplitDataflow/InterCoreTransferAndSync.cpp b/third_party/ascend/lib/DynamicCVPipeline/SplitDataflow/InterCoreTransferAndSync.cpp index b520ee1530..bb76276c40 100644 --- a/third_party/ascend/lib/DynamicCVPipeline/SplitDataflow/InterCoreTransferAndSync.cpp +++ b/third_party/ascend/lib/DynamicCVPipeline/SplitDataflow/InterCoreTransferAndSync.cpp @@ -34,7 +34,7 @@ #include "ascend/include/DynamicCVPipeline/SplitDataflow/DataDependencyAnalysis.h" #include "ascend/include/DynamicCVPipeline/SplitDataflow/FlagIdReuse.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "bishengir/Dialect/Annotation/IR/Annotation.h" #include "bishengir/Dialect/HIVM/IR/HIVM.h" #include "bishengir/Dialect/HIVM/IR/HIVMInterfaces.h" diff --git a/third_party/ascend/lib/TritonControlFlowOpt/BlockPtrDecompose.cpp b/third_party/ascend/lib/TritonControlFlowOpt/BlockPtrDecompose.cpp index 2e03309024..72bbae0a98 100644 --- a/third_party/ascend/lib/TritonControlFlowOpt/BlockPtrDecompose.cpp +++ b/third_party/ascend/lib/TritonControlFlowOpt/BlockPtrDecompose.cpp @@ -23,7 +23,7 @@ #include "TritonControlFlowOpt/BlockPtrDecompose.h" #include "TritonControlFlowOpt/ControlFlowRewrite.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "triton/Dialect/Triton/IR/Dialect.h" diff --git a/third_party/ascend/lib/TritonControlFlowOpt/CFGStructuring.cpp b/third_party/ascend/lib/TritonControlFlowOpt/CFGStructuring.cpp index a8c9553f15..fbb8198f15 100644 --- a/third_party/ascend/lib/TritonControlFlowOpt/CFGStructuring.cpp +++ b/third_party/ascend/lib/TritonControlFlowOpt/CFGStructuring.cpp @@ -21,7 +21,7 @@ */ #include "TritonControlFlowOpt/CFGStructuring.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" #include "mlir/Dialect/Func/IR/FuncOps.h" diff --git a/third_party/ascend/lib/TritonControlFlowOpt/ControlFlowRewrite.cpp b/third_party/ascend/lib/TritonControlFlowOpt/ControlFlowRewrite.cpp index 529dca83eb..86a0208861 100644 --- a/third_party/ascend/lib/TritonControlFlowOpt/ControlFlowRewrite.cpp +++ b/third_party/ascend/lib/TritonControlFlowOpt/ControlFlowRewrite.cpp @@ -21,7 +21,7 @@ */ #include "TritonControlFlowOpt/ControlFlowRewrite.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "bishengir/Dialect/Scope/IR/Scope.h" #include "mlir/Dialect/SCF/IR/SCF.h" diff --git a/third_party/ascend/lib/TritonControlFlowOpt/TensorPtrDecompose.cpp b/third_party/ascend/lib/TritonControlFlowOpt/TensorPtrDecompose.cpp index 0ecfd91a28..ce31a00de7 100644 --- a/third_party/ascend/lib/TritonControlFlowOpt/TensorPtrDecompose.cpp +++ b/third_party/ascend/lib/TritonControlFlowOpt/TensorPtrDecompose.cpp @@ -23,7 +23,7 @@ #include "TritonControlFlowOpt/TensorPtrDecompose.h" #include "TritonControlFlowOpt/ControlFlowRewrite.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/SCF/IR/SCF.h" diff --git a/third_party/ascend/lib/TritonToGraph/GraphOptimizationPass.cpp b/third_party/ascend/lib/TritonToGraph/GraphOptimizationPass.cpp index dd7e8d4018..8557cb98cb 100644 --- a/third_party/ascend/lib/TritonToGraph/GraphOptimizationPass.cpp +++ b/third_party/ascend/lib/TritonToGraph/GraphOptimizationPass.cpp @@ -25,7 +25,7 @@ #include "TritonToGraph/Passes.h" #include "TritonToGraph/ProgramGridTransform.h" #include "TritonToGraph/ResourceCostModel.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" diff --git a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp index 2407745abe..0e9bc563fb 100644 --- a/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp +++ b/third_party/ascend/lib/TritonToGraph/LegacyMemoryAccess/StridedLoadStoreRewrite.cpp @@ -24,7 +24,7 @@ #include "incubated/TritonMemoryAccess/LoadStoreMaskAnalysis.h" #include "incubated/TritonMemoryAccess/MemoryAccessTags.h" #include "TritonToStructured/PtrAnalysis.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "Dialect/TritonAscend/IR/TritonAscendDialect.h" #include "incubated/TritonToGraph/GraphOptimization.h" diff --git a/third_party/ascend/lib/TritonToStructured/CannonicalizerConverter.cpp b/third_party/ascend/lib/TritonToStructured/CannonicalizerConverter.cpp index 58229ed06b..fd9b6bca2c 100644 --- a/third_party/ascend/lib/TritonToStructured/CannonicalizerConverter.cpp +++ b/third_party/ascend/lib/TritonToStructured/CannonicalizerConverter.cpp @@ -62,8 +62,8 @@ #include "TritonToStructured/PtrAnalysis.h" #include "TritonToStructured/TritonToStructuredPass.h" -#include "Utils/InterleaveOptimization.h" -#include "Utils/Utils.h" +#include "incubated/Utils/InterleaveOptimization.h" +#include "incubated/Utils/Utils.h" #include "bishengir/Dialect/Annotation/IR/Annotation.h" #define DEBUG_TYPE "triton-cannonicalizer-converter" diff --git a/third_party/ascend/lib/TritonToStructured/MaskAnalysis.cpp b/third_party/ascend/lib/TritonToStructured/MaskAnalysis.cpp index e638e1994c..94168493b9 100644 --- a/third_party/ascend/lib/TritonToStructured/MaskAnalysis.cpp +++ b/third_party/ascend/lib/TritonToStructured/MaskAnalysis.cpp @@ -60,7 +60,7 @@ #include "triton/Dialect/Triton/IR/Types.h" #include "TritonToStructured/PtrAnalysis.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #define DEBUG_TYPE "triton-to-structured-mask-analysis" diff --git a/third_party/ascend/lib/TritonToStructured/MemOpConverter.cpp b/third_party/ascend/lib/TritonToStructured/MemOpConverter.cpp index ea9a29427c..fd858ef452 100644 --- a/third_party/ascend/lib/TritonToStructured/MemOpConverter.cpp +++ b/third_party/ascend/lib/TritonToStructured/MemOpConverter.cpp @@ -64,8 +64,8 @@ #include "TritonToStructured/MaskAnalysis.h" #include "TritonToStructured/PtrAnalysis.h" #include "TritonToStructured/TritonToStructuredPass.h" -#include "Utils/InterleaveOptimization.h" -#include "Utils/Utils.h" +#include "incubated/Utils/InterleaveOptimization.h" +#include "incubated/Utils/Utils.h" #include "bishengir/Dialect/Annotation/IR/Annotation.h" #include "bishengir/Dialect/HIVM/Utils/Utils.h" diff --git a/third_party/ascend/lib/TritonToStructured/PtrAnalysis.cpp b/third_party/ascend/lib/TritonToStructured/PtrAnalysis.cpp index ed5acbf6fc..127e762658 100644 --- a/third_party/ascend/lib/TritonToStructured/PtrAnalysis.cpp +++ b/third_party/ascend/lib/TritonToStructured/PtrAnalysis.cpp @@ -60,7 +60,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/LogicalResult.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #define DEBUG_TYPE "triton-to-structured-ptr-analysis" diff --git a/third_party/ascend/lib/TritonToStructured/TritonToStructuredPass.cpp b/third_party/ascend/lib/TritonToStructured/TritonToStructuredPass.cpp index 5446efdffa..ebf7b34763 100644 --- a/third_party/ascend/lib/TritonToStructured/TritonToStructuredPass.cpp +++ b/third_party/ascend/lib/TritonToStructured/TritonToStructuredPass.cpp @@ -26,8 +26,8 @@ #include #include -#include "Utils/InterleaveOptimization.h" -#include "Utils/Utils.h" +#include "incubated/Utils/InterleaveOptimization.h" +#include "incubated/Utils/Utils.h" #include "bishengir/Dialect/HIVM/IR/HIVM.h" #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" #include "mlir/Dialect/Func/IR/FuncOps.h" diff --git a/third_party/ascend/lib/TritonToUnstructure/BubbleUpOperation.cpp b/third_party/ascend/lib/TritonToUnstructure/BubbleUpOperation.cpp index c1efb5990a..e29018fe1e 100644 --- a/third_party/ascend/lib/TritonToUnstructure/BubbleUpOperation.cpp +++ b/third_party/ascend/lib/TritonToUnstructure/BubbleUpOperation.cpp @@ -21,7 +21,7 @@ */ #include "TritonToUnstructure/BubbleUpOperation.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/Pass/PassManager.h" #include "mlir/Transforms/GreedyPatternRewriteDriver.h" diff --git a/third_party/ascend/lib/TritonToUnstructure/OffsetAnalysis.cpp b/third_party/ascend/lib/TritonToUnstructure/OffsetAnalysis.cpp index 0bae477acc..68b6bd0933 100644 --- a/third_party/ascend/lib/TritonToUnstructure/OffsetAnalysis.cpp +++ b/third_party/ascend/lib/TritonToUnstructure/OffsetAnalysis.cpp @@ -22,7 +22,7 @@ #include "TritonToUnstructure/OffsetAnalysis.h" #include "TritonControlFlowOpt/ControlFlowRewrite.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "bishengir/Dialect/Annotation/IR/Annotation.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" diff --git a/third_party/ascend/lib/TritonToUnstructure/ReplaceArguments.cpp b/third_party/ascend/lib/TritonToUnstructure/ReplaceArguments.cpp index 76ec6825a6..4b956bcad9 100644 --- a/third_party/ascend/lib/TritonToUnstructure/ReplaceArguments.cpp +++ b/third_party/ascend/lib/TritonToUnstructure/ReplaceArguments.cpp @@ -22,7 +22,7 @@ #include "TritonControlFlowOpt/ControlFlowRewrite.h" #include "TritonToUnstructure/UnstructureConversionPass.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "mlir/IR/Block.h" diff --git a/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp b/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp index fda2a91498..726946465a 100644 --- a/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp +++ b/third_party/ascend/lib/TritonToUnstructure/UnstructureConversionPass.cpp @@ -24,7 +24,7 @@ #include "incubated/TritonMemoryAccess/LoadStoreMaskAnalysis.h" #include "TritonToStructured/CannonicalizerConverter.h" #include "TritonToUnstructure/IndirectAtomicUtils.h" -#include "Utils/Utils.h" +#include "incubated/Utils/Utils.h" #include "triton/Dialect/Triton/IR/Dialect.h" #include "bishengir/Dialect/Annotation/IR/Annotation.h" diff --git a/third_party/ascend/lib/Utils/CMakeLists.txt b/third_party/ascend/lib/Utils/CMakeLists.txt deleted file mode 100644 index db3f60034b..0000000000 --- a/third_party/ascend/lib/Utils/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -add_triton_library(MLIRTritonNPUUtils - Utils.cpp - InterleaveOptimization.cpp - DebugUtils.cpp - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - TritonIR - FlirTritonMemoryAccessAnalysis - FlirTritonMemoryAccessCore -) diff --git a/third_party/ascend/lib/Utils/DebugUtils.cpp b/third_party/ascend/lib/Utils/DebugUtils.cpp deleted file mode 100644 index 4272a79ef2..0000000000 --- a/third_party/ascend/lib/Utils/DebugUtils.cpp +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -//===----------------------------------------------------------------------===// -// DebugUtils.cpp -// -// Implementations for DebugUtils.h: -// * NOP insertion helpers, runtime-gated via the -// LLVM_EXTRACT_DI_LOCAL_VARIABLES environment variable, and -// * the shared location analysis / rewrite helpers used by -// CanonicalizeDebugLocationsPass and DeduplicateDebugNopsPass. -// -// All definitions live in mlir::triton::debug; DebugUtils.h re-exports the NOP -// helpers at global scope via using declarations. -//===----------------------------------------------------------------------===// - -#include "Utils/DebugUtils.h" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -using namespace mlir; - -namespace mlir { -namespace triton { -namespace debug { - -// Guard against pathologically deep / cyclic location trees. -static constexpr unsigned kMaxLocDepth = 16; - -//===----------------------------------------------------------------------===// -// NOP insertion helpers. -//===----------------------------------------------------------------------===// - -static bool moduleContainsScope(PatternRewriter &rewriter) { - Block *insertionBlock = rewriter.getInsertionBlock(); - if (!insertionBlock) - return false; - - Operation *parentOp = insertionBlock->getParentOp(); - if (!parentOp) - return false; - - ModuleOp module = dyn_cast(parentOp); - if (!module) - module = parentOp->getParentOfType(); - if (!module) - return false; - - bool found = false; - module.walk([&](Operation *op) { - if (op->getName().getStringRef() != "scope.scope") - return WalkResult::advance(); - found = true; - return WalkResult::interrupt(); - }); - return found; -} - -Location unwrapFusedLocForDebug(Location loc, unsigned depth) { - if (depth > kMaxLocDepth) - return loc; - if (auto cs = dyn_cast(loc)) - return unwrapFusedLocForDebug(cs.getCaller(), depth + 1); - if (auto fused = dyn_cast(loc)) { - for (auto inner : llvm::reverse(fused.getLocations())) { - if (!isa(inner)) - return unwrapFusedLocForDebug(inner, depth + 1); - } - } - return loc; -} - -void insertDebugNop(Location loc, PatternRewriter &rewriter) { - if (!isDebugNopEnabled() || moduleContainsScope(rewriter)) - return; - auto unwrapped = unwrapFusedLocForDebug(loc); - - auto ctx = rewriter.getContext(); - rewriter.create( - unwrapped, - /*resultTypes=*/TypeRange(), - /*operands=*/ValueRange(), - /*asm_string=*/"nop", - /*constraints=*/"", - /*has_side_effects=*/true, - /*is_align_stack=*/false, LLVM::tailcallkind::TailCallKind::None, - LLVM::AsmDialectAttr::get(ctx, LLVM::AsmDialect::AD_ATT), ArrayAttr()); -} - -void insertDebugNopForMask(Value mask, PatternRewriter &rewriter) { - if (!mask) - return; - if (Operation *def = mask.getDefiningOp()) - insertDebugNop(def->getLoc(), rewriter); -} - -// Collect every distinct user FileLineColLoc reachable through NameLoc / -// CallSiteLoc / (nested) FusedLoc wrappers. Locations are context-uniqued, so -// the Location handle itself is an exact dedup key (file, line, column). -static void collectUserLineLocs(Location loc, - llvm::SmallDenseSet &seen, - llvm::SmallVectorImpl &out, - unsigned depth = 0) { - if (depth > kMaxLocDepth) - return; - if (auto cs = dyn_cast(loc)) { - collectUserLineLocs(cs.getCaller(), seen, out, depth + 1); - return; - } - if (auto named = dyn_cast(loc)) { - collectUserLineLocs(named.getChildLoc(), seen, out, depth + 1); - return; - } - if (auto fused = dyn_cast(loc)) { - for (Location inner : fused.getLocations()) - collectUserLineLocs(inner, seen, out, depth + 1); - return; - } - if (auto flc = dyn_cast(loc)) - if (seen.insert(flc).second) - out.push_back(loc); -} - -void insertDebugNopForAllLines(Location loc, PatternRewriter &rewriter) { - if (!isDebugNopEnabled() || moduleContainsScope(rewriter)) - return; - - llvm::SmallDenseSet seen; - llvm::SmallVector lineLocs; - collectUserLineLocs(loc, seen, lineLocs); - if (lineLocs.empty()) { - insertDebugNop(loc, rewriter); - return; - } - for (Location l : lineLocs) - insertDebugNop(l, rewriter); -} - -//===----------------------------------------------------------------------===// -// Shared location analysis / rewrite helpers (no self-gating; passes gate). -//===----------------------------------------------------------------------===// - -bool isForeignFile(llvm::StringRef filename) { - // A library/stdlib file inlined into the kernel (e.g. triton/language/ - // standard.py). Heuristic: lives under a site-packages or dist-packages tree - return filename.contains("/site-packages/") || - filename.contains("/dist-packages/"); -} - -bool isDebugNop(Operation *op) { - auto asmOp = dyn_cast(op); - if (!asmOp) - return false; - if (!asmOp.getHasSideEffects()) - return false; - if (asmOp.getAsmString() != "nop") - return false; - // NOPs have no results and no operands. - if (asmOp->getNumResults() != 0 || asmOp->getNumOperands() != 0) - return false; - return true; -} - -FileLineColLoc unwrapToUserFileLineCol(Location loc) { - // Reuse the existing caller-preferring unwrapper, then descend NameLoc to the - // underlying FileLineColLoc. callsite(stdlib at user) -> the user's line. - Location l = unwrapFusedLocForDebug(loc); - while (auto named = dyn_cast(l)) - l = named.getChildLoc(); - return dyn_cast(l); -} - -Location collapseForeignCallsites(Location loc, unsigned depth) { - if (depth > kMaxLocDepth) - return loc; - - if (auto named = dyn_cast(loc)) { - Location child = collapseForeignCallsites(named.getChildLoc(), depth + 1); - return child == named.getChildLoc() - ? loc - : Location(NameLoc::get(named.getName(), child)); - } - - if (auto cs = dyn_cast(loc)) { - Location caller = collapseForeignCallsites(cs.getCaller(), depth + 1); - // The callee is the inlined helper frame; resolve its own file. (The - // callee is not itself a call site, so caller-preference is moot here.) - if (FileLineColLoc calleeFlc = unwrapToUserFileLineCol(cs.getCallee())) - if (isForeignFile(calleeFlc.getFilename().getValue())) - return caller; - Location callee = collapseForeignCallsites(cs.getCallee(), depth + 1); - if (callee == cs.getCallee() && caller == cs.getCaller()) - return loc; - return Location(CallSiteLoc::get(callee, caller)); - } - - if (auto fused = dyn_cast(loc)) { - llvm::SmallVector newLocs; - bool changed = false; - for (Location sub : fused.getLocations()) { - Location c = collapseForeignCallsites(sub, depth + 1); - changed |= (c != sub); - newLocs.push_back(c); - } - return changed ? Location(FusedLoc::get(loc.getContext(), newLocs, - fused.getMetadata())) - : loc; - } - - return loc; -} - -bool isDebugNopEnabled() { - // Environment variables do not change mid-process; read once. Function-local - // static initialisation is thread-safe since C++11. - static const bool enabled = [] { - const char *s = std::getenv("LLVM_EXTRACT_DI_LOCAL_VARIABLES"); - if (!s) - return false; - std::string v(s); - std::transform(v.begin(), v.end(), v.begin(), - [](unsigned char c) { return std::tolower(c); }); - return v == "1" || v == "true" || v == "on"; - }(); - return enabled; -} - -} // namespace debug -} // namespace triton -} // namespace mlir diff --git a/third_party/ascend/lib/Utils/InterleaveOptimization.cpp b/third_party/ascend/lib/Utils/InterleaveOptimization.cpp deleted file mode 100644 index 1637d67673..0000000000 --- a/third_party/ascend/lib/Utils/InterleaveOptimization.cpp +++ /dev/null @@ -1,715 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/Utils/InterleaveOptimization.h" -#include "ascend/include/Utils/Utils.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/BuiltinTypeInterfaces.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/ViewLikeInterface.h" -#include "mlir/Support/LogicalResult.h" - -#include "mlir/IR/Operation.h" -#include "triton/Dialect/Triton/IR/Dialect.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" -#include -#include -#include -#include - -namespace mlir { -namespace triton { -// For origin MemRefType of ReinterpretCastOp under interleave state, here wanna -// adjust its shape info by expanding last dimension double. -static MemRefType expandInterleaveMemRefType( - MemRefType originType, - std::optional castOffset = std::nullopt) { - // Double the last dimension shape - SmallVector shape(originType.getShape()); - shape.back() = shape.back() * 2; - - // Adjuest layout attribute - StridedLayoutAttr originLayout = - llvm::dyn_cast(originType.getLayout()); - // If offset is static, just reset it to 0. - auto offset = originLayout.getOffset() == ShapedType::kDynamic - ? originLayout.getOffset() - : 0; - // A reinterpret_cast result type must agree with its offset operand. The - // original interleaved view can carry layout offset 0 while the normalized - // base is a non-zero static offset, so use the new cast offset whenever it - // is available. - if (castOffset) { - if (auto staticOffset = getConstantIntValue(*castOffset)) - offset = staticOffset.value(); - else - offset = ShapedType::kDynamic; - } - // Set last dimension stride to 1 - SmallVector stride(originLayout.getStrides()); - stride.back() = 1; - - return MemRefType::get( - shape, originType.getElementType(), - StridedLayoutAttr::get(originType.getContext(), offset, stride)); -} - -// The last dimension of an interleaved view has stride 2. The address can be -// expressed as an arbitrary base plus a lane selector (0 for even, 1 for odd): -// -// base + 2 * lane + {0, 1} -// -// The base is allowed to contain static and dynamic offsets. In particular, -// a constant such as the RoPE region offset (+64) is part of the base, not an -// odd-lane marker. Keep the analysis side-effect free so callers can safely -// return failure and use the normal lowering path. -struct DeinterleaveOffsetInfo { - SmallVector baseTerms; - int64_t baseConstant = 0; - IndexMode indexMode = IndexMode::EVEN_MODE; - // Reuse the original SSA base whenever the outermost +1 unambiguously - // selects the odd lane. This preserves the existing offset def-use chain - // instead of recreating an equivalent arithmetic expression during type - // conversion. - std::optional materializedBaseOffset; -}; - -static bool addStaticOffset(int64_t offset, int64_t &sum) { - if ((offset > 0 && sum > std::numeric_limits::max() - offset) || - (offset < 0 && sum < std::numeric_limits::min() - offset)) - return false; - sum += offset; - return true; -} - -static bool collectDeinterleaveOffsetTerms(OpFoldResult offset, - SmallVectorImpl &baseTerms, - int64_t &staticOffset) { - if (auto constant = getConstantIntValue(offset)) - return addStaticOffset(constant.value(), staticOffset); - - auto value = llvm::dyn_cast(offset); - if (!value) - return false; - - if (auto addOp = value.getDefiningOp()) { - return collectDeinterleaveOffsetTerms(addOp.getLhs(), baseTerms, - staticOffset) && - collectDeinterleaveOffsetTerms(addOp.getRhs(), baseTerms, - staticOffset); - } - - // The expression below this value is an opaque base term. Do not recurse - // into it: a constant in a multiply/addptr/base-address subgraph is not a - // lane selector. - baseTerms.push_back(value); - return true; -} - -static FailureOr -normalizeDeinterleaveOffset(OpFoldResult originOffset) { - DeinterleaveOffsetInfo result; - int64_t staticOffset = 0; - if (!collectDeinterleaveOffsetTerms(originOffset, result.baseTerms, - staticOffset)) - return failure(); - - for (Value term : result.baseTerms) { - if (!isa(term.getType())) - return failure(); - } - - // Keep the base pair-aligned relative to the original view. This also - // supports folded constants, e.g. +65 becomes base +64 and ODD_MODE. - int64_t laneOffset = staticOffset % 2; - if (laneOffset < 0) - laneOffset += 2; - result.baseConstant = staticOffset - laneOffset; - result.indexMode = - laneOffset == 0 ? IndexMode::EVEN_MODE : IndexMode::ODD_MODE; - - if (auto value = llvm::dyn_cast(originOffset)) { - Value base = value; - if (result.indexMode == IndexMode::ODD_MODE) { - auto addOp = value.getDefiningOp(); - if (addOp) { - if (auto lhsConstant = getConstantIntValue(addOp.getLhs()); - lhsConstant && lhsConstant.value() == 1) { - base = addOp.getRhs(); - result.materializedBaseOffset = base; - } else if (auto rhsConstant = getConstantIntValue(addOp.getRhs()); - rhsConstant && rhsConstant.value() == 1) { - base = addOp.getLhs(); - result.materializedBaseOffset = base; - } - } - } else { - result.materializedBaseOffset = base; - } - } - return result; -} - -static bool hasEquivalentDeinterleaveBase(const DeinterleaveOffsetInfo &lhs, - const DeinterleaveOffsetInfo &rhs) { - if (lhs.baseConstant != rhs.baseConstant || - lhs.baseTerms.size() != rhs.baseTerms.size()) - return false; - - // Addition is commutative. Compare the opaque terms as a multiset so that - // equivalent add trees with a different operand order remain optimizable. - SmallVector matched(rhs.baseTerms.size(), false); - for (Value lhsTerm : lhs.baseTerms) { - bool found = false; - for (auto [index, rhsTerm] : llvm::enumerate(rhs.baseTerms)) { - if (!matched[index] && lhsTerm == rhsTerm) { - matched[index] = true; - found = true; - break; - } - } - if (!found) - return false; - } - return true; -} - -static OpFoldResult -materializeDeinterleaveBaseOffset(const DeinterleaveOffsetInfo &offsetInfo, - OpBuilder &builder, Location loc) { - if (offsetInfo.materializedBaseOffset) - return *offsetInfo.materializedBaseOffset; - if (offsetInfo.baseTerms.empty()) - return builder.getIndexAttr(offsetInfo.baseConstant); - - Value base = offsetInfo.baseTerms.front(); - for (Value term : llvm::drop_begin(offsetInfo.baseTerms)) - base = builder.create(loc, base, term); - - if (offsetInfo.baseConstant != 0) { - Value constant = builder.create( - loc, builder.getIndexAttr(offsetInfo.baseConstant)); - base = builder.create(loc, base, constant); - } - return base; -} - -LogicalResult -DeinterleaveStatusOptimization(triton::LoadOp op, - triton::LoadOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter) { - auto ptr = adaptor.getPtr(); - if (auto reinterpretCast = ptr.getDefiningOp()) { - auto loc = op.getLoc(); - - // 1. Normalize the original view offset, then create the expanded cast. - auto originCastOffset = reinterpretCast.getConstifiedMixedOffset(); - auto castSize = reinterpretCast.getConstifiedMixedSizes(); - auto castStride = reinterpretCast.getConstifiedMixedStrides(); - // Actually, `castSize` is always constant value as `MemRefType` result - if (auto lastDimSize = getConstantIntValue(castSize.back())) { - castSize.back() = rewriter.getIndexAttr(lastDimSize.value() * 2); - } else { - return failure(); - } - // Last element of castStride is also constant value as prerequisite - // is that last dimension stride of casted memref type is always 2. - castStride.back() = rewriter.getIndexAttr(1); - auto offsetInfo = normalizeDeinterleaveOffset(originCastOffset); - if (failed(offsetInfo)) - return failure(); - OpFoldResult castOffset = - materializeDeinterleaveBaseOffset(*offsetInfo, rewriter, loc); - auto srcType = - expandInterleaveMemRefType(reinterpretCast.getType(), castOffset); - auto newCastOp = rewriter.create( - loc, srcType, reinterpretCast.getViewSource(), castOffset, castSize, - castStride); - - // 3. Create new memref allocOp - auto newAllocOp = rewriter.create( - loc, MemRefType::get(srcType.getShape(), srcType.getElementType())); - - // 4. Implement memref copy and bufferization back to tensor - rewriter.create(loc, newCastOp.getResult(), newAllocOp); - Value newTensor = rewriter.create( - loc, - RankedTensorType::get(srcType.getShape(), srcType.getElementType()), - newAllocOp, true /* restrict */, true /* writable */); - - // 5. Implement tensor extract_slice to represent deinterleave - // Here use `castOffset` to determine whether even index deinterleave or - // odd index. - SmallVector extractOffsets(srcType.getRank(), - rewriter.getIndexAttr(0)); - SmallVector extractStrides(srcType.getRank(), - rewriter.getIndexAttr(1)); - SmallVector extractSizes = llvm::to_vector( - llvm::map_range(srcType.getShape(), [&](int64_t dim) -> OpFoldResult { - return rewriter.getIndexAttr(dim); - })); - - // Adjust extract_slice shape - switch (offsetInfo->indexMode) { - case IndexMode::EVEN_MODE: - extractOffsets.back() = rewriter.getIndexAttr(0); - break; - case IndexMode::ODD_MODE: - extractOffsets.back() = rewriter.getIndexAttr(1); - break; - } - extractStrides.back() = rewriter.getIndexAttr(2); - extractSizes.back() = rewriter.getIndexAttr(srcType.getShape().back() / 2); - - Value deinterleaveSlice = rewriter.create( - loc, newTensor, extractOffsets, extractSizes, extractStrides); - - rewriter.replaceOp(op, deinterleaveSlice); - return success(); - } - - return failure(); -} - -LogicalResult DeinterleaveStatusWithMaskOptimization( - triton::LoadOp op, triton::LoadOp::Adaptor adaptor, - ConversionPatternRewriter &rewriter, MaskState &mstate, Value localMem) { - auto ptr = adaptor.getPtr(); - if (auto reinterpretCast = ptr.getDefiningOp()) { - auto loc = op.getLoc(); - - // 1. Normalize the original view offset, then create the expanded cast. - auto originCastOffset = reinterpretCast.getConstifiedMixedOffset(); - auto castSize = reinterpretCast.getConstifiedMixedSizes(); - auto castStride = reinterpretCast.getConstifiedMixedStrides(); - - if (auto lastDimSize = getConstantIntValue(castSize.back())) { - castSize.back() = rewriter.getIndexAttr(lastDimSize.value() * 2); - } else { - return failure(); - } - castStride.back() = rewriter.getIndexAttr(1); - auto offsetInfo = normalizeDeinterleaveOffset(originCastOffset); - if (failed(offsetInfo)) - return failure(); - OpFoldResult castOffset = - materializeDeinterleaveBaseOffset(*offsetInfo, rewriter, loc); - auto srcType = - expandInterleaveMemRefType(reinterpretCast.getType(), castOffset); - - auto newCastOp = rewriter.create( - loc, srcType, reinterpretCast.getViewSource(), castOffset, castSize, - castStride); - - // 3. Create new memref allocOp - // To reuse existing linalg::fill, here need to change insertion point - auto savedInsertPoint = rewriter.saveInsertionPoint(); - rewriter.setInsertionPointAfterValue(localMem); - auto newAllocOp = rewriter.create( - loc, MemRefType::get(srcType.getShape(), srcType.getElementType())); - rewriter.restoreInsertionPoint(savedInsertPoint); - - // 4. Broadcast other value by linalg.fill if necessary - auto other = op.getOther(); - // While deinterleave optimization will just adjust last dimension info - // and origin mask state wouldn't involve last dimension. Therefore in - // current `scf.if + linalg.fill` combination, condition of `if` could be - // kept and just replace linalg.fill' - if (other) { - assert(localMem.hasOneUse() && - llvm::isa(*(localMem.getUsers().begin()))); - auto originFillOp = - llvm::dyn_cast(*(localMem.getUsers().begin())); - - assert(llvm::isa(originFillOp->getParentOp())); - auto ifOp = llvm::dyn_cast(originFillOp->getParentOp()); - - auto newFillOp = ifOp.getThenBodyBuilder().create( - originFillOp.getLoc(), originFillOp.getInputs(), - ValueRange{newAllocOp}); - rewriter.replaceOp(originFillOp, newFillOp); - } - - // 5. Implement new subview, memref copy and bufferization back to tensor - SmallVector subviewStrides(srcType.getRank(), - rewriter.getIndexAttr(1)); - SmallVector subviewOffsets = mstate.offsets; - SmallVector subviewSizes = mstate.dims; - // Just adjust last dimension size to double - std::optional originSubviewLastDim = - getConstantIntValue(subviewSizes.back()); - assert(originSubviewLastDim.has_value()); - subviewSizes.back() = - rewriter.getIndexAttr(originSubviewLastDim.value() * 2); - - auto argSubviewType = memref::SubViewOp::inferResultType( - srcType, subviewOffsets, subviewSizes, subviewStrides); - // alloca subview type doesn't carry layout attribute - auto allocSubviewType = memref::SubViewOp::inferResultType( - newAllocOp.getType(), subviewOffsets, subviewSizes, subviewStrides); - - memref::SubViewOp srcSubview = rewriter.create( - loc, llvm::cast(argSubviewType), newCastOp, subviewOffsets, - subviewSizes, subviewStrides); - memref::SubViewOp dstSubview = rewriter.create( - loc, llvm::cast(allocSubviewType), newAllocOp, - subviewOffsets, subviewSizes, subviewStrides); - rewriter.create(loc, srcSubview, dstSubview); - Value newTensor = rewriter.create( - loc, - RankedTensorType::get(srcType.getShape(), srcType.getElementType()), - newAllocOp, true /* restrict */, true /* writable */); - - // 6. Implement tensor extract_slice to represent deinterleave - // Here use `castOffset` to determine whether even index deinterleave or - // odd index. - SmallVector extractOffsets(srcType.getRank(), - rewriter.getIndexAttr(0)); - SmallVector extractStrides(srcType.getRank(), - rewriter.getIndexAttr(1)); - SmallVector extractSizes = llvm::to_vector( - llvm::map_range(srcType.getShape(), [&](int64_t dim) -> OpFoldResult { - return rewriter.getIndexAttr(dim); - })); - - switch (offsetInfo->indexMode) { - case IndexMode::EVEN_MODE: - extractOffsets.back() = rewriter.getIndexAttr(0); - break; - case IndexMode::ODD_MODE: - extractOffsets.back() = rewriter.getIndexAttr(1); - break; - } - extractStrides.back() = rewriter.getIndexAttr(2); - extractSizes.back() = rewriter.getIndexAttr(srcType.getShape().back() / 2); - - Value deinterleaveSlice = rewriter.create( - loc, newTensor, extractOffsets, extractSizes, extractStrides); - - rewriter.replaceOp(op, deinterleaveSlice); - return success(); - } - return failure(); -} - -LogicalResult -InterleaveStatusOptimization(SmallVector materializeVec) { - OpBuilder builder(materializeVec[1]); - auto loc = materializeVec[1]->getLoc(); - - auto firstReinterpretCastOp = - llvm::dyn_cast( - materializeVec[0]) - .getDest() - .getDefiningOp(); - auto secondReinterpretCastOp = - llvm::dyn_cast( - materializeVec[1]) - .getDest() - .getDefiningOp(); - - assert(firstReinterpretCastOp && secondReinterpretCastOp); - // Judge whether two `ReinterpretCastOp` shape satisfy interleave state - // a. both size are equal - if (!isEqualConstantIntOrValueArray( - firstReinterpretCastOp.getConstifiedMixedSizes(), - secondReinterpretCastOp.getConstifiedMixedSizes())) { - return failure(); - } - // b. both strides are equal - if (!isEqualConstantIntOrValueArray( - firstReinterpretCastOp.getConstifiedMixedStrides(), - secondReinterpretCastOp.getConstifiedMixedStrides())) { - return failure(); - } - // c. both offsets should satisfy tricky rule - auto firstOriginCastOffset = - firstReinterpretCastOp.getConstifiedMixedOffset(); - auto secondOriginCastOffset = - secondReinterpretCastOp.getConstifiedMixedOffset(); - auto firstOffsetInfo = normalizeDeinterleaveOffset(firstOriginCastOffset); - auto secondOffsetInfo = normalizeDeinterleaveOffset(secondOriginCastOffset); - if (failed(firstOffsetInfo) || failed(secondOffsetInfo) || - !hasEquivalentDeinterleaveBase(*firstOffsetInfo, *secondOffsetInfo) || - firstOffsetInfo->indexMode == secondOffsetInfo->indexMode) - return failure(); - - std::pair indexModeRecord = { - firstOffsetInfo->indexMode, secondOffsetInfo->indexMode}; - OpFoldResult newCastOffset = - materializeDeinterleaveBaseOffset(*firstOffsetInfo, builder, loc); - - // Create new op - // 1. Get new destination memref type - auto dstType = expandInterleaveMemRefType(firstReinterpretCastOp.getType(), - newCastOffset); - - // 2. New tensor::EmptyOp - auto emptyTensor = builder.create(loc, dstType.getShape(), - dstType.getElementType()); - - // 3. New insert_slice from materialization source into new empty tensor - SmallVector insertOffsets(dstType.getRank(), - builder.getIndexAttr(0)); - SmallVector insertStrides(dstType.getRank(), - builder.getIndexAttr(1)); - SmallVector insertSizes = llvm::to_vector( - llvm::map_range(dstType.getShape(), [&](int64_t dim) -> OpFoldResult { - return builder.getIndexAttr(dim); - })); - insertStrides.back() = builder.getIndexAttr(2); - insertSizes.back() = builder.getIndexAttr(dstType.getShape().back() / 2); - if (indexModeRecord.first == IndexMode::ODD_MODE) { - insertOffsets.back() = builder.getIndexAttr(1); - } else { - insertOffsets.back() = builder.getIndexAttr(0); - } - auto insertFirst = builder.create( - loc, - llvm::dyn_cast( - materializeVec[0]) - .getSource(), - emptyTensor.getResult(), insertOffsets, insertSizes, insertStrides); - - if (indexModeRecord.second == IndexMode::ODD_MODE) { - insertOffsets.back() = builder.getIndexAttr(1); - } else { - insertOffsets.back() = builder.getIndexAttr(0); - } - auto insertSecond = builder.create( - loc, - llvm::dyn_cast( - materializeVec[1]) - .getSource(), - insertFirst.getResult(), insertOffsets, insertSizes, insertStrides); - - // 4. Reinterpret_cast block arg - auto newCastSize = firstReinterpretCastOp.getConstifiedMixedSizes(); - auto newCastStride = firstReinterpretCastOp.getConstifiedMixedStrides(); - newCastSize.back() = builder.getIndexAttr(dstType.getShape().back()); - newCastStride.back() = builder.getIndexAttr(1); - auto newCastOp = builder.create( - loc, dstType, firstReinterpretCastOp.getViewSource(), newCastOffset, - newCastSize, newCastStride); - - // 5. Create new bufferization::MaterializeInDestinationOp - auto newStoreOp = builder.create( - loc, insertSecond.getResult(), newCastOp.getResult()); - // Setting writable is necessary as dst is memref type - newStoreOp.setWritable(true); - - // 6. Erase origin materialization - materializeVec[0]->erase(); - materializeVec[1]->erase(); - - return success(); -} - -LogicalResult -InterleaveStatusWithMaskOptimization(SmallVector materializeVec) { - OpBuilder builder(materializeVec[1]); - - auto firstSubviewOpOfReCast = - llvm::dyn_cast( - materializeVec[0]) - .getDest() - .getDefiningOp(); - auto firstSrcExtractSlice = - llvm::dyn_cast( - materializeVec[0]) - .getSource() - .getDefiningOp(); - auto firstReinterpretCastOp = firstSubviewOpOfReCast.getSource() - .getDefiningOp(); - - auto secondSubviewOpOfReCast = - llvm::dyn_cast( - materializeVec[1]) - .getDest() - .getDefiningOp(); - auto secondSrcExtractSlice = - llvm::dyn_cast( - materializeVec[1]) - .getSource() - .getDefiningOp(); - auto secondReinterpretCastOp = - secondSubviewOpOfReCast.getSource() - .getDefiningOp(); - - // 1. Both source shapes of subview and extract_slice are equal - if (firstSubviewOpOfReCast.getSourceType().getShape() != - firstSrcExtractSlice.getSourceType().getShape()) - return failure(); - if (secondSubviewOpOfReCast.getSourceType().getShape() != - secondSrcExtractSlice.getSourceType().getShape()) - return failure(); - if (firstSubviewOpOfReCast.getSourceType().getShape() != - secondSubviewOpOfReCast.getSourceType().getShape()) - return failure(); - - // 2. both mask state are equal - std::function cmpFunc = - mlir::isEqualConstantIntOrValue; - if (!mlir::detail::sameOffsetsSizesAndStrides(firstSubviewOpOfReCast, - firstSrcExtractSlice, cmpFunc)) - return failure(); - if (!mlir::detail::sameOffsetsSizesAndStrides(secondSubviewOpOfReCast, - secondSrcExtractSlice, cmpFunc)) - return failure(); - if (!mlir::detail::sameOffsetsSizesAndStrides( - firstSubviewOpOfReCast, secondSubviewOpOfReCast, cmpFunc)) - return failure(); - - // 3. Still judge whether two `ReinterpretCastOp` shape satisfy request - // a. both size are equal - if (!isEqualConstantIntOrValueArray( - firstReinterpretCastOp.getConstifiedMixedSizes(), - secondReinterpretCastOp.getConstifiedMixedSizes())) - return failure(); - // b. both strides are equal - if (!isEqualConstantIntOrValueArray( - firstReinterpretCastOp.getConstifiedMixedStrides(), - secondReinterpretCastOp.getConstifiedMixedStrides())) - return failure(); - // c. both offsets should satisfy tricky rule - auto firstOriginCastOffset = - firstReinterpretCastOp.getConstifiedMixedOffset(); - auto secondOriginCastOffset = - secondReinterpretCastOp.getConstifiedMixedOffset(); - auto firstOffsetInfo = normalizeDeinterleaveOffset(firstOriginCastOffset); - auto secondOffsetInfo = normalizeDeinterleaveOffset(secondOriginCastOffset); - if (failed(firstOffsetInfo) || failed(secondOffsetInfo) || - !hasEquivalentDeinterleaveBase(*firstOffsetInfo, *secondOffsetInfo) || - firstOffsetInfo->indexMode == secondOffsetInfo->indexMode) - return failure(); - - std::pair indexModeRecord = { - firstOffsetInfo->indexMode, secondOffsetInfo->indexMode}; - OpFoldResult newCastOffset = materializeDeinterleaveBaseOffset( - *firstOffsetInfo, builder, materializeVec[1]->getLoc()); - auto loc = materializeVec[1]->getLoc(); - - // Create new op - // 1. Get new destination memref type - auto dstType = expandInterleaveMemRefType(firstReinterpretCastOp.getType(), - newCastOffset); - - // 2. New tensor::EmptyOp - auto emptyTensor = builder.create(loc, dstType.getShape(), - dstType.getElementType()); - - // 3. New insert_slice from extract_slice source into new empty tensor - SmallVector insertOffsets(dstType.getRank(), - builder.getIndexAttr(0)); - SmallVector insertStrides(dstType.getRank(), - builder.getIndexAttr(1)); - SmallVector insertSizes = llvm::to_vector( - llvm::map_range(dstType.getShape(), [&](int64_t dim) -> OpFoldResult { - return builder.getIndexAttr(dim); - })); - insertStrides.back() = builder.getIndexAttr(2); - insertSizes.back() = builder.getIndexAttr(dstType.getShape().back() / 2); - if (indexModeRecord.first == IndexMode::ODD_MODE) { - insertOffsets.back() = builder.getIndexAttr(1); - } else { - insertOffsets.back() = builder.getIndexAttr(0); - } - auto insertFirst = builder.create( - loc, firstSrcExtractSlice.getSource(), emptyTensor.getResult(), - insertOffsets, insertSizes, insertStrides); - - if (indexModeRecord.second == IndexMode::ODD_MODE) { - insertOffsets.back() = builder.getIndexAttr(1); - } else { - insertOffsets.back() = builder.getIndexAttr(0); - } - auto insertSecond = builder.create( - loc, secondSrcExtractSlice.getSource(), insertFirst.getResult(), - insertOffsets, insertSizes, insertStrides); - - // 4. To enable store with mask, create new extract_slice - SmallVector extractOffsets = - firstSrcExtractSlice.getMixedOffsets(); - SmallVector extractStrides = - firstSrcExtractSlice.getMixedStrides(); - SmallVector extractSizes = firstSrcExtractSlice.getMixedSizes(); - if (!llvm::isa(extractSizes.back())) { - return failure(); - } - extractSizes.back() = builder.getIndexAttr( - getConstantIntValue(extractSizes.back()).value() * 2); - auto newSrcExtractSlice = builder.create( - loc, insertSecond.getResult(), extractOffsets, extractSizes, - extractStrides); - - // 5. Reinterpret_cast block arg - auto newCastSize = firstReinterpretCastOp.getConstifiedMixedSizes(); - auto newCastStride = firstReinterpretCastOp.getConstifiedMixedStrides(); - newCastSize.back() = builder.getIndexAttr(dstType.getShape().back()); - newCastStride.back() = builder.getIndexAttr(1); - auto newCastOp = builder.create( - loc, dstType, firstReinterpretCastOp.getViewSource(), newCastOffset, - newCastSize, newCastStride); - - // 6. Create new memref::SubViewOp of above new reinterpret_cast - // Here could reuse shape info of new extract_slice - auto dstSubviewType = memref::SubViewOp::inferResultType( - dstType, extractOffsets, extractSizes, extractStrides); - auto newSubviewOpOfReCast = builder.create( - loc, llvm::cast(dstSubviewType), newCastOp, extractOffsets, - extractSizes, extractStrides); - - // 7. Create new bufferization::MaterializeInDestinationOp - auto newStoreOp = builder.create( - loc, newSrcExtractSlice.getResult(), newSubviewOpOfReCast.getResult()); - // Setting writable is necessary as dst is memref type - newStoreOp.setWritable(true); - - // 8. Erase origin operation - materializeVec[0]->erase(); - materializeVec[1]->erase(); - if (firstSubviewOpOfReCast->use_empty()) { - firstSubviewOpOfReCast->erase(); - } - if (firstSrcExtractSlice->use_empty()) { - firstSrcExtractSlice->erase(); - } - if (secondSubviewOpOfReCast->use_empty()) { - secondSubviewOpOfReCast->erase(); - } - if (secondSrcExtractSlice->use_empty()) { - secondSrcExtractSlice->erase(); - } - - return success(); -} - -} // namespace triton -} // namespace mlir diff --git a/third_party/ascend/lib/Utils/Utils.cpp b/third_party/ascend/lib/Utils/Utils.cpp deleted file mode 100644 index ebe9bc7e23..0000000000 --- a/third_party/ascend/lib/Utils/Utils.cpp +++ /dev/null @@ -1,1337 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "ascend/include/Utils/Utils.h" - -#include "ascend/include/Dialect/TritonAscend/IR/TritonAscendDialect.h" -#include "bishengir/Dialect/Annotation/IR/Annotation.h" -#include "bishengir/Dialect/HIVM/IR/HIVM.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/IR/Attributes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinTypeInterfaces.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Diagnostics.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/IR/Operation.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/IR/Value.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "mlir/Interfaces/LoopLikeInterface.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "triton/Dialect/Triton/IR/Dialect.h" -#include "triton/Dialect/Triton/IR/Types.h" - -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/SmallVectorExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/LogicalResult.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define DEBUG_TYPE "TritonNPU-Utils" - -namespace mlir { - -namespace ConverterUtils { - -std::optional -getLastStrideOfReinterpretCastOp(memref::ReinterpretCastOp op) { - SmallVector mixedStrides = op.getMixedStrides(); - if (mixedStrides.empty()) { - op->emitError("ReinterpretCastOp has no strides"); - return std::nullopt; - } - - OpFoldResult lastStride = mixedStrides.back(); - - if (op.getStaticStrides().back() != ShapedType::kDynamic) { - return op.getStaticStrides().back(); - } else if (isa(op.getStrides().back())) { - auto u = op.getStrides().back(); - while (auto blkArg = dyn_cast(u)) { - if (auto forOp = dyn_cast(blkArg.getOwner()->getParentOp())) { - auto prt = forOp->getOperand(3 + blkArg.getArgNumber() - 1); - u = prt; - } else { - u = nullptr; - break; - } - } - if (!u) - return std::nullopt; - lastStride = u; - } - - if (auto attr = lastStride.dyn_cast()) { - return getConstantOfAttr(lastStride); - } else if (auto value = lastStride.dyn_cast()) { - auto defOp = value.getDefiningOp(); - if (auto constIndexOp = dyn_cast(defOp)) { - int64_t constValue = constIndexOp.value(); - return constValue; - } else if (auto constIntOp = dyn_cast(defOp)) { - int64_t constValue = constIntOp.value(); - return constValue; - } - } - return std::nullopt; -} - -bool isaPermutedMemRefType(MemRefType memRefType) { - auto [ptrStrides, ptrOffsets] = memRefType.getStridesAndOffset(); - LLVM_DEBUG({ - llvm::dbgs() << "---------- [BEG] ptrStrides ----------\n"; - for (auto stride : ptrStrides) - llvm::dbgs() << stride << " "; - llvm::dbgs() << "\n"; - llvm::dbgs() << "---------- [END] ptrStrides ----------\n"; - }); - - switch (ptrStrides.size()) { - case 0: - return false; - case 1: - return false; - default: { - return ptrStrides[ptrStrides.size() - 1] != 1; - } - } -} - -Value getTransposedValue(Value source, const Location loc, - ConversionPatternRewriter &rewriter, - llvm::ArrayRef order) { - auto sourceType = cast(source.getType()); - auto sourceRank = sourceType.getRank(); - - SmallVector perm(order); - SmallVector originalShape(sourceType.getShape()); - SmallVector transposedShape(sourceRank); - for (size_t i = 0; i < sourceRank; i++) { - transposedShape[i] = originalShape[perm[i]]; - } - - Value transposeInit = rewriter.create( - loc, transposedShape, sourceType.getElementType()); - - Value transpose = - rewriter.create(loc, source, transposeInit, perm) - .getResults()[0]; - - return transpose; -} - -SmallVector getNParallelLoopsAttrs(unsigned n) { - return SmallVector(n, utils::IteratorType::parallel); -} - -Value getScalarValue(Value operand, Location loc, - ConversionPatternRewriter &rewriter) { - // Peel splat / cast / select chains down to a scalar; nullptr if non-splat - // tensor. - SmallVector ops; - auto reconstructScalarValue = [&](Value src) { - for (auto op = ops.rbegin(); op != ops.rend(); ++op) { - src = mlir::TypeSwitch(*op) - .Case([&](Operation *op) { - auto resType = op->getResults()[0].getType(); - if (auto shapedType = dyn_cast(resType)) { - resType = shapedType.getElementType(); - } - return rewriter.create(loc, resType, src); - }) - .Case([&](Operation *op) { - auto resType = op->getResults()[0].getType(); - if (auto shapedType = dyn_cast(resType)) { - resType = shapedType.getElementType(); - } - return rewriter.create(loc, resType, src); - }) - .Case([&](Operation *op) { - auto resType = op->getResults()[0].getType(); - if (auto shapedType = dyn_cast(resType)) { - resType = shapedType.getElementType(); - } - return rewriter.create(loc, resType, src); - }) - .Default([](Operation *op) { - llvm_unreachable("unsupported op in generating "); - return nullptr; - }); - } - return src; - }; - - while (true) { - if (!dyn_cast(operand.getType())) { - return reconstructScalarValue(operand); - } else if (auto op = operand.getDefiningOp()) { - if (auto attr = dyn_cast(op.getValue())) { - if (!attr.isSplat()) - return nullptr; - auto elemValue = attr.getSplatValue(); - auto constOp = arith::ConstantOp::materialize( - rewriter, elemValue, attr.getElementType(), op.getLoc()); - return reconstructScalarValue(constOp.getResult()); - } - return nullptr; - } else if (auto op = operand.getDefiningOp()) { - operand = op.getSrc(); - } else if (auto op = operand.getDefiningOp()) { - // Descriptor padding: select(i1, splat_nan, splat_zero) -> scalar select. - Value trueVal = getScalarValue(op.getTrueValue(), loc, rewriter); - Value falseVal = getScalarValue(op.getFalseValue(), loc, rewriter); - if (!trueVal || !falseVal) - return nullptr; - Value cond = op.getCondition(); - if (isa(cond.getType())) { - if (auto splat = cond.getDefiningOp()) - cond = splat.getSrc(); - else if (auto c = cond.getDefiningOp()) { - if (auto attr = dyn_cast(c.getValue()); - attr && attr.isSplat()) { - cond = arith::ConstantOp::materialize( - rewriter, attr.getSplatValue(), - attr.getElementType(), c.getLoc()); - } else { - return nullptr; - } - } else { - return nullptr; - } - } - if (isa(cond.getType())) - return nullptr; - Value scalarSelect = rewriter.create( - loc, trueVal.getType(), cond, trueVal, falseVal); - return reconstructScalarValue(scalarSelect); - } else if (auto op = operand.getDefiningOp()) { - ops.push_back(op.getOperation()); - operand = op.getIn(); - } else if (auto op = operand.getDefiningOp()) { - ops.push_back(op.getOperation()); - operand = op.getIn(); - } else if (auto op = operand.getDefiningOp()) { - ops.push_back(op.getOperation()); - operand = op.getIn(); - } else { - return nullptr; - } - } - return nullptr; -} - -memref::SubViewOp makeSubViewOp(Value src, - const llvm::SmallVector &offsets, - const llvm::SmallVector &sizes, - const Location &loc, - ConversionPatternRewriter &rewriter) { - auto srcType = cast(src.getType()); - SmallVector strides(srcType.getRank(), - rewriter.getIndexAttr(1)); - auto dstType = - memref::SubViewOp::inferResultType(srcType, offsets, sizes, strides); - return rewriter.create(loc, dyn_cast(dstType), - src, offsets, sizes, strides); -} - -tensor::ExtractSliceOp -makeExtractSliceOp(Value src, const llvm::SmallVector &offsets, - const llvm::SmallVector &sizes, - const Location &loc, ConversionPatternRewriter &rewriter) { - auto srcType = cast(src.getType()); - SmallVector strides(srcType.getRank(), - rewriter.getIndexAttr(1)); - auto dstType = - tensor::ExtractSliceOp::inferResultType(srcType, offsets, sizes, strides); - return rewriter.create(loc, dstType, src, offsets, - sizes, strides); -} - -std::optional getFullShapeOp(Value val, - ConversionPatternRewriter &rewriter) { - while (true) { - if (isa(val)) { - auto blockArg = dyn_cast(val); - Operation *parentOp = blockArg.getOwner()->getParentOp(); - - // When BlockArgument is from the scf.for loop, trace its initial value - if (auto forOp = dyn_cast_or_null(parentOp)) { - auto init = forOp.getTiedLoopInit(blockArg); - if (!init) - return std::nullopt; - val = init->get(); - continue; - } - - // When BlockArgument is not scf::ForOp but FuncOp, it means that shape - // information can no longer be tracked. In this case, std::nullopt is - // returned, and getBoundarySizes() is called to return the current shape - // as the boundary. - if (isa(parentOp)) { - return std::nullopt; - } - - emitWarning(val.getLoc()) - << "getFullShapeOp() only support ReinterpretCastOp, " - "UnrealizedConversionCastOp " - "and scf.for's block argument, but got : " - << val << "\n"; - return std::nullopt; - } - - Operation *defOp = val.getDefiningOp(); - if (!defOp) - return std::nullopt; - - if (auto castOp = dyn_cast(defOp)) { - if (castOp.getInputs().size() == 1) { - val = castOp.getInputs()[0]; - continue; - } - return std::nullopt; - } - - if (!isa(val.getType())) - return std::nullopt; - - if (auto reCastOp = dyn_cast(defOp)) { - if (reCastOp->hasAttr("tensor_ptr_full_shape")) - return reCastOp; - val = reCastOp.getSource(); - continue; - } - - emitWarning(val.getLoc()) - << "getFullShapeOp() only support ReinterpretCastOp, " - "UnrealizedConversionCastOp " - "and scf.for's block argument, but got : " - << val << "\n"; - return std::nullopt; - } -} - -SmallVector -getBoundarySizes(llvm::ArrayRef boundaryCheck, Value ptr, - const Location &loc, ConversionPatternRewriter &rewriter) { - if (isa(ptr.getType())) { - ptr = rewriter.getRemappedValue(ptr); - } - - auto shapedType = dyn_cast_if_present(ptr.getType()); - if (!shapedType) { - LLVM_DEBUG(llvm::dbgs() << "ptr is not a ShapedType.\n";); - return {}; - } - - if (!shapedType.hasStaticShape()) { - LLVM_DEBUG(llvm::dbgs() << "shapedType does not have a static shape\n";); - return {}; - } - - auto fullShapeOp = getFullShapeOp(ptr, rewriter); - if (!fullShapeOp.has_value()) { - // If fullShapeOp has no value, the current shape is returned as the - // boundary. - SmallVector boundarySize = - getAsIndexOpFoldResult(rewriter.getContext(), shapedType.getShape()); - - return boundarySize; - } - - SmallVector boundarySize = - getAsIndexOpFoldResult(rewriter.getContext(), shapedType.getShape()); - - auto fullShapeReCast = - dyn_cast(fullShapeOp.value()); - if (!fullShapeReCast) { - return getAsIndexOpFoldResult(rewriter.getContext(), shapedType.getShape()); - } - - OpFoldResult curPtrOffset; - if (auto curReCast = ptr.getDefiningOp()) { - curPtrOffset = curReCast.getConstifiedMixedOffset(); - } else if (isa(ptr) && - isa(ptr.getParentBlock()->getParentOp())) { - // Here's to process loop state where ptr is just from loop interator. - // Following assertion corresponds to conversion result from `rewriteFor` - auto blockArg = dyn_cast(ptr); - auto forOp = dyn_cast(ptr.getParentBlock()->getParentOp()); - auto initReCastOfLoop = forOp.getTiedLoopInit(blockArg) - ->get() - .getDefiningOp(); - assert(initReCastOfLoop && initReCastOfLoop.getOffsets().size() == 1); - Value initReCastOffset = initReCastOfLoop.getOffsets()[0]; - - for (OpOperand &use : initReCastOffset.getUses()) { - if (use.getOwner() == initReCastOfLoop) - continue; - else if (isa(use.getOwner())) - continue; - else if (use.getOwner() == forOp) - curPtrOffset = OpFoldResult(forOp.getTiedLoopRegionIterArg(&use)); - else - llvm_unreachable("Illegal interation offset after rewriteFor"); - } - } else { - llvm_unreachable("Unsupported state when check tensor_ptr boundary"); - } - - assert(curPtrOffset); - - OpFoldResult offsetShift = subOpFoldResult( - curPtrOffset, fullShapeReCast.getConstifiedMixedOffset(), loc, rewriter); - - auto strides = fullShapeReCast.getConstifiedMixedStrides(); - SmallVector axisOrder; - axisOrder.reserve(shapedType.getRank()); - for (int i = 0; i < shapedType.getRank(); ++i) - axisOrder.push_back(i); - - bool allStridesConstant = llvm::all_of(strides, [](OpFoldResult stride) { - return getConstantIntValue(stride).has_value(); - }); - if (allStridesConstant) { - llvm::stable_sort(axisOrder, [&strides](int lhs, int rhs) { - return std::abs(*getConstantIntValue(strides[lhs])) > - std::abs(*getConstantIntValue(strides[rhs])); - }); - } - - for (int i : axisOrder) { - OpFoldResult curStride = strides[i]; - if (llvm::find(boundaryCheck, i) != boundaryCheck.end()) { - if (isZero(curStride)) { - emitWarning(loc) - << "getBoundarySizes() cannot reconstruct boundary on checked " - "zero-stride axis " - << i << "; keep current block size for this axis"; - continue; - } - - OpFoldResult curOffset = - divOpFoldResult(offsetShift, curStride, loc, rewriter); - auto fullShape = fullShapeReCast.getConstifiedMixedSizes()[i]; - OpFoldResult curLeftSize = - maxOpFoldResult(subOpFoldResult(fullShape, curOffset, loc, rewriter), - rewriter.getIndexAttr(0), loc, rewriter); - - boundarySize[i] = - minOpFoldResult(boundarySize[i], curLeftSize, loc, rewriter); - } - - if (!isZero(curStride)) { - offsetShift = remOpFoldResult(offsetShift, curStride, loc, rewriter); - } - } - - return boundarySize; -} - -static std::optional getRootPointer(Value ptr) { - llvm::SmallPtrSet visited; - while (true) { - if (!visited.insert(ptr).second) { - return std::nullopt; - } - - if (auto blockArg = dyn_cast(ptr)) { - // A loop-carried pointer block argument is still rooted at its init - // value unless the loop updates it through an unsupported cycle. - Operation *parentOp = blockArg.getOwner()->getParentOp(); - if (auto whileOp = dyn_cast_or_null(parentOp); - whileOp && whileOp.getAfterBody() == blockArg.getOwner()) { - auto argNum = blockArg.getArgNumber(); - auto conditionArgs = whileOp.getConditionOp().getArgs(); - if (argNum < conditionArgs.size()) { - ptr = conditionArgs[argNum]; - continue; - } - } - - if (auto loopOp = dyn_cast_or_null(parentOp)) { - if (OpOperand *initArgOperand = loopOp.getTiedLoopInit(blockArg)) { - ptr = initArgOperand->get(); - continue; - } - } - - return ptr; - } - - auto *defOp = ptr.getDefiningOp(); - if (!defOp) { - return ptr; - } - - auto nextPtr = - llvm::TypeSwitch>(defOp) - .Case( - [](auto op) -> std::optional { return op.getPtr(); }) - .Case( - [](auto op) -> std::optional { return op.getSrc(); }) - .Case( - [](auto op) -> std::optional { return op.getSrc(); }) - .Case( - [](auto op) -> std::optional { return op.getSrc(); }) - .Case( - [](auto op) -> std::optional { return op.getBase(); }) - .Case( - [](auto op) -> std::optional { return op.getPtr(); }) - .Case( - [](auto op) -> std::optional { return op.getSource(); }) - .Case( - [](auto op) -> std::optional { return op.getSource(); }) - .Case( - [](auto op) -> std::optional { return op.getSource(); }) - .Case( - [](auto op) -> std::optional { - if (op.getInputs().size() != 1) { - return std::nullopt; - } - return op.getInputs().front(); - }) - .Default([](Operation *) -> std::optional { - return std::nullopt; - }); - if (!nextPtr) { - return std::nullopt; - } - ptr = *nextPtr; - } -} - -struct WritePointerInfo { - bool isWrite = false; - std::optional rootPtr; -}; - -static WritePointerInfo getKnownWritePointer(Operation *op) { - return llvm::TypeSwitch(op) - .Case([](auto storeOp) { - return WritePointerInfo{true, getRootPointer(storeOp.getPtr())}; - }) - .Case([](auto atomicOp) { - return WritePointerInfo{true, getRootPointer(atomicOp.getPtr())}; - }) - .Case([](auto atomicOp) { - return WritePointerInfo{true, getRootPointer(atomicOp.getPtr())}; - }) - .Case([](auto storeOp) { - return WritePointerInfo{true, getRootPointer(storeOp.getSrc())}; - }) - .Default([](Operation *) { return WritePointerInfo{}; }); -} - -static bool isLocalMemRef(Value value) { - while (auto *defOp = value.getDefiningOp()) { - if (isa(defOp)) { - return true; - } - auto source = - llvm::TypeSwitch(defOp) - .Case( - [](auto op) { return op.getSource(); }) - .Case([](auto op) { return op.getSource(); }) - .Case([](auto op) { return op.getSource(); }) - .Default([](Operation *) { return Value(); }); - if (!source) { - return false; - } - value = source; - } - return false; -} - -static bool mayWriteRoot(Operation *op, Value loadRootPtr) { - auto effectOp = dyn_cast(op); - if (!effectOp) { - return !mlir::isMemoryEffectFree(op); - } - - SmallVector> effects; - effectOp.getEffects(effects); - for (auto &effect : effects) { - if (!isa(effect.getEffect())) { - continue; - } - - Value value = effect.getValue(); - if (!value) { - return true; - } - if (!isa(value.getType())) { - return true; - } - - auto rootPtr = getRootPointer(value); - if (rootPtr) { - if (*rootPtr == loadRootPtr) { - return true; - } - continue; - } - if (!isLocalMemRef(value)) { - return true; - } - } - return false; -} - -// Keep an indirect load volatile unless its root pointer is proven to have no -// possible prior write. The analysis scans ordered predecessors in the same -// and enclosing blocks, nested writes in predecessor structured-control-flow -// ops, and loop-carried writes where a later store in the loop body may feed a -// following iteration. -bool requiresVolatileIndirectLoad(Value srcPtr, Operation *loadOp) { - auto loadRootPtr = getRootPointer(srcPtr); - if (!loadRootPtr) { - return true; - } - - auto opMayWriteRoot = [&](Operation *op) { - // Device-side diagnostics are modeled as GlobalMemory writes in Triton, - // but they do not write through user GM pointers and must not affect the - // indirect-load source/root analysis. - if (isa(op)) { - return false; - } - - auto memRefWriteTarget = llvm::TypeSwitch(op) - .Case([](auto copyOp) { - return copyOp.getTarget(); - }) - .Case([](auto storeOp) { - return storeOp.getMemRef(); - }) - .Default([](Operation *) { return Value(); }); - if (memRefWriteTarget) { - auto rootPtr = getRootPointer(memRefWriteTarget); - if (rootPtr) { - return *rootPtr == *loadRootPtr; - } - return !isLocalMemRef(memRefWriteTarget); - } - - auto writePtr = getKnownWritePointer(op); - if (writePtr.isWrite) { - return !writePtr.rootPtr || *writePtr.rootPtr == *loadRootPtr; - } - - // For structured control flow, look through regions instead of using the - // op-level MemoryEffect summary, otherwise unrelated nested writes would - // make this load volatile. - if (isa(op) || isa(op)) { - return false; - } - - // Unknown side-effecting ops are conservative: they may write through an - // alias that is no longer recoverable as a Triton root. - return mayWriteRoot(op, *loadRootPtr); - }; - - auto nestedMayWriteRoot = [&](Operation *container) { - bool found = false; - container->walk([&](Operation *nested) { - if (nested == container) { - return WalkResult::advance(); - } - if (opMayWriteRoot(nested)) { - found = true; - return WalkResult::interrupt(); - } - return WalkResult::advance(); - }); - return found; - }; - - Operation *current = loadOp; - - while (current) { - Block *block = current->getBlock(); - if (!block) { - return true; - } - - // First scan operations that are ordered before the current op in this - // block. When such an operation owns regions, its nested writes are also - // considered ordered before this load. - for (auto it = Block::iterator(current); it != block->begin();) { - --it; - Operation *op = &*it; - if ((op->getNumRegions() > 0 && nestedMayWriteRoot(op)) || - opMayWriteRoot(op)) { - return true; - } - } - - Operation *parentOp = block->getParentOp(); - if (!parentOp || isa(parentOp) || - isa(parentOp)) { - return false; - } - - // If the load is inside a loop, a store that is textually after the load - // may still execute before the load in a later iteration. Scan the whole - // loop body before climbing further out. - if (isa(parentOp) && nestedMayWriteRoot(parentOp)) { - return true; - } - - current = parentOp; - } - - return false; -} - -SmallVector getBroadcastDims(RankedTensorType src, - RankedTensorType dst) { - SmallVector broadcastDims; - auto srcShape = src.getShape(); - auto dstShape = dst.getShape(); - - for (size_t i = 0; i < srcShape.size(); ++i) { - if (dstShape[i] != srcShape[i]) { - assert(srcShape[i] == 1 && - "Size of source broadcast dimension must be 1"); - broadcastDims.push_back(i); - } - } - assert(!broadcastDims.empty() && "Cannot identify broadcast dimension"); - return broadcastDims; -} - -// Dimensions of collapesd tensor is all unbroadcast dims -SmallVector getUnbroadcastDims(RankedTensorType src, - RankedTensorType dst) { - SmallVector unbroadcastDims; - auto srcShape = src.getShape(); - auto dstShape = dst.getShape(); - - for (size_t i = 0; i < srcShape.size(); ++i) { - if (dstShape[i] == srcShape[i]) { - unbroadcastDims.emplace_back(srcShape[i]); - } - } - return unbroadcastDims; -} - -} // namespace ConverterUtils - -namespace triton { - -mlir::Operation * -findFirstMatchingOperandDef(mlir::Operation *rootOp, - const std::function &condFn) { - LLVM_DEBUG(llvm::dbgs() << "[findFirstMatchingOperandDef] Current op: " - << *rootOp << "\n"); - mlir::Value lhs = nullptr; - mlir::Value rhs = nullptr; - if (auto op = dyn_cast(rootOp)) { - lhs = op.getPtr(); - rhs = op.getOffset(); - } else if (auto op = dyn_cast(rootOp)) { - lhs = op.getLhs(); - rhs = op.getRhs(); - } else if (auto op = dyn_cast(rootOp)) { - lhs = op.getLhs(); - rhs = op.getRhs(); - } else if (auto op = dyn_cast(rootOp)) { - lhs = op.getLhs(); - rhs = op.getRhs(); - } else if (auto op = dyn_cast(rootOp)) { - lhs = op.getLhs(); - rhs = op.getRhs(); - } else if (auto op = dyn_cast(rootOp)) { - lhs = op.getLhs(); - rhs = op.getRhs(); - } else if (auto op = dyn_cast(rootOp)) { - lhs = op.getSrc(); - } else if (auto op = dyn_cast(rootOp)) { - } else { - rootOp->emitRemark("Backtracing encounters unsupported Operation"); - return nullptr; - } - // Backtrace operands - if (!lhs) { - return nullptr; - } - auto lhsDef = lhs.getDefiningOp(); - mlir::Operation *targetOp; - if (lhsDef) { - if (condFn(lhsDef)) { - targetOp = lhsDef; - } else { - targetOp = findFirstMatchingOperandDef(lhsDef, condFn); - } - if (targetOp) { - return targetOp; - } - } - if (!rhs) { - return nullptr; - } - auto rhsDef = rhs.getDefiningOp(); - if (rhsDef) { - if (condFn(rhsDef)) { - targetOp = rhsDef; - } else { - targetOp = findFirstMatchingOperandDef(rhsDef, condFn); - } - if (targetOp) { - return targetOp; - } - } - return nullptr; -} - -void traverseBackwardUpdateOperandChainIf( - Operation *op, std::function conditionFn, - std::function stopFn, - std::function actionFn, OpBuilder &builder, - DenseSet &handledOperation) { - - if (!op || handledOperation.contains(op)) - return; - - handledOperation.insert(op); - - if (stopFn(op)) - return; - - if (conditionFn(op)) - actionFn(builder, op); - - DenseSet handledOperand; - - std::function handler = [&](Value operand) { - if (handledOperand.contains(operand)) - return; - handledOperand.insert(operand); - if (Operation *defOp = operand.getDefiningOp()) { - traverseBackwardUpdateOperandChainIf(defOp, conditionFn, stopFn, actionFn, - builder, handledOperation); - } else { - auto blockArgument = cast(operand); - auto parentOp = blockArgument.getOwner()->getParentOp(); - if (auto whileOp = dyn_cast(parentOp); - whileOp && whileOp.getAfterBody() == blockArgument.getOwner()) { - auto argNum = blockArgument.getArgNumber(); - auto conditionArg = whileOp.getConditionOp().getArgs()[argNum]; - handler(conditionArg); - } else if (auto loopOp = dyn_cast(parentOp)) { - OpOperand *initArgOperand = loopOp.getTiedLoopInit(blockArgument); - if (!initArgOperand) - return; - Value initArg = initArgOperand->get(); - handler(initArg); - Value yieldedValue = - loopOp.getTiedLoopYieldedValue(blockArgument)->get(); - if (yieldedValue != blockArgument) - handler(yieldedValue); - } - } - }; - - for (Value operand : op->getOperands()) { - handler(operand); - } - - if (auto loopOp = dyn_cast(op)) { - for (auto yieldedValue : loopOp.getYieldedValues()) - handler(yieldedValue); - } -} - -// Note: rootOp will also be processed. -void traverseBackwardUpdateOperandChainIf( - Operation *rootOp, std::function conditionFn, - std::function stopFn, - std::function actionFn) { - - OpBuilder builder(rootOp->getContext()); - DenseSet handledOperation; - - traverseBackwardUpdateOperandChainIf(rootOp, conditionFn, stopFn, actionFn, - builder, handledOperation); -} - -void traverseForwardUpdateUserChainIf( - Operation *op, std::function conditionFn, - std::function stopFn, - std::function actionFn, OpBuilder &builder, - llvm::SmallPtrSet &stopOps) { - - if (!op) { - return; - } - - if (stopFn(op)) { - stopOps.insert(op); - return; - } - - if (conditionFn(op)) { - actionFn(builder, op); - } - - for (auto res : op->getResults()) { - for (auto userOp : res.getUsers()) { - traverseForwardUpdateUserChainIf(userOp, conditionFn, stopFn, actionFn, - builder, stopOps); - } - } -} - -// Note: rootOp will also be processed. -void traverseForwardUpdateUserChainIf( - Operation *rootOp, std::function conditionFn, - std::function stopFn, - std::function actionFn, - llvm::SmallPtrSet &stopOps) { - - OpBuilder builder(rootOp->getContext()); - - traverseForwardUpdateUserChainIf(rootOp, conditionFn, stopFn, actionFn, - builder, stopOps); -} - -bool isMetaUse(Operation *op) { return op->hasAttr("MetaUse"); } - -bool isMixUse(Operation *op) { return op->hasAttr("MixUse"); } - -IndirectLoadInterfaceOpType getIndirectLoadInterfaceOpType(Operation *op) { - auto ty = IndirectLoadInterfaceOpType::Undefined; - if (isMetaUse(op)) { - if (isa(op)) { - ty = IndirectLoadInterfaceOpType::Load; - } else if (isa(op)) { - ty = IndirectLoadInterfaceOpType::Calc; - } - } - return ty; -} - -bool opIsIndirectLoad(Operation *op) { - auto opType = getIndirectLoadInterfaceOpType(op); - return opType == IndirectLoadInterfaceOpType::Load; -} - -bool opIsIndirectCalc(Operation *op) { - auto opType = getIndirectLoadInterfaceOpType(op); - return opType == IndirectLoadInterfaceOpType::Calc; -} - -scf::ForOp createNestedLoops( - OpBuilder &builder, Location loc, unsigned currentDim, unsigned totalDims, - ValueRange LBs, ValueRange UBs, ValueRange steps, SmallVector &ivs, - ValueRange initArgs, - function_ref &, ValueRange)> - bodyBuilder) { - - if (currentDim >= totalDims) { - bodyBuilder(builder, loc, ivs, initArgs); - return nullptr; - } - - auto loop = builder.create( - loc, LBs[currentDim], UBs[currentDim], steps[currentDim], initArgs, - [&](OpBuilder &nestedBuilder, Location nestedLoc, Value iv, - ValueRange iterArgs) { - ivs.push_back(iv); - auto innerLoop = createNestedLoops(nestedBuilder, nestedLoc, - currentDim + 1, totalDims, LBs, UBs, - steps, ivs, iterArgs, bodyBuilder); - if (innerLoop) { - nestedBuilder.create(loc, innerLoop.getResults()); - } - }); - - return loop; -} - -ModuleOp getModuleOpFromOperation(Operation *op) { - Operation *parent = op; - while (parent != nullptr && !isa(parent)) { - parent = parent->getParentOp(); // 向上查找 - } - return cast(parent); // 如果没找到会抛出异常 -} - -bool isTensorPtrType(Type type) { - auto ptrType = dyn_cast(type); - if (!ptrType) - return false; - return isa(ptrType.getPointeeType()); -} - -} // namespace triton - -bool haveSameTypes(TypeRange lhs, TypeRange rhs) { - return llvm::equal(lhs, rhs); -} - -void addReduceWithIndexAttr(ReduceWithIndexParams params, - ConversionPatternRewriter &rewriter, - linalg::ReduceOp reduceOp) { - const StringRef reduceRef = "reduce_mode"; - const StringRef tieBreakLeftRef = "tie_break_left"; - const StringRef unsignedSrcRef = "unsigned_src"; - - const StringRef tieBreakStr = - params.tieBreakType == TieBreakType::LEFT ? "true" : "false"; - const StringRef withIndexStr = - params.withIndexType == ReduceWithIndexType::MAX ? "max_with_index" - : "min_with_index"; - const StringRef unsignedSrcStr = params.isUnsignedSrc ? "true" : "false"; - - reduceOp->setAttr(reduceRef, rewriter.getStringAttr(withIndexStr)); - reduceOp->setAttr(tieBreakLeftRef, rewriter.getStringAttr(tieBreakStr)); - reduceOp->setAttr(unsignedSrcRef, rewriter.getStringAttr(unsignedSrcStr)); -} - -llvm::FailureOr -getReduceWithIndexParams(triton::ReduceOp op) { - auto tritonReduceBlock = op.getBody(); - auto *tritonYield = tritonReduceBlock->getTerminator(); - auto yieldValues = tritonYield->getOperands(); - constexpr int yieldValuesNum = 2; - if (yieldValues.empty()) { - return llvm::failure(); - } - if (yieldValues.size() != yieldValuesNum) { - return ReduceWithIndexParams{}; - } - - // Unify signed/unsigned and int/float predicate - enum class Predicate { Undefined = 0, lt = 1, gt = 2, eq = 3 }; - enum class Signedness { NotApplicable = 0, Signed = 1, Unsigned = 2 }; - auto unifyPredicateI = - [](arith::CmpIPredicate p) -> std::pair { - switch (p) { - case arith::CmpIPredicate::slt: - return {Predicate::lt, Signedness::Signed}; - case arith::CmpIPredicate::ult: - return {Predicate::lt, Signedness::Unsigned}; - case arith::CmpIPredicate::sgt: - return {Predicate::gt, Signedness::Signed}; - case arith::CmpIPredicate::ugt: - return {Predicate::gt, Signedness::Unsigned}; - case arith::CmpIPredicate::eq: - return {Predicate::eq, Signedness::NotApplicable}; - default: - return {Predicate::Undefined, Signedness::NotApplicable}; - } - }; - auto unifyPredicateF = - [](arith::CmpFPredicate p) -> std::pair { - switch (p) { - case arith::CmpFPredicate::OLT: - return {Predicate::lt, Signedness::Signed}; - case arith::CmpFPredicate::ULT: - return {Predicate::lt, Signedness::Unsigned}; - case arith::CmpFPredicate::OGT: - return {Predicate::gt, Signedness::Signed}; - case arith::CmpFPredicate::UGT: - return {Predicate::gt, Signedness::Unsigned}; - case arith::CmpFPredicate::OEQ: - return {Predicate::eq, Signedness::Signed}; - case arith::CmpFPredicate::UEQ: - return {Predicate::eq, Signedness::Unsigned}; - default: - return {Predicate::Undefined, Signedness::NotApplicable}; - } - }; - - // Composite predicate to pick index of min (or max) element have to be - // written in following form: (v means value and i means index) - // For leftmost element: - // (new_v == old_v and new_i < old_i) or new_v < old_v - // new_v < old_v or (new_v == old_v and new_i < old_i) - // new_v < old_v // python3.11 ttir - // (new_v == old_v and new_i < old_i) or new_v > old_v - // new_v > old_v or (new_v == old_v and new_i < old_i) - // new_v > old_v // python3.11 ttir - // For rightmost element: - // (new_v == old_v and new_i > old_i) or new_v < old_v - // new_v < old_v or (new_v == old_v and new_i > old_i) - // (new_v == old_v and new_i > old_i) or new_v > old_v - // new_v > old_v or (new_v == old_v and new_i > old_i) - - std::map, std::pair> - m{ - // leftmost - {{Predicate::eq, Predicate::lt, Predicate::lt}, - {ReduceWithIndexType::MIN, TieBreakType::LEFT}}, - {{Predicate::lt, Predicate::eq, Predicate::lt}, - {ReduceWithIndexType::MIN, TieBreakType::LEFT}}, - {{Predicate::lt}, {ReduceWithIndexType::MIN, TieBreakType::LEFT}}, - {{Predicate::eq, Predicate::lt, Predicate::gt}, - {ReduceWithIndexType::MAX, TieBreakType::LEFT}}, - {{Predicate::gt, Predicate::eq, Predicate::lt}, - {ReduceWithIndexType::MAX, TieBreakType::LEFT}}, - {{Predicate::gt}, {ReduceWithIndexType::MAX, TieBreakType::LEFT}}, - // rightmost - {{Predicate::eq, Predicate::gt, Predicate::lt}, - {ReduceWithIndexType::MIN, TieBreakType::RIGHT}}, - {{Predicate::lt, Predicate::eq, Predicate::gt}, - {ReduceWithIndexType::MIN, TieBreakType::RIGHT}}, - {{Predicate::eq, Predicate::gt, Predicate::gt}, - {ReduceWithIndexType::MAX, TieBreakType::RIGHT}}, - {{Predicate::gt, Predicate::eq, Predicate::gt}, - {ReduceWithIndexType::MAX, TieBreakType::RIGHT}}, - }; - - std::vector preds; - std::vector signednesses; - // A better way is to trace the arith.select - // Checking the operations one by one is hacky :( - for (auto &op : tritonReduceBlock->without_terminator()) { - Predicate pred = Predicate::Undefined; - Signedness signedness = Signedness::NotApplicable; - if (auto cmpiOp = dyn_cast(op)) { - auto predi = cmpiOp.getPredicate(); - std::tie(pred, signedness) = unifyPredicateI(predi); - } - if (auto cmpfOp = dyn_cast(op)) { - auto predf = cmpfOp.getPredicate(); - std::tie(pred, signedness) = unifyPredicateF(predf); - } - if (pred != Predicate::Undefined) { - preds.push_back(pred); - signednesses.push_back(signedness); - } - } - - // check if sequence of predicates matches any sequence for min/max - // leftmost/rightmost - if (m.find(preds) == m.end()) { - return llvm::failure(); - } - - assert(!signednesses.empty()); - const bool isUnsignedSrc = - signednesses[0] == Signedness::Unsigned || - signednesses[signednesses.size() - 1] == Signedness::Unsigned; - return ReduceWithIndexParams{.withIndexType = m.at(preds).first, - .tieBreakType = m.at(preds).second, - .isUnsignedSrc = isUnsignedSrc}; -} - -// Specialize the Typeless Value (Zero, Min, Max) into a mlir TypedAttr -FailureOr specializeTypelessValueToAttr(TypelessValue value, - Type type, OpBuilder &b) { - mlir::Type f16Ty = Float16Type::get(b.getContext()); - mlir::Type bf16Ty = BFloat16Type::get(b.getContext()); - mlir::Type f32Ty = Float32Type::get(b.getContext()); - mlir::Type i8TySL = IntegerType::get( - b.getContext(), 8, IntegerType::SignednessSemantics::Signless); - mlir::Type i8TyS = IntegerType::get(b.getContext(), 8, - IntegerType::SignednessSemantics::Signed); - mlir::Type i8TyU = IntegerType::get( - b.getContext(), 8, IntegerType::SignednessSemantics::Unsigned); - mlir::Type i16TySL = IntegerType::get( - b.getContext(), 16, IntegerType::SignednessSemantics::Signless); - mlir::Type i16TyS = IntegerType::get( - b.getContext(), 16, IntegerType::SignednessSemantics::Signed); - mlir::Type i16TyU = IntegerType::get( - b.getContext(), 16, IntegerType::SignednessSemantics::Unsigned); - mlir::Type i32TySL = IntegerType::get( - b.getContext(), 32, IntegerType::SignednessSemantics::Signless); - mlir::Type i32TyS = IntegerType::get( - b.getContext(), 32, IntegerType::SignednessSemantics::Signed); - mlir::Type i32TyU = IntegerType::get( - b.getContext(), 32, IntegerType::SignednessSemantics::Unsigned); - mlir::Type i64TySL = IntegerType::get( - b.getContext(), 64, IntegerType::SignednessSemantics::Signless); - mlir::Type i64TyS = IntegerType::get( - b.getContext(), 64, IntegerType::SignednessSemantics::Signed); - mlir::Type i64TyU = IntegerType::get( - b.getContext(), 64, IntegerType::SignednessSemantics::Unsigned); - llvm::APFloat halfZero = llvm::APFloat::getZero(llvm::APFloat::IEEEhalf()); - llvm::APFloat halfOne(llvm::APFloat::IEEEhalf(), 1); - llvm::APFloat halfMax = llvm::APFloat::getInf(llvm::APFloat::IEEEhalf()); - llvm::APFloat halfMin = - llvm::APFloat::getInf(llvm::APFloat::IEEEhalf(), true); - llvm::APFloat bfloatZero = llvm::APFloat::getZero(llvm::APFloat::BFloat()); - llvm::APFloat bfloatOne(llvm::APFloat::BFloat(), 1); - llvm::APFloat bfloatMax = llvm::APFloat::getInf(llvm::APFloat::BFloat()); - llvm::APFloat bfloatMin = - llvm::APFloat::getInf(llvm::APFloat::BFloat(), true); - llvm::APFloat floatZero = llvm::APFloat::getZero(llvm::APFloat::IEEEsingle()); - llvm::APFloat floatOne(llvm::APFloat::IEEEsingle(), 1); - llvm::APFloat floatMax = llvm::APFloat::getInf(llvm::APFloat::IEEEsingle()); - llvm::APFloat floatMin = - llvm::APFloat::getInf(llvm::APFloat::IEEEsingle(), true); - auto toPtr = [](mlir::Type ty) { return ty.getAsOpaquePointer(); }; - - std::map, - std::variant> - initMap = { - {{TypelessValue::Zero, toPtr(f16Ty)}, halfZero}, - {{TypelessValue::Zero, toPtr(bf16Ty)}, bfloatZero}, - {{TypelessValue::Zero, toPtr(f32Ty)}, floatZero}, - {{TypelessValue::Zero, toPtr(i8TySL)}, llvm::APInt(8, 0, true)}, - {{TypelessValue::Zero, toPtr(i8TyS)}, llvm::APInt(8, 0, true)}, - {{TypelessValue::Zero, toPtr(i8TyU)}, llvm::APInt(8, 0, false)}, - {{TypelessValue::Zero, toPtr(i16TySL)}, llvm::APInt(16, 0, true)}, - {{TypelessValue::Zero, toPtr(i16TyS)}, llvm::APInt(16, 0, true)}, - {{TypelessValue::Zero, toPtr(i16TyU)}, llvm::APInt(16, 0, false)}, - {{TypelessValue::Zero, toPtr(i32TySL)}, llvm::APInt(32, 0, true)}, - {{TypelessValue::Zero, toPtr(i32TyS)}, llvm::APInt(32, 0, true)}, - {{TypelessValue::Zero, toPtr(i32TyU)}, llvm::APInt(32, 0, false)}, - {{TypelessValue::Zero, toPtr(i64TySL)}, llvm::APInt(64, 0, true)}, - {{TypelessValue::Zero, toPtr(i64TyS)}, llvm::APInt(64, 0, true)}, - {{TypelessValue::Zero, toPtr(i64TyU)}, llvm::APInt(64, 0, false)}, - {{TypelessValue::Min, toPtr(f16Ty)}, halfMin}, - {{TypelessValue::Min, toPtr(bf16Ty)}, bfloatMin}, - {{TypelessValue::Min, toPtr(f32Ty)}, floatMin}, - {{TypelessValue::Min, toPtr(i8TySL)}, llvm::APInt(8, -128, true)}, - {{TypelessValue::Min, toPtr(i8TyS)}, llvm::APInt(8, -128, true)}, - {{TypelessValue::Min, toPtr(i8TyU)}, llvm::APInt(8, 0, false)}, - {{TypelessValue::Min, toPtr(i16TySL)}, llvm::APInt(16, -32768, true)}, - {{TypelessValue::Min, toPtr(i16TyS)}, llvm::APInt(16, -32768, true)}, - {{TypelessValue::Min, toPtr(i16TyU)}, llvm::APInt(16, 0, false)}, - {{TypelessValue::Min, toPtr(i32TySL)}, - llvm::APInt(32, std::numeric_limits::min(), true)}, - {{TypelessValue::Min, toPtr(i32TyS)}, - llvm::APInt(32, std::numeric_limits::min(), true)}, - {{TypelessValue::Min, toPtr(i32TyU)}, llvm::APInt(32, 0, false)}, - {{TypelessValue::Min, toPtr(i64TySL)}, - llvm::APInt(64, std::numeric_limits::min(), true)}, - {{TypelessValue::Min, toPtr(i64TyS)}, - llvm::APInt(64, std::numeric_limits::min(), true)}, - {{TypelessValue::Min, toPtr(i64TyU)}, llvm::APInt(64, 0, false)}, - {{TypelessValue::Max, toPtr(f16Ty)}, halfMax}, - {{TypelessValue::Max, toPtr(bf16Ty)}, bfloatMax}, - {{TypelessValue::Max, toPtr(f32Ty)}, floatMax}, - {{TypelessValue::Max, toPtr(i8TySL)}, llvm::APInt(8, 127, true)}, - {{TypelessValue::Max, toPtr(i8TyS)}, llvm::APInt(8, 127, true)}, - {{TypelessValue::Max, toPtr(i8TyU)}, llvm::APInt::getAllOnes(8)}, - {{TypelessValue::Max, toPtr(i16TySL)}, llvm::APInt(16, 32767, true)}, - {{TypelessValue::Max, toPtr(i16TyS)}, llvm::APInt(16, 32767, true)}, - {{TypelessValue::Max, toPtr(i16TyU)}, llvm::APInt::getAllOnes(16)}, - {{TypelessValue::Max, toPtr(i32TySL)}, - llvm::APInt(32, std::numeric_limits::max(), true)}, - {{TypelessValue::Max, toPtr(i32TyS)}, - llvm::APInt(32, std::numeric_limits::max(), true)}, - {{TypelessValue::Max, toPtr(i32TyU)}, llvm::APInt::getAllOnes(32)}, - {{TypelessValue::Max, toPtr(i64TySL)}, - llvm::APInt(64, std::numeric_limits::max(), true)}, - {{TypelessValue::Max, toPtr(i64TyS)}, - llvm::APInt(64, std::numeric_limits::max(), true)}, - {{TypelessValue::Max, toPtr(i64TyU)}, llvm::APInt::getAllOnes(64)}, - }; - - std::pair key = - std::make_pair(value, toPtr(type)); - if (initMap.find(key) == initMap.end()) - return failure(); - if (auto intType = dyn_cast(type)) - return success( - IntegerAttr::get(intType, std::get(initMap.at(key)))); - if (isa(type)) - return success( - FloatAttr::get(f16Ty, std::get(initMap.at(key)))); - if (isa(type)) - return success( - FloatAttr::get(bf16Ty, std::get(initMap.at(key)))); - if (isa(type)) - return success( - FloatAttr::get(f32Ty, std::get(initMap.at(key)))); - return failure(); -} - -// Specialize the Typeless Value (Zero, Min, Max) into a mlir constant value -FailureOr specializeTypelessValueToConstant(TypelessValue value, - Type type, Location loc, - OpBuilder &b) { - std::function getElemType = [&](mlir::Type ty) { - if (auto ptrType = dyn_cast(getElementTypeOrSelf(ty))) - return getElemType(ptrType.getPointeeType()); - if (auto tensorType = mlir::dyn_cast(ty)) - return getElemType(tensorType.getElementType()); - return ty; - }; - - if (value == TypelessValue::Undefined) - return failure(); - if (auto tensorType = mlir::dyn_cast(type)) { - auto elemType = getElemType(tensorType); - FailureOr typedAttr = - specializeTypelessValueToAttr(value, elemType, b); - if (failed(typedAttr)) - return failure(); - auto otherTensorType = - RankedTensorType::get(tensorType.getShape(), elemType); - auto denseAttr = DenseElementsAttr::get(otherTensorType, *typedAttr); - return b.create(loc, denseAttr).getResult(); - } - if (mlir::isa(type) || mlir::isa(type)) { - FailureOr typedAttr = - specializeTypelessValueToAttr(value, type, b); - if (failed(typedAttr)) - return failure(); - return b.create(loc, *typedAttr).getResult(); - } - return failure(); -} - -bool checkStructureAnnotated(Operation *op, RewriterBase &rewriter) { - return llvm::any_of(op->getUsers(), [&rewriter](Operation *user) { - auto annotationOp = dyn_cast(user); - if (annotationOp && - annotationOp->hasAttr(ConverterUtils::continuousAttrName)) { - rewriter.eraseOp(annotationOp); - return true; - } - return false; - }); -} - -bool isDistributedTypeCustomOp(Operation *op) { - return op->hasAttr("hivm.is_distributed") && - (llvm::isa(op) || llvm::isa(op)); -} -} // namespace mlir From 66cad6240a8873b765b38e18adbd715d2736846a Mon Sep 17 00:00:00 2001 From: zhongsanming Date: Fri, 20 Nov 2026 08:43:27 +0800 Subject: [PATCH 7/7] [Ascend] Remove accidental AscendNPU-IR symlink Drop the local symlink committed with the FLIR wiring and ignore the AscendNPU-IR checkout path so it cannot be re-added. --- third_party/ascend/.gitignore | 1 + third_party/ascend/AscendNPU-IR | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 120000 third_party/ascend/AscendNPU-IR diff --git a/third_party/ascend/.gitignore b/third_party/ascend/.gitignore index a277bb8c49..835f5336c5 100644 --- a/third_party/ascend/.gitignore +++ b/third_party/ascend/.gitignore @@ -1 +1,2 @@ ascendnpu_ir/ +AscendNPU-IR diff --git a/third_party/ascend/AscendNPU-IR b/third_party/ascend/AscendNPU-IR deleted file mode 120000 index 214d7f7805..0000000000 --- a/third_party/ascend/AscendNPU-IR +++ /dev/null @@ -1 +0,0 @@ -/home/serephus/dev/work/flagos/AscendNPU-IR \ No newline at end of file