diff --git a/.github/workflows/riscv64-ci.yml b/.github/workflows/riscv64-ci.yml new file mode 100644 index 0000000000000..75fa670634bf9 --- /dev/null +++ b/.github/workflows/riscv64-ci.yml @@ -0,0 +1,121 @@ +# Note: this runner is provided externally, so we minimize its access to +# secrets. + +name: CI (riscv64) + +on: + push: + branches: [ riscv ] + + pull_request: + branches: [ riscv ] + + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +env: + LLVM_SRC: ${{ github.workspace }} + LLVM_BUILD_DIR: ${{ github.workspace }}/llvm-build + +jobs: + build: + name: Build and test + runs-on: [self-hosted, linux, RISCV64] + + # Optional: + # isolate runner environment + environment: riscv64 + + steps: + - name: Print environment + shell: bash + run: | + echo "------ uname ------" + uname -a + + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 300 + submodules: false + + - name: Show compiler versions + shell: bash + run: | + gcc --version || true + clang --version || true + cmake --version + ninja --version + python3 --version + + - name: Setup ccache + shell: bash + run: | + mkdir -p ~/.ccache + + cat >> ~/.ccache/ccache.conf <&1 \ // RUN: | FileCheck %s --check-prefix=DEFAULT // RUN: %clang -### -target arm64-apple-macosx11 -fobjc-constant-literals -c %s 2>&1 \ diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c index a9af276367817..3ad603d594dae 100644 --- a/clang/test/Driver/print-supported-extensions-riscv.c +++ b/clang/test/Driver/print-supported-extensions-riscv.c @@ -171,6 +171,8 @@ // CHECK-NEXT: xandesvpackfph 5.0 'XAndesVPackFPH' (Andes Vector Packed FP16 Extension) // CHECK-NEXT: xandesvsinth 5.0 'XAndesVSIntH' (Andes Vector Small INT Handling Extension) // CHECK-NEXT: xandesvsintload 5.0 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension) +// CHECK-NEXT: xbbmeteor 1.0 'XBBMeteor' (Buckyball Large Language Model extension) +// CHECK-NEXT: xbbpebble 1.0 'XBBPebble' (Buckyball CNN extension) // CHECK-NEXT: xcheriot 1.0 'XCheriot' (CHERIoT extension) // CHECK-NEXT: xcvalu 1.0 'XCValu' (CORE-V ALU Operations) // CHECK-NEXT: xcvbi 1.0 'XCVbi' (CORE-V Immediate Branching) @@ -179,6 +181,7 @@ // CHECK-NEXT: xcvmac 1.0 'XCVmac' (CORE-V Multiply-Accumulate) // CHECK-NEXT: xcvmem 1.0 'XCVmem' (CORE-V Post-incrementing Load & Store) // CHECK-NEXT: xcvsimd 1.0 'XCVsimd' (CORE-V SIMD ALU) +// CHECK-NEXT: xgemmini 1.0 'XGemmini' (Gemmini extension) // CHECK-NEXT: xmipscbop 1.0 'XMIPSCBOP' (MIPS Software Prefetch) // CHECK-NEXT: xmipscmov 1.0 'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov)) // CHECK-NEXT: xmipsexectl 1.0 'XMIPSEXECTL' (MIPS execution control) @@ -227,6 +230,7 @@ // CHECK-NEXT: xsfvqmaccqoq 1.0 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4)) // CHECK-NEXT: xsifivecdiscarddlone 1.0 'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction) // CHECK-NEXT: xsifivecflushdlone 1.0 'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction) +// CHECK-NEXT: xsmtime 1.0 'XSMTIME' (SpacemiT IME extension) // CHECK-NEXT: xsmtvdot 1.0 'XSMTVDot' (SpacemiT Vector Dot Product Extension) // CHECK-NEXT: xsmtvdotii 1.0 'XSMTVDotII' (SpacemiT Vector Extension for Matrix 2.0) // CHECK-NEXT: xtheadba 1.0 'XTHeadBa' (T-Head address calculation instructions) @@ -236,12 +240,45 @@ // CHECK-NEXT: xtheadcondmov 1.0 'XTHeadCondMov' (T-Head conditional move instructions) // CHECK-NEXT: xtheadfmemidx 1.0 'XTHeadFMemIdx' (T-Head FP Indexed Memory Operations) // CHECK-NEXT: xtheadmac 1.0 'XTHeadMac' (T-Head Multiply-Accumulate Instructions) +// CHECK-NEXT: xtheadmatrix 3.0 'XTHeadMatrix' (T-Head AME matrix extension) +// CHECK-NEXT: xtheadmatrixmin 0.5 'XTHeadMatrixMin' (T-Head AME minimal matrix extension) +// CHECK-NEXT: xtheadmdma 0.5 'XTHeadMDMA' (T-Head AME matrix DMA extension) // CHECK-NEXT: xtheadmemidx 1.0 'XTHeadMemIdx' (T-Head Indexed Memory Operations) // CHECK-NEXT: xtheadmempair 1.0 'XTHeadMemPair' (T-Head two-GPR Memory Operations) +// CHECK-NEXT: xtheadmew16b 0.5 'XTHeadMEW16B' (T-Head AME 16-bit element width extension) +// CHECK-NEXT: xtheadmew32b 0.5 'XTHeadMEW32B' (T-Head AME 32-bit element width extension) +// CHECK-NEXT: xtheadmew4b 0.5 'XTHeadMEW4B' (T-Head AME 4-bit element width extension) +// CHECK-NEXT: xtheadmew64b 0.5 'XTHeadMEW64B' (T-Head AME 64-bit element width extension) +// CHECK-NEXT: xtheadmew8b 0.5 'XTHeadMEW8B' (T-Head AME 8-bit element width extension) +// CHECK-NEXT: xtheadmfew 0.5 'XTHeadMFEW' (T-Head AME matrix fractional element width extension) +// CHECK-NEXT: xtheadmfic 0.5 'XTHeadMFIC' (T-Head AME matrix fixed-point conversion extension) +// CHECK-NEXT: xtheadmhp 0.5 'XTHeadMHP' (T-Head AME matrix half precision extension) +// CHECK-NEXT: xtheadmiew 0.5 'XTHeadMIEW' (T-Head AME matrix integer element width extension) +// CHECK-NEXT: xtheadmmbf16bbf16b 0.5 'XTHeadMMBF16BBF16B' (T-Head AME BF16-by-BF16 matrix multiply extension) +// CHECK-NEXT: xtheadmmbf16bf32b 0.5 'XTHeadMMBF16BF32B' (T-Head AME BF16-by-F32 matrix multiply extension) +// CHECK-NEXT: xtheadmmbf20bf32b 0.5 'XTHeadMMBF20BF32B' (T-Head AME BF20-by-F32 matrix multiply extension) +// CHECK-NEXT: xtheadmmf16bf16b 0.5 'XTHeadMMF16BF16B' (T-Head AME F16-by-BF16 matrix multiply extension) +// CHECK-NEXT: xtheadmmf16bf32b 0.5 'XTHeadMMF16BF32B' (T-Head AME F16-by-F32 matrix multiply extension) +// CHECK-NEXT: xtheadmmf32bf32b 0.5 'XTHeadMMF32BF32B' (T-Head AME F32-by-F32 matrix multiply extension) +// CHECK-NEXT: xtheadmmf32bf64b 0.5 'XTHeadMMF32BF64B' (T-Head AME F32-by-F64 matrix multiply extension) +// CHECK-NEXT: xtheadmmf4bbf16b 0.5 'XTHeadMMF4BBF16B' (T-Head AME F4-by-BF16 matrix multiply extension) +// CHECK-NEXT: xtheadmmf4bf16b 0.5 'XTHeadMMF4BF16B' (T-Head AME F4-by-F16 matrix multiply extension) +// CHECK-NEXT: xtheadmmf4bf32b 0.5 'XTHeadMMF4BF32B' (T-Head AME F4-by-F32 matrix multiply extension) +// CHECK-NEXT: xtheadmmf64bf64b 0.5 'XTHeadMMF64BF64B' (T-Head AME F64-by-F64 matrix multiply extension) +// CHECK-NEXT: xtheadmmf8bbf16b 0.5 'XTHeadMMF8BBF16B' (T-Head AME F8-by-BF16 matrix multiply extension) +// CHECK-NEXT: xtheadmmf8bf16b 0.5 'XTHeadMMF8BF16B' (T-Head AME F8-by-F16 matrix multiply extension) +// CHECK-NEXT: xtheadmmf8bf32b 0.5 'XTHeadMMF8BF32B' (T-Head AME F8-by-F32 matrix multiply extension) +// CHECK-NEXT: xtheadmmi4bi32b 0.5 'XTHeadMMI4BI32B' (T-Head AME I4-by-I32 matrix multiply extension) +// CHECK-NEXT: xtheadmmi8bi32b 0.5 'XTHeadMMI8BI32B' (T-Head AME I8-by-I32 matrix multiply extension) +// CHECK-NEXT: xtheadmmmxf4b 0.5 'XTHeadMMMXF4B' (T-Head AME MXF4 matrix multiply extension) +// CHECK-NEXT: xtheadmmmxf8b 0.5 'XTHeadMMMXF8B' (T-Head AME MXF8 matrix multiply extension) +// CHECK-NEXT: xtheadmmmxf8bmxf4b 0.5 'XTHeadMMMXF8BMXF4B' (T-Head AME MXF8-by-MXF4 matrix multiply extension) +// CHECK-NEXT: xtheadmred 0.5 'XTHeadMRED' (T-Head AME matrix reduction extension) // CHECK-NEXT: xtheadsync 1.0 'XTHeadSync' (T-Head multicore synchronization instructions) // CHECK-NEXT: xtheadvdot 1.0 'XTHeadVdot' (T-Head Vector Extensions for Dot) // CHECK-NEXT: xventanacondops 1.0 'XVentanaCondOps' (Ventana Conditional Ops) // CHECK-NEXT: xwchc 2.2 'Xwchc' (WCH/QingKe additional compressed opcodes) +// CHECK-NEXT: xxiangshaname 1.0 'XXiangShanAME' (OpenXiangShan Advanced Matrix Extension) // CHECK-EMPTY: // CHECK-NEXT: Experimental extensions // CHECK-NEXT: p 0.21 'P' ('Base P' (Packed SIMD)) @@ -249,6 +286,7 @@ // CHECK-NEXT: zibi 0.1 'Zibi' (Branch with Immediate) // CHECK-NEXT: zicfilp 1.0 'Zicfilp' (Landing pad) // CHECK-NEXT: zicfiss 1.0 'Zicfiss' (Shadow stack) +// CHECK-NEXT: ztt 0.1 'Ztt' (Attached Matrix Extension) // CHECK-NEXT: zvabd 0.7 'Zvabd' (Vector Absolute Difference) // CHECK-NEXT: zvbc32e 0.7 'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements) // CHECK-NEXT: zvdot4a8i 0.1 'Zvdot4a8i' (Vector 4-element Dot Product of packed 8-bit Integers) diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index fca28455f2117..80a73671b4c99 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -654,7 +654,7 @@ let TargetPrefix = "riscv" in { class RISCVClassifyMasked : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>], [LLVMVectorOfBitcastsToInt<0>, llvm_anyvector_ty, - LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_anyint_ty, LLVMMatchType<1>], [IntrNoMem, ImmArg>]>, RISCVVIntrinsic { let VLOperand = 3; @@ -2140,3 +2140,7 @@ include "llvm/IR/IntrinsicsRISCVXsf.td" include "llvm/IR/IntrinsicsRISCVXCV.td" include "llvm/IR/IntrinsicsRISCVXAndes.td" include "llvm/IR/IntrinsicsRISCVXMIPS.td" +include "llvm/IR/IntrinsicsRISCVExt.td" +include "llvm/IR/IntrinsicsRISCVStdExtZtt.td" +include "llvm/IR/IntrinsicsRISCVBOSCExt.td" +include "llvm/IR/IntrinsicsRISCVBuckyballExt.td" diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVBOSCExt.td b/llvm/include/llvm/IR/IntrinsicsRISCVBOSCExt.td new file mode 100644 index 0000000000000..176d12c7ceb44 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVBOSCExt.td @@ -0,0 +1,768 @@ +//===- IntrinsicsRISCVBOSCExt.td ------------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// This file defines the intrinsics for the BOSC AME extension. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// AME (RISC-V Matrix Extension) Intrinsics +//===----------------------------------------------------------------------===// + +// Matrix configuration intrinsics (with immediate) +let TargetPrefix = "riscv" in { + // Configuration Register Instructions + // msettype - Set the mtype CSR based on register value + def int_riscv_bosc_msettype : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + + // msettilem - Set tile M dimension from register value + def int_riscv_bosc_msettilem : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + + // msettilen - Set tile N dimension from register value + def int_riscv_bosc_msettilen : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + + // msettilek - Set tile K dimension from register value + def int_riscv_bosc_msettilek : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + + // msettypei - Set the mtype CSR based on immediate + def int_riscv_bosc_msettypei : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + def int_riscv_bosc_msettypehi : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // msettilemi - Set tile M dimension with immediate + def int_riscv_bosc_msettilemi : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // msettileni - Set tile N dimension with immediate + def int_riscv_bosc_msettileni : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // msettileki - Set tile K dimension with immediate + def int_riscv_bosc_msettileki : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; +} + +//===----------------------------------------------------------------------===// +// BOSC AME Load/Store Intrinsic Base Classes +//===----------------------------------------------------------------------===// +// Base class for all Matrix Load Intrinsics +// Signature: void (index: i64, base_ptr: ptr, stride: i64) +class BOSC_AME_Load_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + +// Base class for all Matrix Store Intrinsics +// Signature: void (index: i64, base_ptr: ptr, stride: i64) +class BOSC_AME_Store_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + +//===----------------------------------------------------------------------===// +// Load/Store Intrinsic Instantiations +//===----------------------------------------------------------------------===// +let TargetPrefix = "riscv" in { + //---------------------------------------------------------------------- + // 1. Matrix Load Intrinsics + //---------------------------------------------------------------------- + // Matrix A (TileReg) + def int_riscv_bosc_mlae8_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlae16_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlae32_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlae64_m : BOSC_AME_Load_Intr; + + def int_riscv_bosc_mlate8_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlate16_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlate32_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlate64_m : BOSC_AME_Load_Intr; + + def int_riscv_bosc_mltre8_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mltre16_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mltre32_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mltre64_m : BOSC_AME_Load_Intr; + + // Matrix B (TileReg) + def int_riscv_bosc_mlbe8_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlbe16_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlbe32_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlbe64_m : BOSC_AME_Load_Intr; + + def int_riscv_bosc_mlbte8_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlbte16_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlbte32_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlbte64_m : BOSC_AME_Load_Intr; + + // Matrix C (AccReg) + def int_riscv_bosc_mlce8_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlce16_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlce32_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlce64_m : BOSC_AME_Load_Intr; + + def int_riscv_bosc_mlcte8_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlcte16_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlcte32_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlcte64_m : BOSC_AME_Load_Intr; + + def int_riscv_bosc_mlacce8_m : BOSC_AME_Load_Intr; + def int_riscv_bosc_mlacce16_m: BOSC_AME_Load_Intr; + def int_riscv_bosc_mlacce32_m: BOSC_AME_Load_Intr; + def int_riscv_bosc_mlacce64_m: BOSC_AME_Load_Intr; + //---------------------------------------------------------------------- + // 2. Matrix Store Intrinsics + //---------------------------------------------------------------------- + // Matrix A (TileReg) + def int_riscv_bosc_msae8_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msae16_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msae32_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msae64_m : BOSC_AME_Store_Intr; + + def int_riscv_bosc_msate8_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msate16_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msate32_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msate64_m : BOSC_AME_Store_Intr; + + def int_riscv_bosc_mstre8_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_mstre16_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_mstre32_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_mstre64_m : BOSC_AME_Store_Intr; + + // Matrix B (TileReg) + def int_riscv_bosc_msbe8_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msbe16_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msbe32_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msbe64_m : BOSC_AME_Store_Intr; + + def int_riscv_bosc_msbte8_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msbte16_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msbte32_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msbte64_m : BOSC_AME_Store_Intr; + + // Matrix C (AccReg) + def int_riscv_bosc_msce8_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msce16_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msce32_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msce64_m : BOSC_AME_Store_Intr; + + def int_riscv_bosc_mscte8_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_mscte16_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_mscte32_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_mscte64_m : BOSC_AME_Store_Intr; + + def int_riscv_bosc_msacce8_m : BOSC_AME_Store_Intr; + def int_riscv_bosc_msacce16_m: BOSC_AME_Store_Intr; + def int_riscv_bosc_msacce32_m: BOSC_AME_Store_Intr; + def int_riscv_bosc_msacce64_m: BOSC_AME_Store_Intr; +} + +//===----------------------------------------------------------------------===// +// Data Move / Broadcast / Transpose Intrinsic Base Classes +//===----------------------------------------------------------------------===// + +// 1. Matrix to Matrix (2 Immediates: md, ms1) -> No return +class BOSC_AME_Move_T_T_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, ImmArg>]>; + +// 2. Matrix to Matrix with GPR Index (2 Immediates: md, ms1; 1 GPR: rs2) -> No return +class BOSC_AME_Move_A_T_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, ImmArg>]>; + +// 3. Matrix to Matrix with Immediate Index (3 Immediates: md, ms1, imm) -> No return +class BOSC_AME_MoveImm_A_T_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, ImmArg>, + ImmArg>]>; + +// 4. Matrix to Scalar Move (1 Immediate: ms1; 1 GPR: rs2) -> Returns GPR (i64) +class BOSC_AME_Move_X_T_Intr + : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + +// 5. Scalar to Matrix Move (1 Immediate: md; 2 GPRs: rs1, rs2) -> No return +class BOSC_AME_Move_T_X_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + +//===----------------------------------------------------------------------===// +// Intrinsic Instantiations +//===----------------------------------------------------------------------===// +let TargetPrefix = "riscv" in { + //---------------------------------------------------------------------- + // 1. Matrix-Matrix Move Pseudos (Pure Matrix) + //---------------------------------------------------------------------- + def int_riscv_bosc_mmve8_t_t : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mmve16_t_t : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mmve32_t_t : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mmve64_t_t : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mmve8_a_a : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mmve16_a_a : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mmve32_a_a : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mmve64_a_a : BOSC_AME_Move_T_T_Intr; + //---------------------------------------------------------------------- + // 2. Matrix-Matrix Move (with GPR Index) + //---------------------------------------------------------------------- + def int_riscv_bosc_mmve8_a_t : BOSC_AME_Move_A_T_Intr; + def int_riscv_bosc_mmve16_a_t : BOSC_AME_Move_A_T_Intr; + def int_riscv_bosc_mmve32_a_t : BOSC_AME_Move_A_T_Intr; + def int_riscv_bosc_mmve64_a_t : BOSC_AME_Move_A_T_Intr; + + def int_riscv_bosc_mmve8_t_a : BOSC_AME_Move_A_T_Intr; + def int_riscv_bosc_mmve16_t_a : BOSC_AME_Move_A_T_Intr; + def int_riscv_bosc_mmve32_t_a : BOSC_AME_Move_A_T_Intr; + def int_riscv_bosc_mmve64_t_a : BOSC_AME_Move_A_T_Intr; + //---------------------------------------------------------------------- + // 3. Matrix-Matrix Move (with Immediate Index) + //---------------------------------------------------------------------- + def int_riscv_bosc_mmvie8_a_t : BOSC_AME_MoveImm_A_T_Intr; + def int_riscv_bosc_mmvie16_a_t : BOSC_AME_MoveImm_A_T_Intr; + def int_riscv_bosc_mmvie32_a_t : BOSC_AME_MoveImm_A_T_Intr; + def int_riscv_bosc_mmvie64_a_t : BOSC_AME_MoveImm_A_T_Intr; + + def int_riscv_bosc_mmvie8_t_a : BOSC_AME_MoveImm_A_T_Intr; + def int_riscv_bosc_mmvie16_t_a : BOSC_AME_MoveImm_A_T_Intr; + def int_riscv_bosc_mmvie32_t_a : BOSC_AME_MoveImm_A_T_Intr; + def int_riscv_bosc_mmvie64_t_a : BOSC_AME_MoveImm_A_T_Intr; + //---------------------------------------------------------------------- + // 4. Matrix to Scalar Move (Returns GPR) + //---------------------------------------------------------------------- + def int_riscv_bosc_mmve8_x_t : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mmve16_x_t : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mmve32_x_t : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mmve64_x_t : BOSC_AME_Move_X_T_Intr; + + def int_riscv_bosc_mmve8_x_a : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mmve16_x_a : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mmve32_x_a : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mmve64_x_a : BOSC_AME_Move_X_T_Intr; + + def int_riscv_bosc_mfmve8_f_t : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mfmve16_f_t : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mfmve32_f_t : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mfmve64_f_t : BOSC_AME_Move_X_T_Intr; + + def int_riscv_bosc_mfmve8_f_a : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mfmve16_f_a : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mfmve32_f_a : BOSC_AME_Move_X_T_Intr; + def int_riscv_bosc_mfmve64_f_a : BOSC_AME_Move_X_T_Intr; + //---------------------------------------------------------------------- + // 5. Scalar GPR to Matrix Move + //---------------------------------------------------------------------- + def int_riscv_bosc_mmve8_t_x : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mmve16_t_x : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mmve32_t_x : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mmve64_t_x : BOSC_AME_Move_T_X_Intr; + + def int_riscv_bosc_mmve8_a_x : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mmve16_a_x : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mmve32_a_x : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mmve64_a_x : BOSC_AME_Move_T_X_Intr; + + def int_riscv_bosc_mfmve8_t_f : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mfmve16_t_f : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mfmve32_t_f : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mfmve64_t_f : BOSC_AME_Move_T_X_Intr; + + def int_riscv_bosc_mfmve8_a_f : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mfmve16_a_f : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mfmve32_a_f : BOSC_AME_Move_T_X_Intr; + def int_riscv_bosc_mfmve64_a_f : BOSC_AME_Move_T_X_Intr; + //---------------------------------------------------------------------- + // 6. Broadcast Intrinsics + //---------------------------------------------------------------------- + def int_riscv_bosc_mbcar_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcbr_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbccr_m : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mbcace8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcace16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcace32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcace64_m : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mbcbce8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcbce16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcbce32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcbce64_m : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mbccce8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbccce16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbccce32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbccce64_m : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mbcaee8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcaee16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcaee32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcaee64_m : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mbcbee8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcbee16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcbee32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbcbee64_m : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mbccee8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbccee16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbccee32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mbccee64_m : BOSC_AME_Move_T_T_Intr; + //---------------------------------------------------------------------- + // 7. Transpose Intrinsics + //---------------------------------------------------------------------- + def int_riscv_bosc_mtae8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtae16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtae32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtae64_m : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mtbe8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtbe16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtbe32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtbe64_m : BOSC_AME_Move_T_T_Intr; + + def int_riscv_bosc_mtce8_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtce16_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtce32_m : BOSC_AME_Move_T_T_Intr; + def int_riscv_bosc_mtce64_m : BOSC_AME_Move_T_T_Intr; +} + +//===----------------------------------------------------------------------===// +// Element-Wise Intrinsic Base Classes +//===----------------------------------------------------------------------===// +// Base class for 3-operand element-wise intrinsics (md, ms1, ms2) +// Signature: void (md_idx: i64, ms1_idx: i64, ms2_idx: i64) +class BOSC_AME_EleWise3_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, ImmArg>, + ImmArg>]>; + +// Base class for 2-operand element-wise intrinsics (md, ms1) - Used for SQRT +// Signature: void (md_idx: i64, ms1_idx: i64) +class BOSC_AME_EleWise2_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, ImmArg>]>; + +//===----------------------------------------------------------------------===// +// Element-Wise Intrinsic Instantiations +//===----------------------------------------------------------------------===// +let TargetPrefix = "riscv" in { + //===--------------------------------------------------------------------===// + // 1. ADD / SUB - No Widen + //===--------------------------------------------------------------------===// + def int_riscv_bosc_maddu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_maddu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_maddu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_maddu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_maddu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_maddu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msaddu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msaddu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msaddu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msaddu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msaddu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msaddu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_madd_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_madd_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_madd_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_madd_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_madd_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_madd_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msadd_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msadd_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msadd_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msadd_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msadd_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msadd_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfadd_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfadd_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfadd_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfadd_f_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfadd_d_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msubu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msubu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msubu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msubu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msubu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msubu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mssubu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssubu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssubu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssubu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssubu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssubu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msub_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msub_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msub_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msub_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msub_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msub_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mssub_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssub_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssub_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssub_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssub_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mssub_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfsub_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfsub_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfsub_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfsub_f_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfsub_d_mm : BOSC_AME_EleWise3_Intr; + //===--------------------------------------------------------------------===// + // 2. ADD / SUB - Double Widen + //===--------------------------------------------------------------------===// + def int_riscv_bosc_mwaddu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwaddu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwaddu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwaddu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwaddu_w_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mwadd_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwadd_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwadd_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwadd_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwadd_w_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfwadd_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwadd_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwadd_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwadd_f_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mwsubu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwsubu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwsubu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwsubu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwsubu_w_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mwsub_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwsub_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwsub_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwsub_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwsub_w_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfwsub_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwsub_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwsub_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwsub_f_mm : BOSC_AME_EleWise3_Intr; + //===--------------------------------------------------------------------===// + // 3. MIN / MAX - No Widen + //===--------------------------------------------------------------------===// + def int_riscv_bosc_mminu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mminu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mminu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mminu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mminu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mminu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mmaxu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmaxu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmaxu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmaxu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmaxu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmaxu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mmin_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmin_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmin_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmin_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmin_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmin_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mmax_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmax_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmax_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmax_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmax_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmax_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfmin_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmin_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmin_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmin_f_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmin_d_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfmax_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmax_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmax_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmax_f_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmax_d_mm : BOSC_AME_EleWise3_Intr; + //===--------------------------------------------------------------------===// + // 4. MUL / MULH / DIV - No Widen + //===--------------------------------------------------------------------===// + def int_riscv_bosc_msmulu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mmul_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmul_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmul_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmul_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmul_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmul_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msmul_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmul_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmul_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmul_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmul_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmul_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfmul_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmul_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmul_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmul_f_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfmul_d_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mmulhu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mmulh_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulh_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulh_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulh_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulh_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulh_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mmulhsu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhsu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhsu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhsu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhsu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mmulhsu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msmulsu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulsu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulsu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulsu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulsu_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msmulsu_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfdiv_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfdiv_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfdiv_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfdiv_f_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfdiv_d_mm : BOSC_AME_EleWise3_Intr; + //===--------------------------------------------------------------------===// + // 5. MUL - Double Widen + //===--------------------------------------------------------------------===// + def int_riscv_bosc_mwmulu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmulu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmulu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmulu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmulu_w_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mwmul_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmul_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmul_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmul_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmul_w_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mwmulsu_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmulsu_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmulsu_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmulsu_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mwmulsu_w_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_mfwmul_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwmul_cf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwmul_hf_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mfwmul_f_mm : BOSC_AME_EleWise3_Intr; + //===--------------------------------------------------------------------===// + // 6. LOGIC & SHIFT - No Widen + //===--------------------------------------------------------------------===// + def int_riscv_bosc_mand_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mor_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_mxor_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msll_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msll_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msll_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msll_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msll_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msll_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msrl_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msrl_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msrl_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msrl_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msrl_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msrl_dw_mm : BOSC_AME_EleWise3_Intr; + + def int_riscv_bosc_msra_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msra_hb_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msra_b_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msra_h_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msra_w_mm : BOSC_AME_EleWise3_Intr; + def int_riscv_bosc_msra_dw_mm : BOSC_AME_EleWise3_Intr; + //===--------------------------------------------------------------------===// + // 7. SQRT - No Widen (Unary, 2-Operand) + // NOTE: Uses EleWise2_Intr + //===--------------------------------------------------------------------===// + def int_riscv_bosc_mfsqrt_mm : BOSC_AME_EleWise2_Intr; + def int_riscv_bosc_mfsqrt_cf_mm : BOSC_AME_EleWise2_Intr; + def int_riscv_bosc_mfsqrt_hf_mm : BOSC_AME_EleWise2_Intr; + def int_riscv_bosc_mfsqrt_f_mm : BOSC_AME_EleWise2_Intr; + def int_riscv_bosc_mfsqrt_d_mm : BOSC_AME_EleWise2_Intr; +} + +//===----------------------------------------------------------------------===// +// Matrix Multiply-Accumulate (MMA) Intrinsic Base Class +//===----------------------------------------------------------------------===// +// Base class for all 3-operand Matrix Multiply-Accumulate intrinsics +// Signature: void (md_idx: i64, ms1_idx: i64, ms2_idx: i64) +class BOSC_AME_MMA_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, ImmArg>, + ImmArg>]>; + +//===----------------------------------------------------------------------===// +// MMA Intrinsic Instantiations +//===----------------------------------------------------------------------===// +let TargetPrefix = "riscv" in { + //===--------------------------------------------------------------------===// + // 1. No-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // int + def int_riscv_bosc_mma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mma_h_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mma_w_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mma_dw_mm : BOSC_AME_MMA_Intr; + + // int, saturated + def int_riscv_bosc_msma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msma_h_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msma_w_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msma_dw_mm : BOSC_AME_MMA_Intr; + + // uint + def int_riscv_bosc_mmau_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mmau_h_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mmau_w_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mmau_dw_mm : BOSC_AME_MMA_Intr; + + // uint, saturated + def int_riscv_bosc_msmau_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msmau_h_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msmau_w_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msmau_dw_mm : BOSC_AME_MMA_Intr; + + // float point + def int_riscv_bosc_mfma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mfma_hf_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mfma_f_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mfma_d_mm : BOSC_AME_MMA_Intr; + //===--------------------------------------------------------------------===// + // 2. Double-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def int_riscv_bosc_mwmau_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mwmau_h_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mwmau_w_mm : BOSC_AME_MMA_Intr; + + // uint, saturated + def int_riscv_bosc_mswmau_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mswmau_h_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mswmau_w_mm : BOSC_AME_MMA_Intr; + + // int + def int_riscv_bosc_mwma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mwma_h_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mwma_w_mm : BOSC_AME_MMA_Intr; + + // int, saturated + def int_riscv_bosc_mswma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mswma_h_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mswma_w_mm : BOSC_AME_MMA_Intr; + + // float point + def int_riscv_bosc_mfwma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mfwma_cf_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mfwma_hf_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mfwma_f_mm : BOSC_AME_MMA_Intr; + //===--------------------------------------------------------------------===// + // 3. Quad-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def int_riscv_bosc_mqmau_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mqmau_b_mm : BOSC_AME_MMA_Intr; + + // uint, saturated + def int_riscv_bosc_msqmau_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msqmau_b_mm : BOSC_AME_MMA_Intr; + + // int + def int_riscv_bosc_mqma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mqma_b_mm : BOSC_AME_MMA_Intr; + + // int, saturated + def int_riscv_bosc_msqma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msqma_b_mm : BOSC_AME_MMA_Intr; + + // float point + def int_riscv_bosc_mfqma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_mfqma_cf_mm : BOSC_AME_MMA_Intr; + //===--------------------------------------------------------------------===// + // 4. Oct-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def int_riscv_bosc_momau_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_momau_hb_mm : BOSC_AME_MMA_Intr; + + // uint, saturated + def int_riscv_bosc_msomau_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msomau_hb_mm : BOSC_AME_MMA_Intr; + + // int + def int_riscv_bosc_moma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_moma_hb_mm : BOSC_AME_MMA_Intr; + + // int, saturated + def int_riscv_bosc_msoma_mm : BOSC_AME_MMA_Intr; + def int_riscv_bosc_msoma_hb_mm : BOSC_AME_MMA_Intr; +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVBuckyballExt.td b/llvm/include/llvm/IR/IntrinsicsRISCVBuckyballExt.td new file mode 100644 index 0000000000000..bd5faf6f6c808 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVBuckyballExt.td @@ -0,0 +1,25 @@ +//===- IntrinsicsRISCVBuckyballExt.td -------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "riscv" in { + def int_riscv_bb_mvin : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + def int_riscv_bb_mvout : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + def int_riscv_bb_mset : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + def int_riscv_bb_fence : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; +} + +include "llvm/IR/IntrinsicsRISCVXBBPebble.td" +include "llvm/IR/IntrinsicsRISCVXBBMeteor.td" diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVBuddyBOSCExt.td b/llvm/include/llvm/IR/IntrinsicsRISCVBuddyBOSCExt.td new file mode 100644 index 0000000000000..a902bae9a65c0 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVBuddyBOSCExt.td @@ -0,0 +1,81 @@ +//===- IntrinsicsRISCVBuddyBOSCExt.td -------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// This file defines the intrinsics for the BOSC AME extension. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// AME (RISC-V Matrix Extension) Intrinsics +//===----------------------------------------------------------------------===// + +// Matrix configuration intrinsics (with immediate) +let TargetPrefix = "riscv" in { + // msettilemi - Set tile M dimension with immediate + def int_riscv_buddy_bosc_msettilemi : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // msettileni - Set tile N dimension with immediate + def int_riscv_buddy_bosc_msettileni : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // msettileki - Set tile K dimension with immediate + def int_riscv_buddy_bosc_msettileki : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Matrix load intrinsics (load to tile register) +// Format: md = tile register index, base = address, stride = row byte stride +let TargetPrefix = "riscv" in { + // mlae32.m - Load 32-bit left matrix A to tile register + def int_riscv_buddy_bosc_mlae32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // mlbe32.m - Load 32-bit right matrix B to tile register + def int_riscv_buddy_bosc_mlbe32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Matrix store intrinsics (store from tile register) +let TargetPrefix = "riscv" in { + // msce32.m - Store 32-bit output matrix C from accumulator + def int_riscv_buddy_bosc_msce32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Matrix zero intrinsic +let TargetPrefix = "riscv" in { + def int_riscv_buddy_bosc_mzero : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Tile register matrix multiplication intrinsics (operate on tile registers) +let TargetPrefix = "riscv" in { + // mma.w.mm - int32 tile matrix multiply: md = md + ms1 x ms2 + def int_riscv_buddy_bosc_mma_w_mm : Intrinsic<[],[llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVBuddyExt.td b/llvm/include/llvm/IR/IntrinsicsRISCVBuddyExt.td new file mode 100644 index 0000000000000..d48e5af9644e1 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVBuddyExt.td @@ -0,0 +1,677 @@ +//===- IntrinsicsRISCVBuddyExt.td -----------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// This is the costum intrinsic definition file of RISC-V buddy extension. +// +//===----------------------------------------------------------------------===// +let TargetPrefix = "riscv" in +def int_riscv_mvin : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty],[]>; + +let TargetPrefix = "riscv" in +def int_riscv_mvin2 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty],[]>; + +let TargetPrefix = "riscv" in +def int_riscv_mvin3 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty],[]>; + +let TargetPrefix = "riscv" in +def int_riscv_mvout : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_flush : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_config : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_preload : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_compute_preloaded : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_compute_accumulated : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_bounds : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_addrs_ab : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_addrs_dc : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_strides_ab : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_strides_dc : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config1 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config2 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config3 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config4 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config5 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config6 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +//===----------------------------------------------------------------------===// +// IME Extension Intrinsics +//===----------------------------------------------------------------------===// + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotu : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotsu : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotus : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadot : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot1 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot1u : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot1su : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot1us : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot2 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot2u : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot2su : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot2us : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot3 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot3u : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot3su : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot3us : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotn : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotnu : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotnsu : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotnus : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadot1 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadot2 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadot3 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadotn : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +//===----------------------------------------------------------------------===// +// AME (RISC-V Matrix Extension) Intrinsics +//===----------------------------------------------------------------------===// + +// Matrix configuration intrinsics (with register) +let TargetPrefix = "riscv" in { + // msettype - Set matrix type configuration + def int_riscv_buddy_msettype : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>; + + // msettilem - Set tile M dimension from register + def int_riscv_buddy_msettilem : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>; + + // msettilen - Set tile N dimension from register + def int_riscv_buddy_msettilen : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>; + + // msettilek - Set tile K dimension from register + def int_riscv_buddy_msettilek : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>; +} + +// Matrix configuration intrinsics (with immediate) +let TargetPrefix = "riscv" in { + // msettilemi - Set tile M dimension with immediate + def int_riscv_buddy_msettilemi : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>]>; + + // msettileni - Set tile N dimension with immediate + def int_riscv_buddy_msettileni : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>]>; + + // msettileki - Set tile K dimension with immediate + def int_riscv_buddy_msettileki : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>]>; +} + +// Matrix load intrinsics (load to tile register) +// Format: md = tile register index, base = address, stride = row byte stride +let TargetPrefix = "riscv" in { + // mlae32.m - Load 32-bit left matrix A to tile register + def int_riscv_buddy_mlae32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlae64.m - Load 64-bit left matrix A to tile register + def int_riscv_buddy_mlae64_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlbe32.m - Load 32-bit right matrix B to tile register + def int_riscv_buddy_mlbe32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlbe64.m - Load 64-bit right matrix B to tile register + def int_riscv_buddy_mlbe64_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlce32.m - Load 32-bit output matrix C to accumulator + def int_riscv_buddy_mlce32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlce64.m - Load 64-bit output matrix C to accumulator + def int_riscv_buddy_mlce64_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; +} + +// Matrix store intrinsics (store from tile register) +let TargetPrefix = "riscv" in { + // msce32.m - Store 32-bit output matrix C from accumulator + def int_riscv_buddy_msce32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrWriteMem, IntrHasSideEffects, ImmArg>]>; + + // msce64.m - Store 64-bit output matrix C from accumulator + def int_riscv_buddy_msce64_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrWriteMem, IntrHasSideEffects, ImmArg>]>; +} + +// Matrix zero intrinsic +let TargetPrefix = "riscv" in { + def int_riscv_buddy_mzero : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>]>; +} + +// Tile register matrix multiplication intrinsics (operate on tile registers) +let TargetPrefix = "riscv" in { + // mma.w.mm - int32 tile matrix multiply: md = md + ms1 x ms2 + def int_riscv_buddy_mma_w_mm_tile : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // mma.dw.mm - int64 tile matrix multiply: md = md + ms1 x ms2 + def int_riscv_buddy_mma_dw_mm_tile : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; +} + +// Legacy matrix load/store intrinsics (for backward compatibility) +let TargetPrefix = "riscv" in { + // mlae - Load matrix A with element width + def int_riscv_buddy_mlae : Intrinsic<[llvm_anyvector_ty], + [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrReadMem]>; + + // mlbe - Load matrix B with element width + def int_riscv_buddy_mlbe : Intrinsic<[llvm_anyvector_ty], + [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrReadMem]>; + + // mlce - Load matrix C (accumulator) + def int_riscv_buddy_mlce : Intrinsic<[llvm_anyvector_ty], + [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrReadMem]>; + + // msce - Store matrix C (accumulator) + def int_riscv_buddy_msce : Intrinsic<[], + [llvm_anyvector_ty, llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrWriteMem]>; +} + +// Signed integer matrix multiplication intrinsics +let TargetPrefix = "riscv" in { + // mqma.b.mm - int8 quad-widen matrix multiply (int8 x int8 -> int32) + def int_riscv_buddy_mqma_b_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mma.h.mm - int16 matrix multiply + def int_riscv_buddy_mma_h_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mma.w.mm - int32 matrix multiply + def int_riscv_buddy_mma_w_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mma.dw.mm - int64 matrix multiply + def int_riscv_buddy_mma_dw_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mwma.h.mm - int16 double-widen matrix multiply (int16 x int16 -> int32) + def int_riscv_buddy_mwma_h_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; +} + +// Unsigned integer matrix multiplication intrinsics +let TargetPrefix = "riscv" in { + // mqmau.b.mm - uint8 quad-widen matrix multiply + def int_riscv_buddy_mqmau_b_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mmau.h.mm - uint16 matrix multiply + def int_riscv_buddy_mmau_h_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; +} + +// Floating-point matrix multiplication intrinsics +let TargetPrefix = "riscv" in { + // mfma.f.mm - fp32 matrix multiply + def int_riscv_buddy_mfma_f_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mfma.hf.mm - fp16 matrix multiply + def int_riscv_buddy_mfma_hf_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mfwma.hf.mm - fp16 double-widen matrix multiply (fp16 x fp16 -> fp32) + def int_riscv_buddy_mfwma_hf_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; +} + +//===----------------------------------------------------------------------===// +// XuanTie AME (xuantie Matrix Extension 0.5.0) Intrinsics +//===----------------------------------------------------------------------===// + +// Matrix configuration intrinsics (with register rs1) +let TargetPrefix = "riscv" in { + // th.mcfg - Set entire tile from register + def int_riscv_buddy_th_mcfg : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem]>; + + // th.mcfgm - Set tile M dimension from register + def int_riscv_buddy_th_mcfgm : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem]>; + + // th.mcfgn - Set tile N dimension from register + def int_riscv_buddy_th_mcfgn : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem]>; + + // th.mcfgk - Set tile K dimension from register + def int_riscv_buddy_th_mcfgk : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrNoMem]>; +} + +// Matrix configuration intrinsics (with immediate) +let TargetPrefix = "riscv" in { + // th.mcfgmi - Set tile M dimension with immediate + def int_riscv_buddy_th_mcfgmi : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mcfgni - Set tile N dimension with immediate + def int_riscv_buddy_th_mcfgni : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mcfgki - Set tile K dimension with immediate + def int_riscv_buddy_th_mcfgki : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Matrix zero intrinsic +let TargetPrefix = "riscv" in { + // zero 1 matrix register (all elements in the tile register are set to zero) + def int_riscv_buddy_th_mzero : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // zero 2 matrix registers (all elements in the two tile registers are set to zero) + def int_riscv_buddy_th_mzero2r : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // zero 4 matrix registers (all elements in the four tile registers are set to zero) + def int_riscv_buddy_th_mzero4r : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // zero 8 matrix registers (all elements in the eight tile registers are set to zero) + def int_riscv_buddy_th_mzero8r : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Matrix load intrinsics (load to tile register) +// Format: md = tile register index, base = address, stride = row byte stride +let TargetPrefix = "riscv" in { + // th.mlde8 - Load 8-bit matrix to tile register + def int_riscv_buddy_th_mlde8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mlde16 - Load 16-bit matrix to tile register + def int_riscv_buddy_th_mlde16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mlde32 - Load 32-bit matrix to tile register + def int_riscv_buddy_th_mlde32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mlde64 - Load 64-bit matrix to tile register + def int_riscv_buddy_th_mlde64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // Matrix prefetch load intrinsics (load from memory to cache) + def int_riscv_buddy_th_mplde8 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + def int_riscv_buddy_th_mplde16 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + def int_riscv_buddy_th_mplde32 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + def int_riscv_buddy_th_mplde64 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + // Transposed Load Instructions + // th.mldte8 - Load 8-bit transposed matrix to tile register + def int_riscv_buddy_th_mldte8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mldte16 - Load 16-bit transposed matrix to tile register + def int_riscv_buddy_th_mldte16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mldte32 - Load 32-bit transposed matrix to tile register + def int_riscv_buddy_th_mldte32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mldte64 - Load 64-bit transposed matrix to tile register + def int_riscv_buddy_th_mldte64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Matrix store intrinsics (store from tile register) +let TargetPrefix = "riscv" in { + // th.mste8 - Store 8-bit matrix from tile register + def int_riscv_buddy_th_mste8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mste16 - Store 16-bit matrix from tile register + def int_riscv_buddy_th_mste16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mste32 - Store 32-bit matrix from tile register + def int_riscv_buddy_th_mste32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mste64 - Store 64-bit matrix from tile register + def int_riscv_buddy_th_mste64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Tile register matrix multiplication intrinsics (operate on tile registers) +// integer matrix multiplication and add, md = md + ms1 * ms2. +let TargetPrefix = "riscv" in { + // th.mmacc.w.b - signed 8bit, output quad-widen + def int_riscv_buddy_th_mmacc_w_b : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mmaccu.w.b - unsigned 8bit, output quad-widen + def int_riscv_buddy_th_mmaccu_w_b : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mmaccus.w.b - unsigned-signed 8bit, output quad-widen + def int_riscv_buddy_th_mmaccus_w_b : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mmaccsu.w.b - signed-unsigned 8bit, output quad-widen + def int_riscv_buddy_th_mmaccsu_w_b : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; +} + +// Tile register Float point matrix multiplication and add, md = md + ms1 * ms2. +let TargetPrefix = "riscv" in { + // th.mfmacc.h - 16-bit float point(fp16) + def int_riscv_buddy_th_mfmacc_h : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.bf16 - 16-bit float point(bf16) + def int_riscv_buddy_th_mfmacc_bf16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s - 32-bit float point + def int_riscv_buddy_th_mfmacc_s : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.d - 64-bit float point + def int_riscv_buddy_th_mfmacc_d : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.h.e4m3 - 8-bit float point, output double-widen + def int_riscv_buddy_th_mfmacc_h_e4m3 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.h.e5m2 - 8-bit float point, output double-widen + def int_riscv_buddy_th_mfmacc_h_e5m2 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.bf16.e4m3 - 8-bit float point, output double-widen + def int_riscv_buddy_th_mfmacc_bf16_e4m3 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc_bf16.e5m2 - 8-bit float point, output double-widen + def int_riscv_buddy_th_mfmacc_bf16_e5m2 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s.h - 16-bit float point, output double-widen + def int_riscv_buddy_th_mfmacc_s_h : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s.bf16 - 16-bit float point, output double-widen + def int_riscv_buddy_th_mfmacc_s_bf16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.d.s - 32-bit float point, output double-widen + def int_riscv_buddy_th_mfmacc_d_s : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s.e4m3 8-bit float point, output quad-widen + def int_riscv_buddy_th_mfmacc_s_e4m3 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s.e5m2 8-bit float point, output quad-widen + def int_riscv_buddy_th_mfmacc_s_e5m2 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVExt.td b/llvm/include/llvm/IR/IntrinsicsRISCVExt.td new file mode 100644 index 0000000000000..cd2a7c6b02c29 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVExt.td @@ -0,0 +1,875 @@ +//===------ IntrinsicsRISCVExt.td -----------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// This is the costum intrinsic definition file of RISC-V buddy extension. +// +//===----------------------------------------------------------------------===// +let TargetPrefix = "riscv" in +def int_riscv_mvin : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty],[]>; + +let TargetPrefix = "riscv" in +def int_riscv_mvin2 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty],[]>; + +let TargetPrefix = "riscv" in +def int_riscv_mvin3 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty],[]>; + +let TargetPrefix = "riscv" in +def int_riscv_mvout : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_flush : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_config : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_preload : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_compute_preloaded : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_compute_accumulated : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_bounds : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_addrs_ab : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_addrs_dc : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_strides_ab : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws_config_strides_dc : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_ws : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config1 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config2 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config3 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config4 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config5 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_loop_conv_ws_config6 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +//===----------------------------------------------------------------------===// +// IME Extension Intrinsics +//===----------------------------------------------------------------------===// + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotu : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotsu : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotus : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadot : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot1 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot1u : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot1su : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot1us : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot2 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot2u : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot2su : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot2us : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot3 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot3u : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot3su : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadot3us : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotn : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotnu : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotnsu : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vmadotnus : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadot1 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadot2 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadot3 : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty], + [IntrNoMem]>; + +let TargetPrefix = "riscv" in +def int_riscv_ime_vfmadotn : Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_anyvector_ty, llvm_i64_ty], + [IntrNoMem]>; + +//===----------------------------------------------------------------------===// +// AME (RISC-V Matrix Extension) Intrinsics +//===----------------------------------------------------------------------===// + +// Matrix configuration intrinsics (with register) +let TargetPrefix = "riscv" in { + // msettype - Set matrix type configuration + def int_riscv_buddy_msettype : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>; + + // msettilem - Set tile M dimension from register + def int_riscv_buddy_msettilem : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>; + + // msettilen - Set tile N dimension from register + def int_riscv_buddy_msettilen : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>; + + // msettilek - Set tile K dimension from register + def int_riscv_buddy_msettilek : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>; +} + +// Matrix configuration intrinsics (with immediate) +let TargetPrefix = "riscv" in { + // msettilemi - Set tile M dimension with immediate + def int_riscv_buddy_msettilemi : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>]>; + + // msettileni - Set tile N dimension with immediate + def int_riscv_buddy_msettileni : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>]>; + + // msettileki - Set tile K dimension with immediate + def int_riscv_buddy_msettileki : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>]>; +} + +// Matrix load intrinsics (load to tile register) +// Format: md = tile register index, base = address, stride = row byte stride +let TargetPrefix = "riscv" in { + // mlae32.m - Load 32-bit left matrix A to tile register + def int_riscv_buddy_mlae32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlae64.m - Load 64-bit left matrix A to tile register + def int_riscv_buddy_mlae64_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlbe32.m - Load 32-bit right matrix B to tile register + def int_riscv_buddy_mlbe32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlbe64.m - Load 64-bit right matrix B to tile register + def int_riscv_buddy_mlbe64_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlce32.m - Load 32-bit output matrix C to accumulator + def int_riscv_buddy_mlce32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + + // mlce64.m - Load 64-bit output matrix C to accumulator + def int_riscv_buddy_mlce64_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; +} + +// Matrix store intrinsics (store from tile register) +let TargetPrefix = "riscv" in { + // msce32.m - Store 32-bit output matrix C from accumulator + def int_riscv_buddy_msce32_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrWriteMem, IntrHasSideEffects, ImmArg>]>; + + // msce64.m - Store 64-bit output matrix C from accumulator + def int_riscv_buddy_msce64_m : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty], + [IntrWriteMem, IntrHasSideEffects, ImmArg>]>; +} + +// Matrix zero intrinsic +let TargetPrefix = "riscv" in { + def int_riscv_buddy_mzero : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, ImmArg>]>; +} + +// Tile register matrix multiplication intrinsics (operate on tile registers) +let TargetPrefix = "riscv" in { + // mma.w.mm - int32 tile matrix multiply: md = md + ms1 x ms2 + def int_riscv_buddy_mma_w_mm_tile : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // mma.dw.mm - int64 tile matrix multiply: md = md + ms1 x ms2 + def int_riscv_buddy_mma_dw_mm_tile : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; +} + +// Legacy matrix load/store intrinsics (for backward compatibility) +let TargetPrefix = "riscv" in { + // mlae - Load matrix A with element width + def int_riscv_buddy_mlae : Intrinsic<[llvm_anyvector_ty], + [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrReadMem]>; + + // mlbe - Load matrix B with element width + def int_riscv_buddy_mlbe : Intrinsic<[llvm_anyvector_ty], + [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrReadMem]>; + + // mlce - Load matrix C (accumulator) + def int_riscv_buddy_mlce : Intrinsic<[llvm_anyvector_ty], + [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrReadMem]>; + + // msce - Store matrix C (accumulator) + def int_riscv_buddy_msce : Intrinsic<[], + [llvm_anyvector_ty, llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrWriteMem]>; +} + +// Signed integer matrix multiplication intrinsics +let TargetPrefix = "riscv" in { + // mqma.b.mm - int8 quad-widen matrix multiply (int8 x int8 -> int32) + def int_riscv_buddy_mqma_b_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mma.h.mm - int16 matrix multiply + def int_riscv_buddy_mma_h_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mma.w.mm - int32 matrix multiply + def int_riscv_buddy_mma_w_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mma.dw.mm - int64 matrix multiply + def int_riscv_buddy_mma_dw_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mwma.h.mm - int16 double-widen matrix multiply (int16 x int16 -> int32) + def int_riscv_buddy_mwma_h_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; +} + +// Unsigned integer matrix multiplication intrinsics +let TargetPrefix = "riscv" in { + // mqmau.b.mm - uint8 quad-widen matrix multiply + def int_riscv_buddy_mqmau_b_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mmau.h.mm - uint16 matrix multiply + def int_riscv_buddy_mmau_h_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; +} + +// Floating-point matrix multiplication intrinsics +let TargetPrefix = "riscv" in { + // mfma.f.mm - fp32 matrix multiply + def int_riscv_buddy_mfma_f_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mfma.hf.mm - fp16 matrix multiply + def int_riscv_buddy_mfma_hf_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; + + // mfwma.hf.mm - fp16 double-widen matrix multiply (fp16 x fp16 -> fp32) + def int_riscv_buddy_mfwma_hf_mm : Intrinsic<[], + [llvm_ptr_ty, llvm_ptr_ty, llvm_ptr_ty, + llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + []>; +} + +//===----------------------------------------------------------------------===// +// XuanTie AME (xuantie Matrix Extension 0.5.0) Intrinsics +//===----------------------------------------------------------------------===// + +// Matrix configuration intrinsics (with register rs1) +let TargetPrefix = "riscv" in { + // th.mcfg - Set entire tile from register + def int_riscv_th_mcfg : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + + // th.mcfgm - Set tile M dimension from register + def int_riscv_th_mcfgm : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + + // th.mcfgn - Set tile N dimension from register + def int_riscv_th_mcfgn : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; + + // th.mcfgk - Set tile K dimension from register + def int_riscv_th_mcfgk : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects]>; +} + +// Matrix configuration intrinsics (with immediate) +let TargetPrefix = "riscv" in { + // th.mcfgmi - Set tile M dimension with immediate + def int_riscv_th_mcfgmi : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mcfgni - Set tile N dimension with immediate + def int_riscv_th_mcfgni : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mcfgki - Set tile K dimension with immediate + def int_riscv_th_mcfgki : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Matrix zero intrinsic +let TargetPrefix = "riscv" in { + // zero 1 matrix register (all elements in the tile register are set to zero) + def int_riscv_th_mzero : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // zero 2 matrix registers (all elements in the two tile registers are set to zero) + def int_riscv_th_mzero2r : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // zero 4 matrix registers (all elements in the four tile registers are set to zero) + def int_riscv_th_mzero4r : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // zero 8 matrix registers (all elements in the eight tile registers are set to zero) + def int_riscv_th_mzero8r : Intrinsic<[], [llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; +} + +// MISC Instructions +let TargetPrefix = "riscv" in { + // Data Move Instructions between Matrix Registers + def int_riscv_th_mmov_mm : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, ImmArg>]>; + + // Data Move Instructions between Integer and Matrix (Duplicate) + def int_riscv_th_mdupb_m_x : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mduph_m_x : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mdupw_m_x : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mdupd_m_x : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // Data Move Instructions between Integer and Matrix (Scalar to Matrix) + def int_riscv_th_mmovb_m_x : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mmovh_m_x : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mmovw_m_x : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mmovd_m_x : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // Data Move Instructions between Integer and Matrix (Matrix to Scalar) + def int_riscv_th_mmovb_x_m : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mmovh_x_m : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mmovw_x_m : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mmovd_x_m : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>]>; + + // Data Broadcast Instructions + def int_riscv_th_mmov_mv_i : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, + ImmArg>, + ImmArg>]>; + def int_riscv_th_mcmovb_mv_i : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, + ImmArg>, + ImmArg>]>; + def int_riscv_th_mcmovh_mv_i : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, + ImmArg>, + ImmArg>]>; + def int_riscv_th_mcmovw_mv_i : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, + ImmArg>, + ImmArg>]>; + def int_riscv_th_mcmovd_mv_i : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, + ImmArg>, + ImmArg>]>; + + // Matrix Pack Instructions + def int_riscv_th_mpack_mm : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, + ImmArg>, + ImmArg>]>; + def int_riscv_th_mpackhl_mm : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, + ImmArg>, + ImmArg>]>; + def int_riscv_th_mpackhh_mm : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, IntrHasSideEffects, + ImmArg>, + ImmArg>, + ImmArg>]>; +} + +// Matrix load intrinsics (load to tile register) +// Format: md = tile register index, base = address, stride = row byte stride +let TargetPrefix = "riscv" in { + // th.mlde8 - Load 8-bit matrix to tile register + def int_riscv_th_mlde8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mlde16 - Load 16-bit matrix to tile register + def int_riscv_th_mlde16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mlde32 - Load 32-bit matrix to tile register + def int_riscv_th_mlde32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mlde64 - Load 64-bit matrix to tile register + def int_riscv_th_mlde64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // Matrix prefetch load intrinsics (load from memory to cache) + def int_riscv_th_mplde8 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + def int_riscv_th_mplde16 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + def int_riscv_th_mplde32 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + def int_riscv_th_mplde64 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + // transposed prefetch load instructions (load from memory to cache) + def int_riscv_th_mpldte8 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + def int_riscv_th_mpldte16 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + def int_riscv_th_mpldte32 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + def int_riscv_th_mpldte64 : Intrinsic<[], + [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects]>; + + // Transposed Load Instructions + // th.mldte8 - Load 8-bit transposed matrix to tile register + def int_riscv_th_mldte8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mldte16 - Load 16-bit transposed matrix to tile register + def int_riscv_th_mldte16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mldte32 - Load 32-bit transposed matrix to tile register + def int_riscv_th_mldte32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mldte64 - Load 64-bit transposed matrix to tile register + def int_riscv_th_mldte64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // Stream Load Instructions + def int_riscv_th_mslde8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mslde16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mslde32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mslde64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + + // Transposed Stream Load Instructions + def int_riscv_th_msldte8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msldte16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msldte32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msldte64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Matrix store intrinsics (store from tile register) +let TargetPrefix = "riscv" in { + // th.mste8 - Store 8-bit matrix from tile register + def int_riscv_th_mste8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mste16 - Store 16-bit matrix from tile register + def int_riscv_th_mste16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mste32 - Store 32-bit matrix from tile register + def int_riscv_th_mste32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // th.mste64 - Store 64-bit matrix from tile register + def int_riscv_th_mste64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // Transposed Store Instructions + def int_riscv_th_mstte8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mstte16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mstte32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_mstte64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // Stream Store Instructions + def int_riscv_th_msste8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msste16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msste32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msste64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + + // Transposed Stream Store Instructions + def int_riscv_th_msstte8 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msstte16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msstte32 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; + def int_riscv_th_msstte64 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, + ImmArg>]>; +} + +// Tile register matrix multiplication intrinsics (operate on tile registers) +// integer matrix multiplication and add, md = md + ms1 * ms2. +let TargetPrefix = "riscv" in { + // th.mmacc.w.b - signed 8bit, output quad-widen + def int_riscv_th_mmacc_w_b : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mmaccu.w.b - unsigned 8bit, output quad-widen + def int_riscv_th_mmaccu_w_b : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mmaccus.w.b - unsigned-signed 8bit, output quad-widen + def int_riscv_th_mmaccus_w_b : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mmaccsu.w.b - signed-unsigned 8bit, output quad-widen + def int_riscv_th_mmaccsu_w_b : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; +} + +// Tile register Float point matrix multiplication and add, md = md + ms1 * ms2. +let TargetPrefix = "riscv" in { + // th.mfmacc.h - 16-bit float point(fp16) + def int_riscv_th_mfmacc_h : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.bf16 - 16-bit float point(bf16) + def int_riscv_th_mfmacc_bf16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s - 32-bit float point + def int_riscv_th_mfmacc_s : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.d - 64-bit float point + def int_riscv_th_mfmacc_d : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.h.e4m3 - 8-bit float point, output double-widen + def int_riscv_th_mfmacc_h_e4m3 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.h.e5m2 - 8-bit float point, output double-widen + def int_riscv_th_mfmacc_h_e5m2 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.bf16.e4m3 - 8-bit float point, output double-widen + def int_riscv_th_mfmacc_bf16_e4m3 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.bf16.e5m2 - 8-bit float point, output double-widen + def int_riscv_th_mfmacc_bf16_e5m2 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s.h - 16-bit float point, output double-widen + def int_riscv_th_mfmacc_s_h : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s.bf16 - 16-bit float point, output double-widen + def int_riscv_th_mfmacc_s_bf16 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.d.s - 32-bit float point, output double-widen + def int_riscv_th_mfmacc_d_s : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s.e4m3 8-bit float point, output quad-widen + def int_riscv_th_mfmacc_s_e4m3 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; + + // th.mfmacc.s.e5m2 8-bit float point, output quad-widen + def int_riscv_th_mfmacc_s_e5m2 : Intrinsic<[], + [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, ImmArg>, + ImmArg>, ImmArg>]>; +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVStdExtZtt.td b/llvm/include/llvm/IR/IntrinsicsRISCVStdExtZtt.td new file mode 100644 index 0000000000000..3484f552dbb50 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVStdExtZtt.td @@ -0,0 +1,574 @@ +//===- IntrinsicsRISCVZTTExt.td ------------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// This file defines the intrinsics for Ztt (RISC-V AME Matrix Extension) +// extension. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Datatype management intrinsics +//===----------------------------------------------------------------------===// +// Base class for "get datatype" instrinsics: reads the dtype of an M/Acc +// register into a GPR value. +// Signature: i32 (dtype_reg_index: i32) +class ZTT_DataType_Get_Intr + : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>]>; + +// Base class for "set datatype" instrinsics: writes a GPR value into the dtype +// of an M/Acc register and zeroes that M/Acc register's contents. +// Signature: void (dtype_reg_index: i32, dtype_value: i32) +class ZTT_DataType_Set_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>]>; + +let TargetPrefix = "riscv" in { + // agettyp - get the dtype of an Acc register + def int_riscv_ztt_agettyp : ZTT_DataType_Get_Intr; + + // asettyp - set the dtype of an Acc register; zeroes the Acc register + def int_riscv_ztt_asettyp : ZTT_DataType_Set_Intr; + + // mgettyp - get the dtype of a M (matrix) register + def int_riscv_ztt_mgettyp : ZTT_DataType_Get_Intr; + + // sgettyp - set the dtype of a M (matrix) register; zeroes the M register + def int_riscv_ztt_msettyp : ZTT_DataType_Set_Intr; +} + +//===----------------------------------------------------------------------===// +// Elementwise arithmetic intrinsics +//===----------------------------------------------------------------------===// +// Base class for matrix-matrix elementwise arithmetic intrinsics +// Signature: void (md_reg_index: i32, ms1_reg_index: i32, ms2_reg_index: i32) +class ZTT_Elementwise_Arithmetic_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>, + ImmArg> + ]>; + +// Base class for matrix-scalar elementwise arithmetic intrinsics +// Signature: void (md_reg_index: i32, scalar: i32, ms2_reg_index: i32) +class ZTT_Elementwise_Arithmetic_Scalar_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>]>; + +let TargetPrefix = "riscv" in { + // mabs.ew - md[i] = |ms1[i]| + // Signature: void (md_reg_index: i32, ms1_reg_index: i32) + def int_riscv_ztt_mabs_ew : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg> + ]>; + + // mabsdiff.ew - md[i] = |ms1[i] - ms2[i]| + def int_riscv_ztt_mabsdiff_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mabsdiff.ew.x - md[i] = |scalar - ms2[i]| + def int_riscv_ztt_mabsdiff_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // madd.ew - md[i] = ms1[i] + ms2[i] + def int_riscv_ztt_madd_ew : ZTT_Elementwise_Arithmetic_Intr; + + // madd.ew.x - md[i] = scalar + ms2[i] + def int_riscv_ztt_madd_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // msub.ew - md[i] = ms2[i] - ms1[i] + def int_riscv_ztt_msub_ew : ZTT_Elementwise_Arithmetic_Intr; + + // msub.ew.x - md[i] = scalar - ms2[i] + def int_riscv_ztt_msub_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mhdiff.ew - md[i] = (ms2[i] - ms1[i]) * 0.5 + def int_riscv_ztt_mhdiff_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mhdiff.ew.x - md[i] = (scalar - ms2[i]) * 0.5 + def int_riscv_ztt_mhdiff_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmax.ew - md[i] = max(ms1[i], ms2[i]) + def int_riscv_ztt_mmax_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmax.ew.x - md[i] = max(scalar, ms2[i]) + def int_riscv_ztt_mmax_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmean.ew - md[i] = (ms1[i] + ms2[i]) * 0.5 + def int_riscv_ztt_mmean_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmean.ew.x - md[i] = (scalar + ms2[i]) * 0.5 + def int_riscv_ztt_mmean_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmin.ew - md[i] = min(ms1[i], ms2[i]) + def int_riscv_ztt_mmin_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmin.ew.x - md[i] = min(scalar, ms2[i]) + def int_riscv_ztt_mmin_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmul.ew - md[i] = ms1[i] * ms2[i] + def int_riscv_ztt_mmul_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmul.ew.x - md[i] = scalar * ms2[i] + def int_riscv_ztt_mmul_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmulacc.ew - md[i] = md[i] + ms1[i] * ms2[i] + def int_riscv_ztt_mmulacc_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmulacc.ew.x - md[i] = md[i] + scalar * ms2[i] + def int_riscv_ztt_mmulacc_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmulaccneg.ew - md[i] = md[i] - ms1[i] * ms2[i] + def int_riscv_ztt_mmulaccneg_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmulaccneg.ew.x - md[i] = md[i] - scalar * ms2[i] + def int_riscv_ztt_mmulaccneg_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmuladd.ew - md[i] = ms1[i] + ms2[i] * md[i] + def int_riscv_ztt_mmuladd_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmuladd.ew.x - md[i] = scalar + ms2[i] * md[i] + def int_riscv_ztt_mmuladd_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmulneg.ew - md[i] = -(ms1[i] * ms2[i]) + def int_riscv_ztt_mmulneg_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmulneg.ew.x - md[i] = -(scalar * ms2[i]) + def int_riscv_ztt_mmulneg_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; + + // mmulsub.ew - md[i] = ms1[i] - ms2[i] * md[i] + def int_riscv_ztt_mmulsub_ew : ZTT_Elementwise_Arithmetic_Intr; + + // mmulsub.ew.x - md[i] = scalar - ms2[i] * md[i] + def int_riscv_ztt_mmulsub_ew_x : ZTT_Elementwise_Arithmetic_Scalar_Intr; +} + +//===----------------------------------------------------------------------===// +// Bitwise intrinsics +//===----------------------------------------------------------------------===// +// Base class for matrix-matrix bitwise ops. +// Signature: void (md_reg_index: i32, ms1_reg_index: i32, ms2_reg_index: i32) +class ZTT_Bitwise_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>, + ImmArg> + ]>; + +// Base class for matrix-scalar bitwise ops. +// Signature: void (md_reg_index: i32, scalar: i32, ms2_reg_index: i32) +class ZTT_Bitwise_Scalar_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>]>; + +let TargetPrefix = "riscv" in { + // mand.ew - md[i] = ms1[i] & ms2[i] + def int_riscv_ztt_mand_ew : ZTT_Bitwise_Intr; + + // mand.ew.x - md[i] = scalar & ms2[i] + def int_riscv_ztt_mand_ew_x : ZTT_Bitwise_Scalar_Intr; + + // mandnot.ew - md[i] = ms1[i] & ~ms2[i] + def int_riscv_ztt_mandnot_ew : ZTT_Bitwise_Intr; + + // mandnot.ew.x - md[i] = scalar & ~ms2[i] + def int_riscv_ztt_mandnot_ew_x : ZTT_Bitwise_Scalar_Intr; + + // mor.ew - md[i] = ms1[i] | ms2[i] + def int_riscv_ztt_mor_ew : ZTT_Bitwise_Intr; + + // mor.ew.x - md[i] = scalar | ms2[i] + def int_riscv_ztt_mor_ew_x : ZTT_Bitwise_Scalar_Intr; + + // mornot.ew - md[i] = ms1[i] | ~ms2[i] + def int_riscv_ztt_mornot_ew : ZTT_Bitwise_Intr; + + // mornot.ew.x - md[i] = scalar | ~ms2[i] + def int_riscv_ztt_mornot_ew_x : ZTT_Bitwise_Scalar_Intr; + + // mxor.ew - md[i] = ms1[i] ^ ms2[i] + def int_riscv_ztt_mxor_ew : ZTT_Bitwise_Intr; + + // mxor.ew.x - md[i] = scalar ^ ms2[i] + def int_riscv_ztt_mxor_ew_x : ZTT_Bitwise_Scalar_Intr; +} + +//===----------------------------------------------------------------------===// +// Scalar broadcast intrinsic +//===----------------------------------------------------------------------===// +let TargetPrefix = "riscv" in { + // mbcast.x - broadcast a GPR value to every element of a matrix register + // (md). Signature: void (md_reg_index: i32, scalar: i32) + def int_riscv_ztt_mbcast_x + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>]>; +} + +//===----------------------------------------------------------------------===// +// Compare and predication intrinsics +//===----------------------------------------------------------------------===// +// Base class for matrix-matrix compare/predicate ops. +// Signature: void (md_reg_index: i32, pred_or_src0_reg_index: i32, +// src1_reg_index: i32) +class ZTT_Comp_Pred_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>, + ImmArg> + ]>; + +// Base class for matrix-scalar compare ops. +// Signature: void (md_reg_index: i32, scalar: i32, src1_reg_index: i32) +class ZTT_Comp_Pred_Scalar_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>]>; + +let TargetPrefix = "riscv" in { + // mcmovge.ew - md[i] = src1[i] if pred[i] >= 0, else md[i] unchanged. + def int_riscv_ztt_mcmovge_ew : ZTT_Comp_Pred_Intr; + + // mcmovlt.ew - md[i] = src1[i] if pred[i] < 0, else md[i] unchanged. + def int_riscv_ztt_mcmovlt_ew : ZTT_Comp_Pred_Intr; + + // mcmpge.ew - md[i] = (src0[i] >= src1[i]) ? -1 : 0. + def int_riscv_ztt_mcmpge_ew : ZTT_Comp_Pred_Intr; + + // mcmpge.ew.x - md[i] = (scalar >= src1[i]) ? -1 : 0. + def int_riscv_ztt_mcmpge_ew_x : ZTT_Comp_Pred_Scalar_Intr; + + // mcmplt.ew - md[i] = (src0[i] < src1[i]) ? -1 : 0. + def int_riscv_ztt_mcmplt_ew : ZTT_Comp_Pred_Intr; + + // mcmplt.ew.x - md[i] = (scalar < src1[i]) ? -1 : 0. + def int_riscv_ztt_mcmplt_ew_x : ZTT_Comp_Pred_Scalar_Intr; + + // mselge.ew - md[i] = src1[i] if pred[i] >= 0, else md[i] = 0 + def int_riscv_ztt_mselge_ew : ZTT_Comp_Pred_Intr; + + // msellt.ew - md[i] = src1[i] if pred[i] < 0, else md[i] = 0 + def int_riscv_ztt_msellt_ew : ZTT_Comp_Pred_Intr; +} + +//===----------------------------------------------------------------------===// +// Permutation intrinsics +//===----------------------------------------------------------------------===// +// Base class for 2-operand permutation intrinsics. +// Signature: void (md_reg_index: i32, ms1_reg_index: i32) +class ZTT_Perm2_Intr : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, + IntrNoMem, + ImmArg>, + ImmArg>, +]>; + +// Base class for 3-operand permutation intrinsics. +// Signature: void (md_reg_index: i32, ms1_reg_index: i32, ms2_reg_index: i32) +class ZTT_Perm3_Intr : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>, + ImmArg> +]>; + +let TargetPrefix = "riscv" in { + // Note: N = sqrt(ZTT_NELEM) + + // mcolunzip.ew - elementwise column un-zip (de-interleave). + // Given 0 <= i < N and 0 <= j < N/2 + // md[i, j] = ms1[i, 2j] + // md[i, j + N/2] = ms1[i, 2j + 1] + def int_riscv_ztt_mcolunzip_ew : ZTT_Perm2_Intr; + + // mcolzip.ew - elementwise column zip (interleave). + // Given 0 <= i < N and 0 <= j < N/2 + // md[i, 2j] = ms1[i, j] + // md[i, 2j + 1] = ms1[i, j + N/2] + def int_riscv_ztt_mcolzip_ew : ZTT_Perm2_Intr; + + // mgather.ew - elementwise gather + // Given 0 <= i < N and 0 <= j < N + // md[i, j] = ms1[i, ms2[i, j]] + def int_riscv_ztt_mgather_ew : ZTT_Perm3_Intr; + + // mrowunzip.ew - row un-zip (de-interleave rows) + // Given 0 <= k < N, 0 <= j < N and that ms1 is treated as a conceptual 2N-row + // matrix stored across (ms1, ms+1). + // md[k, j] = ms1[2k, j] + // md+1[k, j] = ms1[2k+1, j] + def int_riscv_ztt_mrowunzip_ew : ZTT_Perm2_Intr; + + // mrowzip.ew - row zip (interleave rows from 2 inputs) + // Given 0 <= k < 2N-1, 0 <= j < N and that md is treated as a conceptual + // 2N-row matrix stord across (md, md+1). md[2k, j] = ms1[k, j] md[2k+1, j] = + // ms2[k, j] + def int_riscv_ztt_mrowzip_ew : ZTT_Perm3_Intr; + + // mscatadd.col - elementwise column scatter-add + // Given i <= 0 < N, 0 <= j < N + // md[ms2[i, j], j] += ms1[i, j] + def int_riscv_ztt_mscatadd_col : ZTT_Perm3_Intr; + + // mscatadd.row - elementwise scatter-add + // Given i <= 0 < N, 0 <= j < N + // md[i, ms2[i, j]] += ms1[i, j] + def int_riscv_ztt_mscatadd_row : ZTT_Perm3_Intr; + + // mscatmax.col - elementwise column scatter-max + // Given i <= 0 < N, 0 <= j < N + // md[ms2[i, j], j] = max(md[ms2[i, j], j], ms1[i, j]) + def int_riscv_ztt_mscatmax_col : ZTT_Perm3_Intr; + + // mscatmax.row - elementwise scatter-max + // Given i <= 0 < N, 0 <= j < N + // md[i, ms2[i, j]] = max(md[i, ms2[i, j]], ms1[i, j]) + def int_riscv_ztt_mscatmax_row : ZTT_Perm3_Intr; + + // mshift.ew - elementwise row shift by constant offset + // Given i <= 0 < N, 0 <= j < N + // md[i, j] = md[i, j + imm] if 0 <= j + imm < N else md[i, j] = 0 + def int_riscv_ztt_mshift_ew : ZTT_Perm3_Intr; + + // mshift.m1 - alias for mshift.ew when imm = -1 + def int_riscv_ztt_mshift_m1 : ZTT_Perm2_Intr; + + // mshift.p1 - alias for mshift.ew when imm = 1 + def int_riscv_ztt_mshift_p1 : ZTT_Perm2_Intr; +} + +//===----------------------------------------------------------------------===// +// Register move / data conversion intrinsics +//===----------------------------------------------------------------------===// +// Base class for packing / unpacking intrinsics. +// Signature: void (md_reg_index: i32, ms1_reg_index: i32, scalar: i32) +class ZTT_PackUnpack_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, + IntrNoMem, + ImmArg>, + ImmArg>, + ]>; + +let TargetPrefix = "riscv" in { + // mconv.ew - convert between packed and non-packed dtype (Md[ms1] vs + // Md[md]); exactly one side must be packed, else amestatus.UN. + // NOTE: pseudocode in spec assumes non-packed side is unit-sized (1 + // reg/square); unclear if "wide" non-packed dtypes (>1 reg/square) are valid + // operands here. + def int_riscv_ztt_mconv_ew : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg> + ]>; + + // mpack.ew.x - pack the square from the ms1 register (unpacked, + // single-square) into the packed md (matrix) register's sub-square slot + // selected by scalar index + // NOTE: pseudocode in spec assumes non-packed side is unit-sized (1 + // reg/square); unclear if "wide" non-packed dtypes (>1 reg/square) are valid + // operands here. + def int_riscv_ztt_mpack_ew_x : ZTT_PackUnpack_Intr; + + // munpack.ew.x - extracts a square (selected by scalar index) from the ms1 + // register (packed) into the unpacked md (matrix) register + // NOTE: pseudocode in spec assumes non-packed side is unit-sized (1 + // reg/square); unclear if "wide" non-packed dtypes (>1 reg/square) are valid + // operands here. + def int_riscv_ztt_munpack_ew_x : ZTT_PackUnpack_Intr; +} + +//===----------------------------------------------------------------------===// +// Log2 and Exp2 intrinsics +//===----------------------------------------------------------------------===// +// Base class for 2-operand elementwise matrix-matrix log2/exp2 intrinsics. +// Signature: void (md_reg_index: i32, ms1_reg_index: i32) +class ZTT_Log2Exp2_2_Intr : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, + IntrNoMem, + ImmArg>, + ImmArg>, +]>; + +// Base class for 3-operand elementwise matrix-matrix log2/exp2 intrinsics. +// Signature: void (md_reg_index: i32, ms1_reg_index: i32, ms2_reg_index: i32) +class ZTT_Log2Exp2_3_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>, + ImmArg> + ]>; + +// Base class for elementwise matrix-scalar log2/exp2 intrinsics. +// Signature: void (md_reg_index: i32, scalar: i32, ms2_reg_index: i32) +class ZTT_Log2Exp2_Scalar_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>]>; + +let TargetPrefix = "riscv" in { + // mexp2.ew - md[i] = 2^ms1[i] + def int_riscv_ztt_mexp2_ew : ZTT_Log2Exp2_2_Intr; + + // mldexp.ew - md[i] = ms1[i] * 2^ms1[i] + def int_riscv_ztt_mldexp_ew : ZTT_Log2Exp2_3_Intr; + + // mldexp.ew.x - md[i] = ms2[i] * 2^scalar + def int_riscv_ztt_mldexp_ew_x : ZTT_Log2Exp2_Scalar_Intr; + + // mldexpacc.ew - md[i] = md[i] + ms1[i] * 2^ms1[i] + def int_riscv_ztt_mldexpacc_ew : ZTT_Log2Exp2_3_Intr; + + // mldexpacc.ew.x - md[i] = md[i] + ms2[i] * 2^scalar + def int_riscv_ztt_mldexpacc_ew_x : ZTT_Log2Exp2_Scalar_Intr; + + // mlog2.ew - md[i] = log2(ms1[i]) + def int_riscv_ztt_mlog2_ew : ZTT_Log2Exp2_2_Intr; + + // mlog2sub.ew - md[i] = log2(|ms1[i]|) - ms2[i] + def int_riscv_ztt_mlog2sub_ew : ZTT_Log2Exp2_3_Intr; + + // mlog2sub.ew.x - md[i] = log2(|ms2[i]|) - scalar + def int_riscv_ztt_mlog2sub_ew_x : ZTT_Log2Exp2_Scalar_Intr; + + // mrdexp.ew - md[i] = ms1[i] * 2^(-ms2[i]) + def int_riscv_ztt_mrdexp_ew : ZTT_Log2Exp2_3_Intr; + + // mrdexpacc.ew - md[i] = md[i] + ms1[i] * 2^(-ms2[i]) + def int_riscv_ztt_mrdexpacc_ew : ZTT_Log2Exp2_3_Intr; + + // msublog2.ew - md[i] = ms2[i] - log2(|ms1[i]|) + def int_riscv_ztt_msublog2_ew : ZTT_Log2Exp2_3_Intr; + + // msublog2.ew.x - md[i] = scalar - log2(|ms2[i]|) + def int_riscv_ztt_msublog2_ew_x : ZTT_Log2Exp2_Scalar_Intr; +} + +//===----------------------------------------------------------------------===// +// Load/store (memory) intrinsics +//===----------------------------------------------------------------------===// +// Base class for square loads: loads ZTT_NELEM elements from memory at addr +// into the matrix register group (md). +// Signature: void (md_reg_index: i32, addr: i32) +class ZTT_Load_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], + [IntrReadMem, IntrHasSideEffects, ImmArg>]>; + +// Base class for square stores: stores ZTT_NELEM elements from the matrix +// register group (md) into memory at addr. +// Signature: void (md_reg_index: i32, addr: i32) +class ZTT_Store_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty], + [IntrWriteMem, IntrHasSideEffects, ImmArg>]>; + +let TargetPrefix = "riscv" in { + // mls - load using the implementation-defined layout (must match a pripr mss + // to memory). + def int_riscv_ztt_mls : ZTT_Load_Intr; + + // mls.cm / mls.rm - load with an explicit column-major / row-major layout. + def int_riscv_ztt_mls_cm : ZTT_Load_Intr; + def int_riscv_ztt_mls_rm : ZTT_Load_Intr; + + // mss - store using the implementation-defined layout. + def int_riscv_ztt_mss : ZTT_Store_Intr; + + // mss.cm / mss.rm - store with an explicit column-major / row-major layout. + def int_riscv_ztt_mss_cm : ZTT_Store_Intr; + def int_riscv_ztt_mss_rm : ZTT_Store_Intr; +} + +//===----------------------------------------------------------------------===// +// State management intrinsics +//===----------------------------------------------------------------------===// +// Base class for register-to-register move intrinsics: both operands are M/Acc +// registers Signature: void (dest_reg_index: i32, src_reg_index: i32) +class ZTT_Move_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>]>; + +let TargetPrefix = "riscv" in { + // mmov.a.m - copy an Acc register to an M register group; dtypes must + // match, else amestatus.UN is set and dest is left untouched + def int_riscv_ztt_mmov_a_m : ZTT_Move_Intr; + + // mmov.m.m - copy one M register to another; dtypes must + // match, else amestatus.UN is set and dest is left untouched + def int_riscv_ztt_mmov_m_m : ZTT_Move_Intr; + + // mzero.2d - zero every element of an Acc register. + // Signature: void (acc_reg_index: i32) + def int_riscv_ztt_mzero_2d + : Intrinsic<[], [llvm_i64_ty], [IntrHasSideEffects, IntrNoMem, ImmArg>]>; +} + +//===----------------------------------------------------------------------===// +// Matrix multiply intrinsics +//===----------------------------------------------------------------------===// +// Base class for matrix multiply intrinsics +// Signature: void (acc_reg_index: i32, ms1_reg_index: i32, ms2_reg_index: i32) +class ZTT_MatMul_Intr : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], [ + IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>, + ImmArg> +]>; + +let TargetPrefix = "riscv" in { + // mmul.2d - acc[i, j] = sum over k of (ms1[i, k] * ms2[k, j] + def int_riscv_ztt_mmul_2d : ZTT_MatMul_Intr; + + // mmulneg.2d - acc[i, j] = -(sum over k of (ms1[i, k] * ms2[k, j]) + def int_riscv_ztt_mmulneg_2d : ZTT_MatMul_Intr; + + // mmulacc.2d - acc[i, j] += sum over k of (ms1[i, k] * ms2[k, j] + def int_riscv_ztt_mmulacc_2d : ZTT_MatMul_Intr; + + // mmulaccneg.2d - acc[i, j] -= sum over k of (ms1[i, k] * ms2[k, j] + def int_riscv_ztt_mmulaccneg_2d : ZTT_MatMul_Intr; + + // mmulat.2d - acc[i, j] = sum over k of (ms1[k, i] * ms2[k, j] + def int_riscv_ztt_mmulat_2d : ZTT_MatMul_Intr; + + // mmulatacc.2d - acc[i, j] += sum over k of (ms1[k, i] * ms2[k, j] + def int_riscv_ztt_mmulatacc_2d : ZTT_MatMul_Intr; + + // mmulbt.2d - acc[i, j] = sum over k of (ms1[i, k] * ms2[j, k] + def int_riscv_ztt_mmulbt_2d : ZTT_MatMul_Intr; + + // mmulbtacc.2d - acc[i, j] += sum over k of (ms1[i, k] * ms2[j, k] + def int_riscv_ztt_mmulbtacc_2d : ZTT_MatMul_Intr; +} + +//===----------------------------------------------------------------------===// +// Reduction intrinsics +//===----------------------------------------------------------------------===// +// Base class for reduction intrinsics. +// Signature: void (md_reg_index: i32, ms1_reg_index: i32) +class ZTT_Reduction_Intr + : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], + [IntrHasSideEffects, IntrNoMem, ImmArg>, ImmArg>]>; + +let TargetPrefix = "riscv" in { + // mprefixadd.col - md[i, j] = sum over k <= i of ms1[k, j] + def int_riscv_ztt_mprefixadd_col : ZTT_Reduction_Intr; + + // mprefixadd.row - md[i, j] = sum over k <= j of ms1[i, k] + def int_riscv_ztt_mprefixadd_row : ZTT_Reduction_Intr; + + // mprefixmax.col - md[i, j] = max over k <= i of ms1[k, j] + def int_riscv_ztt_mprefixmax_col : ZTT_Reduction_Intr; + + // mprefixmax.row - md[i, j] = max over k <= j of ms1[i, k] + def int_riscv_ztt_mprefixmax_row : ZTT_Reduction_Intr; + + // mreduceadd.col - md[i, j] = sum over k of ms1[k, j] + def int_riscv_ztt_mreduceadd_col : ZTT_Reduction_Intr; + + // mreduceadd.row - md[i, j] = sum over k of ms1[i, k] + def int_riscv_ztt_mreduceadd_row : ZTT_Reduction_Intr; + + // mreducemax.col - md[i, j] = max over k of ms1[k, j] + def int_riscv_ztt_mreducemax_col : ZTT_Reduction_Intr; + + // mreducemax.row - md[i, j] = max over k of ms1[i, k] + def int_riscv_ztt_mreducemax_row : ZTT_Reduction_Intr; +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXBBMeteor.td b/llvm/include/llvm/IR/IntrinsicsRISCVXBBMeteor.td new file mode 100644 index 0000000000000..9c1b2fa7f551f --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVXBBMeteor.td @@ -0,0 +1,18 @@ +//===- IntrinsicsRISCVXBBMeteor.td ----------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "riscv" in +def int_riscv_bb_bbfp_mul : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXBBPebble.td b/llvm/include/llvm/IR/IntrinsicsRISCVXBBPebble.td new file mode 100644 index 0000000000000..85831d44d8173 --- /dev/null +++ b/llvm/include/llvm/IR/IntrinsicsRISCVXBBPebble.td @@ -0,0 +1,33 @@ +//===- IntrinsicsRISCVXBBPebble.td ----------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +let TargetPrefix = "riscv" in +def int_riscv_bb_mul_warp16 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_bb_relu : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_bb_transpose : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_bb_im2col : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_bb_fp2int : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; + +let TargetPrefix = "riscv" in +def int_riscv_bb_int2fp : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/llvm/include/llvm/TargetParser/SubtargetFeature.h b/llvm/include/llvm/TargetParser/SubtargetFeature.h index 3c136b27374dd..42e31d3556dba 100644 --- a/llvm/include/llvm/TargetParser/SubtargetFeature.h +++ b/llvm/include/llvm/TargetParser/SubtargetFeature.h @@ -33,7 +33,7 @@ namespace llvm { class raw_ostream; class Triple; -const unsigned MAX_SUBTARGET_WORDS = 6; +const unsigned MAX_SUBTARGET_WORDS = 8; const unsigned MAX_SUBTARGET_FEATURES = MAX_SUBTARGET_WORDS * 64; /// Container class for subtarget features. @@ -183,7 +183,7 @@ class FeatureBitset { bool operator!=(const FeatureBitset &RHS) const { return !(*this == RHS); } - bool operator < (const FeatureBitset &Other) const { + bool operator<(const FeatureBitset &Other) const { for (unsigned I = 0, E = size(); I != E; ++I) { bool LHS = test(I), RHS = Other.test(I); if (LHS != RHS) @@ -213,7 +213,7 @@ class FeatureBitArray : public FeatureBitset { /// that feature should be enabled or disabled contrary to the cpu /// specification. class SubtargetFeatures { - std::vector Features; ///< Subtarget features as a vector + std::vector Features; ///< Subtarget features as a vector public: LLVM_ABI explicit SubtargetFeatures(StringRef Initial = ""); @@ -244,7 +244,7 @@ class SubtargetFeatures { // Get first character char Ch = Feature[0]; // Check if first character is '+' or '-' flag - return Ch == '+' || Ch =='-'; + return Ch == '+' || Ch == '-'; } /// Return string stripped of flag. diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 8563f678464a6..8f8a8ae9efd82 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -221,6 +221,9 @@ class RISCVAsmParser : public MCTargetAsmParser { ParseStatus parseJALOffset(OperandVector &Operands); ParseStatus parseVTypeI(OperandVector &Operands); ParseStatus parseMaskReg(OperandVector &Operands); + ParseStatus parseTHeadAMEMatrixReg(OperandVector &Operands); + ParseStatus parseZttMatrixRegIndex(OperandVector &Operands); + ParseStatus parseZttAccRegIndex(OperandVector &Operands); ParseStatus parseVScaleReg(OperandVector &Operands); ParseStatus parseTileLambda(OperandVector &Operands); ParseStatus parseInsnDirectiveOpcode(OperandVector &Operands); @@ -493,6 +496,10 @@ struct RISCVOperand final : public MCParsedAsmOperand { bool isV0Reg() const { return Kind == KindTy::Register && Reg.Reg == RISCV::V0; } + bool isTHeadAMEMatrixReg() const { + return Kind == KindTy::Register && Reg.Reg >= RISCV::THeadAMEM0 && + Reg.Reg <= RISCV::THeadAMEM7; + } bool isAnyReg() const { return Kind == KindTy::Register && (getRISCVMCRegisterClass(RISCV::GPRRegClassID).contains(Reg.Reg) || @@ -1050,6 +1057,8 @@ struct RISCVOperand final : public MCParsedAsmOperand { [](int64_t Imm) { return Imm != INT64_MIN && isInt<5>(Imm - 1); }); } + bool isSImm7() const { return isSImm<7>(); } + bool isSImm18() const { return isSImmPred([](int64_t Imm) { return isInt<18>(Imm); }); } @@ -2649,6 +2658,105 @@ ParseStatus RISCVAsmParser::parseMaskReg(OperandVector &Operands) { return ParseStatus::Success; } +ParseStatus RISCVAsmParser::parseTHeadAMEMatrixReg(OperandVector &Operands) { + if (getLexer().isNot(AsmToken::Identifier)) + return ParseStatus::NoMatch; + + StringRef Name = getLexer().getTok().getIdentifier(); + if (Name.size() != 2 || Name[0] != 'm' || Name[1] < '0' || Name[1] > '7') + return ParseStatus::NoMatch; + + SMLoc S = getLoc(); + SMLoc E = getTok().getEndLoc(); + getLexer().Lex(); + Operands.push_back( + RISCVOperand::createReg(RISCV::THeadAMEM0 + (Name[1] - '0'), S, E)); + return ParseStatus::Success; +} + +ParseStatus RISCVAsmParser::parseZttMatrixRegIndex(OperandVector &Operands) { + if (getLexer().isNot(AsmToken::Identifier)) + return ParseStatus::NoMatch; + + StringRef Name = getLexer().getTok().getIdentifier(); + if (!Name.consume_front("m")) + return ParseStatus::NoMatch; + + unsigned Index; + if (Name.getAsInteger(10, Index)) + return ParseStatus::NoMatch; + + unsigned MaxIndex; + const auto &Features = STI->getFeatureBits(); + + bool HasZttMatrixRegs32 = Features[RISCV::FeatureStdExtZttMatrixRegs32]; + bool HasZttMatrixRegs16 = Features[RISCV::FeatureStdExtZttMatrixRegs16]; + if (HasZttMatrixRegs16 && HasZttMatrixRegs32) + return Error(getLoc(), + "conflicting ztt (AME) matrix register bounds chosen; cannot enable both 16 and 32 matrix registers simultaneously"); + + if (HasZttMatrixRegs32) + MaxIndex = 31; + else if (HasZttMatrixRegs16) + MaxIndex = 15; + else + return Error(getLoc(), + "no ztt (AME) matrix register count feature enabled"); + + if (Index > MaxIndex) + return generateImmOutOfRangeError(getLoc(), 0, MaxIndex); + + SMLoc S = getLoc(); + SMLoc E = getTok().getEndLoc(); + getLexer().Lex(); + Operands.push_back(RISCVOperand::createExpr( + MCConstantExpr::create(Index, getContext()), S, E, isRV64())); + return ParseStatus::Success; +} + +ParseStatus RISCVAsmParser::parseZttAccRegIndex(OperandVector &Operands) { + if (getLexer().isNot(AsmToken::Identifier)) + return ParseStatus::NoMatch; + + StringRef Name = getLexer().getTok().getIdentifier(); + if (!Name.consume_front("acc")) + return ParseStatus::NoMatch; + + unsigned Index; + if (Name.getAsInteger(10, Index)) + return ParseStatus::NoMatch; + + unsigned MaxIndex; + const auto &Features = STI->getFeatureBits(); + + bool HasZttAccRegs4 = Features[RISCV::FeatureStdExtZttAccRegs4]; + bool HasZttAccRegs2 = Features[RISCV::FeatureStdExtZttAccRegs2]; + bool HasZttAccRegs1 = Features[RISCV::FeatureStdExtZttAccRegs1]; + if ((HasZttAccRegs4 + HasZttAccRegs2 + HasZttAccRegs1) > 1) + return Error(getLoc(), + "conflicting ztt (AME) accumulator register bounds chosen; cannot enable multiple accumulator register configurations simultaneously"); + + if (Features[RISCV::FeatureStdExtZttAccRegs4]) + MaxIndex = 3; + else if (Features[RISCV::FeatureStdExtZttAccRegs2]) + MaxIndex = 1; + else if (Features[RISCV::FeatureStdExtZttAccRegs1]) + MaxIndex = 0; + else + return Error(getLoc(), + "no ztt (AME) accumulator register count feature enabled"); + + if (Index > MaxIndex) + return generateImmOutOfRangeError(getLoc(), 0, MaxIndex); + + SMLoc S = getLoc(); + SMLoc E = getTok().getEndLoc(); + getLexer().Lex(); + Operands.push_back(RISCVOperand::createExpr( + MCConstantExpr::create(Index, getContext()), S, E, isRV64())); + return ParseStatus::Success; +} + ParseStatus RISCVAsmParser::parseVScaleReg(OperandVector &Operands) { if (getLexer().isNot(AsmToken::Identifier)) return ParseStatus::NoMatch; diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp index 3b3eb5195a9b4..a2459df0cfa65 100644 --- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp +++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp @@ -24,6 +24,7 @@ #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Endian.h" +#include using namespace llvm; using namespace llvm::MCD; @@ -96,6 +97,32 @@ static DecodeStatus DecodeSimpleRegisterClass(MCInst &Inst, uint32_t RegNo, constexpr auto DecodeGPRRegisterClass = DecodeSimpleRegisterClass; +static DecodeStatus +DecodeBOSCAMETileRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { + return DecodeSimpleRegisterClass(Inst, RegNo, Address, + Decoder); +} + +static DecodeStatus +DecodeBOSCAMEAccRegRegisterClass(MCInst &Inst, uint32_t RegNo, uint64_t Address, + const MCDisassembler *Decoder) { + if (RegNo < 8 || RegNo > 15) + return MCDisassembler::Fail; + + Inst.addOperand(MCOperand::createReg(RISCV::BOSCAMEACC0 + (RegNo - 8))); + return MCDisassembler::Success; +} + +static DecodeStatus +DecodeTHeadAMEMatrixRegRegisterClass(MCInst &Inst, uint32_t RegNo, + uint64_t Address, + const MCDisassembler *Decoder) { + return DecodeSimpleRegisterClass(Inst, RegNo, Address, + Decoder); +} + static DecodeStatus DecodeGPRX1X5RegisterClass(MCInst &Inst, uint32_t RegNo, uint64_t Address, const MCDisassembler *Decoder) { diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h index e54d57d9f4451..61f7c8a138aef 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h @@ -146,6 +146,9 @@ enum OperandType : unsigned { OPERAND_VMASK, OPERAND_SMTVType, OPERAND_SMTI8, + + OPERAND_ZTT_MATRIX_REG, + OPERAND_ZTT_ACC_REG, }; } // namespace RISCVOp diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp index 1604645520dd1..56f729673c699 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp @@ -13,6 +13,7 @@ #include "RISCVInstPrinter.h" #include "RISCVBaseInfo.h" #include "RISCVMCAsmInfo.h" +#include "RISCVMCTargetDesc.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" @@ -346,6 +347,37 @@ void RISCVInstPrinter::printVMaskReg(const MCInst *MI, unsigned OpNo, O << ".t"; } +void RISCVInstPrinter::printTHeadAMEMatrixReg(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, + raw_ostream &O) { + const MCOperand &MO = MI->getOperand(OpNo); + + assert(MO.isReg() && + "printTHeadAMEMatrixReg can only print register operands"); + assert(MO.getReg() >= RISCV::THeadAMEM0 && MO.getReg() <= RISCV::THeadAMEM7 && + "unexpected matrix register"); + O << "m" << (MO.getReg() - RISCV::THeadAMEM0); +} + +void RISCVInstPrinter::printZttMatrixRegIndex(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, + raw_ostream &O) { + const MCOperand &MO = MI->getOperand(OpNo); + + assert(MO.isImm() && + "printZttMatrixRegIndex can only print immediate operands"); + O << "m" << MO.getImm(); +} + +void RISCVInstPrinter::printZttAccRegIndex(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, + raw_ostream &O) { + const MCOperand &MO = MI->getOperand(OpNo); + + assert(MO.isImm() && "printZttAccRegIndex can only print immediate operands"); + O << "acc" << MO.getImm(); +} + void RISCVInstPrinter::printVScaleReg(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O) { diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h index a9cfdbdf99503..ca5e1dbd17b32 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h @@ -54,6 +54,12 @@ class RISCVInstPrinter : public MCInstPrinter { const MCSubtargetInfo &STI, raw_ostream &O); void printVMaskReg(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O); + void printTHeadAMEMatrixReg(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O); + void printZttMatrixRegIndex(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O); + void printZttAccRegIndex(const MCInst *MI, unsigned OpNo, + const MCSubtargetInfo &STI, raw_ostream &O); void printVScaleReg(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O); void printTileLambda(const MCInst *MI, unsigned OpNo, diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp index e37ea27364e6e..2757b57bf2883 100644 --- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp +++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp @@ -15,6 +15,7 @@ #include "MCTargetDesc/RISCVELFStreamer.h" #include "MCTargetDesc/RISCVInstPrinter.h" #include "MCTargetDesc/RISCVMCAsmInfo.h" +#include "MCTargetDesc/RISCVMCTargetDesc.h" #include "MCTargetDesc/RISCVMatInt.h" #include "MCTargetDesc/RISCVTargetStreamer.h" #include "RISCV.h" @@ -109,6 +110,8 @@ class RISCVAsmPrinter : public AsmPrinter { // Wrapper needed for tblgenned pseudo lowering. bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const; + bool lowerMatrixExtPseudo(const MachineInstr *MI, MCInst &Inst); + void emitStartOfAsmFile(Module &M) override; void emitEndOfAsmFile(Module &M) override; @@ -377,6 +380,199 @@ void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, Hint); } +static unsigned getBOSCAMETileReg(unsigned Index) { + static const unsigned Regs[] = {RISCV::BOSCAMETR0, RISCV::BOSCAMETR1, + RISCV::BOSCAMETR2, RISCV::BOSCAMETR3, + RISCV::BOSCAMETR4, RISCV::BOSCAMETR5, + RISCV::BOSCAMETR4, RISCV::BOSCAMETR7}; + assert(Index < 8 && "invalid AME tile register index"); + return Regs[Index]; +} + +static unsigned getBOSCAMEAccReg(unsigned Index) { + static const unsigned Regs[] = {RISCV::BOSCAMEACC0, RISCV::BOSCAMEACC1, + RISCV::BOSCAMEACC2, RISCV::BOSCAMEACC3, + RISCV::BOSCAMEACC4, RISCV::BOSCAMEACC5, + RISCV::BOSCAMEACC6, RISCV::BOSCAMEACC7}; + assert(Index < 8 && "invalid AME accumulator register index"); + return Regs[Index]; +} + +static unsigned getTHeadAMEMatrixReg(unsigned Index) { + static const unsigned Regs[] = {RISCV::THeadAMEM0, RISCV::THeadAMEM1, + RISCV::THeadAMEM2, RISCV::THeadAMEM3, + RISCV::THeadAMEM4, RISCV::THeadAMEM5, + RISCV::THeadAMEM6, RISCV::THeadAMEM7}; + assert(Index < 8 && "invalid AME matrix register index"); + return Regs[Index]; +} + +static void addImmOperand(MCInst &Inst, const MachineOperand &MO) { + Inst.addOperand(MCOperand::createImm(MO.getImm())); +} + +bool RISCVAsmPrinter::lowerMatrixExtPseudo(const MachineInstr *MI, + MCInst &Inst) { + auto AddLoweredOperand = [&](unsigned OpNo) -> bool { + MCOperand MCOp; + if (!lowerOperand(MI->getOperand(OpNo), MCOp)) + return false; + Inst.addOperand(MCOp); + return true; + }; + + auto AddTileIndex = [&](unsigned OpNo) { + Inst.addOperand( + MCOperand::createReg(getBOSCAMETileReg(MI->getOperand(OpNo).getImm()))); + }; + auto AddAccIndex = [&](unsigned OpNo) { + Inst.addOperand( + MCOperand::createReg(getBOSCAMEAccReg(MI->getOperand(OpNo).getImm()))); + }; + auto AddMatrixIndex = [&](unsigned OpNo) { + Inst.addOperand(MCOperand::createReg( + getTHeadAMEMatrixReg(MI->getOperand(OpNo).getImm()))); + }; + + switch (MI->getOpcode()) { + default: + return false; + case RISCV::BOSC_AME_MSETTILEMI_PSEUDO: + Inst.setOpcode(RISCV::BOSC_AME_MSETTILEMI); + if (!AddLoweredOperand(0)) + return false; + addImmOperand(Inst, MI->getOperand(1)); + return true; + case RISCV::BOSC_AME_MSETTILENI_PSEUDO: + Inst.setOpcode(RISCV::BOSC_AME_MSETTILENI); + if (!AddLoweredOperand(0)) + return false; + addImmOperand(Inst, MI->getOperand(1)); + return true; + case RISCV::BOSC_AME_MSETTILEKI_PSEUDO: + Inst.setOpcode(RISCV::BOSC_AME_MSETTILEKI); + if (!AddLoweredOperand(0)) + return false; + addImmOperand(Inst, MI->getOperand(1)); + return true; + // case RISCV::BOSC_AME_MZERO_PSEUDO: + // Inst.setOpcode(RISCV::BOSC_AME_MZERO_M); + // AddAccIndex(0); + // return true; + case RISCV::BOSC_AME_MLAE32_M_PSEUDO: + Inst.setOpcode(RISCV::BOSC_AME_MLAE32_M); + AddTileIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::BOSC_AME_MLBE32_M_PSEUDO: + Inst.setOpcode(RISCV::BOSC_AME_MLBE32_M); + AddTileIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::BOSC_AME_MSCE32_M_PSEUDO: + Inst.setOpcode(RISCV::BOSC_AME_MSCE32_M); + AddAccIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::BOSC_AME_MMA_W_MM_PSEUDO: + Inst.setOpcode(RISCV::BOSC_AME_MMA_W_MM); + AddAccIndex(0); + AddAccIndex(0); + AddTileIndex(1); + AddTileIndex(2); + return true; + case RISCV::TH_MCFGMI_PSEUDO: + Inst.setOpcode(RISCV::TH_MCFGMI); + addImmOperand(Inst, MI->getOperand(0)); + return true; + case RISCV::TH_MCFGNI_PSEUDO: + Inst.setOpcode(RISCV::TH_MCFGNI); + addImmOperand(Inst, MI->getOperand(0)); + return true; + case RISCV::TH_MCFGKI_PSEUDO: + Inst.setOpcode(RISCV::TH_MCFGKI); + addImmOperand(Inst, MI->getOperand(0)); + return true; + case RISCV::TH_MZERO_PSEUDO: + Inst.setOpcode(RISCV::TH_MZERO); + AddMatrixIndex(0); + return true; + case RISCV::TH_MLDE8_PSEUDO: + Inst.setOpcode(RISCV::TH_MLDE8); + AddMatrixIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::TH_MLDE16_PSEUDO: + Inst.setOpcode(RISCV::TH_MLDE16); + AddMatrixIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::TH_MLDE32_PSEUDO: + Inst.setOpcode(RISCV::TH_MLDE32); + AddMatrixIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::TH_MLDE64_PSEUDO: + Inst.setOpcode(RISCV::TH_MLDE64); + AddMatrixIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::TH_MLDTE8_PSEUDO: + Inst.setOpcode(RISCV::TH_MLDTE8); + AddMatrixIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::TH_MLDTE16_PSEUDO: + Inst.setOpcode(RISCV::TH_MLDTE16); + AddMatrixIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::TH_MLDTE32_PSEUDO: + Inst.setOpcode(RISCV::TH_MLDTE32); + AddMatrixIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::TH_MLDTE64_PSEUDO: + Inst.setOpcode(RISCV::TH_MLDTE64); + AddMatrixIndex(0); + return AddLoweredOperand(1) && AddLoweredOperand(2); + case RISCV::TH_MSTE8_PSEUDO: + Inst.setOpcode(RISCV::TH_MSTE8); + AddMatrixIndex(0); + return AddLoweredOperand(2) && AddLoweredOperand(1); + case RISCV::TH_MSTE16_PSEUDO: + Inst.setOpcode(RISCV::TH_MSTE16); + AddMatrixIndex(0); + return AddLoweredOperand(2) && AddLoweredOperand(1); + case RISCV::TH_MSTE32_PSEUDO: + Inst.setOpcode(RISCV::TH_MSTE32); + AddMatrixIndex(0); + return AddLoweredOperand(2) && AddLoweredOperand(1); + case RISCV::TH_MSTE64_PSEUDO: + Inst.setOpcode(RISCV::TH_MSTE64); + AddMatrixIndex(0); + return AddLoweredOperand(2) && AddLoweredOperand(1); + case RISCV::TH_MMACC_W_B_PSEUDO: + Inst.setOpcode(RISCV::TH_MMACC_W_B); + AddMatrixIndex(0); + AddMatrixIndex(0); + AddMatrixIndex(1); + AddMatrixIndex(2); + return true; + case RISCV::TH_MMACCU_W_B_PSEUDO: + Inst.setOpcode(RISCV::TH_MMACCU_W_B); + AddMatrixIndex(0); + AddMatrixIndex(0); + AddMatrixIndex(1); + AddMatrixIndex(2); + return true; + case RISCV::TH_MFMACC_S_PSEUDO: + Inst.setOpcode(RISCV::TH_MFMACC_S); + AddMatrixIndex(0); + AddMatrixIndex(0); + AddMatrixIndex(1); + AddMatrixIndex(2); + return true; + case RISCV::TH_MFMACC_D_S_PSEUDO: + Inst.setOpcode(RISCV::TH_MFMACC_D_S); + AddMatrixIndex(0); + AddMatrixIndex(0); + AddMatrixIndex(1); + AddMatrixIndex(2); + return true; + } +} + void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) { RISCV_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits()); @@ -388,6 +584,11 @@ void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) { return; } + if (MCInst OutInst; lowerMatrixExtPseudo(MI, OutInst)) { + EmitToStreamer(*OutStreamer, OutInst); + return; + } + switch (MI->getOpcode()) { case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: LowerHWASAN_CHECK_MEMACCESS(*MI); diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index 03d28b8448ba4..73b4de09e253f 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -957,6 +957,36 @@ def HasStdExtZvvmttls : Predicate<"Subtarget->hasStdExtZvvmttls()">, AssemblerPredicate<(all_of FeatureStdExtZvvmttls), "'Zvvmttls' (Transposing Matrix Tile Load/Store)">; +// Attached Matrix Extension +def FeatureStdExtZttMatrixRegs16 + : SubtargetFeature<"experimental-ztt-ame-mregs-16", "HasZttMatrixRegs16", "true", + "Enable support for 16 AME matrix registers">; + +def FeatureStdExtZttMatrixRegs32 + : SubtargetFeature<"experimental-ztt-ame-mregs-32", "HasZttMatrixRegs32", "true", + "Enable support for 32 AME matrix registers">; + +def FeatureStdExtZttAccRegs1 + : SubtargetFeature<"experimental-ztt-ame-accregs-1", "HasZttAccRegs1", "true", + "Enable support for AME accumulator register">; + +def FeatureStdExtZttAccRegs2 + : SubtargetFeature<"experimental-ztt-ame-accregs-2", "HasZttAccRegs2", "true", + "Enable support for 2 AME accumulator registers">; + +def FeatureStdExtZttAccRegs4 + : SubtargetFeature<"experimental-ztt-ame-accregs-4", "HasZttAccRegs4", "true", + "Enable support for 4 AME accumulator registers">; + +let Implies = [FeatureStdExtZttMatrixRegs32, FeatureStdExtZttAccRegs4] in { + def FeatureStdExtZtt // By default, enabling "+experimental-ztt" implies 32 Matrix Regs and 4 Accumulators. + : RISCVExperimentalExtension<0, 1, "Attached Matrix Extension">; +} + +def HasStdExtZtt : Predicate<"Subtarget->hasStdExtZtt()">, + AssemblerPredicate<(all_of FeatureStdExtZtt), + "'Ztt' (Attached Matrix Extension)">; + // Zvbdota family of batched dot-product extensions def FeatureStdExtZvqwbdota8i : RISCVExperimentalExtension<0, 2, @@ -1341,6 +1371,154 @@ def HasVendorXTHeadVdot : Predicate<"Subtarget->hasVendorXTHeadVdot()">, AssemblerPredicate<(all_of FeatureVendorXTHeadVdot), "'XTHeadVdot' (T-Head Vector Extensions for Dot)">; +def FeatureVendorXGemmini + : RISCVExtension<1, 0, "Gemmini extension">; +def HasVendorXGemmini + : Predicate<"Subtarget->hasVendorXGemmini()">, + AssemblerPredicate<(all_of FeatureVendorXGemmini), + "'XGemmini' (Gemmini extension)">; + +def FeatureVendorXXiangShanAME + : RISCVExtension<1, 0, "OpenXiangShan Advanced Matrix Extension">; + +def FeatureVendorXBOSCAME + : SubtargetFeature<"xboscame", "HasVendorXBOSCAME", "true", + "BOSC AME Alias (Enable xxiangshaname)", + [FeatureVendorXXiangShanAME]>; +def HasVendorXBOSCAME + : Predicate<"Subtarget->hasVendorXBOSCAME()">, + AssemblerPredicate<(all_of FeatureVendorXBOSCAME), + "'XBOSCAME' (BOSC AME extension)">; + +def FeatureVendorXBBPebble + : RISCVExtension<1, 0, "Buckyball CNN extension">; +def HasVendorXBBPebble + : Predicate<"Subtarget->hasVendorXBBPebble()">, + AssemblerPredicate<(all_of FeatureVendorXBBPebble), + "'XBBPebble' (Buckyball CNN extension)">; + +def FeatureVendorXBBMeteor + : RISCVExtension<1, 0, "Buckyball Large Language Model extension">; +def HasVendorXBBMeteor + : Predicate<"Subtarget->hasVendorXBBMeteor()">, + AssemblerPredicate<(all_of FeatureVendorXBBMeteor), + "'XBBMeteor' (Buckyball Large Language Model extension)">; + +def HasVendorXBuckyballBase + : Predicate<"Subtarget->hasVendorXBBPebble() || " + "Subtarget->hasVendorXBBMeteor()">, + AssemblerPredicate<(any_of FeatureVendorXBBPebble, + FeatureVendorXBBMeteor), + "'XBBPebble' (Buckyball CNN extension) or " + "'XBBMeteor' (Buckyball Large Language Model extension)">; + +class XTHeadAMEExtension + : RISCVExtension; + +def FeatureVendorXTHeadMatrix + : XTHeadAMEExtension<3, 0, "matrix extension">; +def FeatureVendorXTHeadMatrixMin + : XTHeadAMEExtension<0, 5, "minimal matrix extension">; +def FeatureVendorXTHeadMDMA + : XTHeadAMEExtension<0, 5, "matrix DMA extension">; +def FeatureVendorXTHeadMEW4B + : XTHeadAMEExtension<0, 5, "4-bit element width extension">; +def FeatureVendorXTHeadMEW8B + : XTHeadAMEExtension<0, 5, "8-bit element width extension">; +def FeatureVendorXTHeadMEW16B + : XTHeadAMEExtension<0, 5, "16-bit element width extension">; +def FeatureVendorXTHeadMEW32B + : XTHeadAMEExtension<0, 5, "32-bit element width extension">; +def FeatureVendorXTHeadMEW64B + : XTHeadAMEExtension<0, 5, "64-bit element width extension">; +def FeatureVendorXTHeadMFEW + : XTHeadAMEExtension<0, 5, "matrix fractional element width extension">; +def FeatureVendorXTHeadMFIC + : XTHeadAMEExtension<0, 5, "matrix fixed-point conversion extension">; +def FeatureVendorXTHeadMHP + : XTHeadAMEExtension<0, 5, "matrix half precision extension">; +def FeatureVendorXTHeadMIEW + : XTHeadAMEExtension<0, 5, "matrix integer element width extension">; +def FeatureVendorXTHeadMMBF16BBF16B + : XTHeadAMEExtension<0, 5, "BF16-by-BF16 matrix multiply extension">; +def FeatureVendorXTHeadMMBF16BF32B + : XTHeadAMEExtension<0, 5, "BF16-by-F32 matrix multiply extension">; +def FeatureVendorXTHeadMMBF20BF32B + : XTHeadAMEExtension<0, 5, "BF20-by-F32 matrix multiply extension">; +def FeatureVendorXTHeadMMF4BBF16B + : XTHeadAMEExtension<0, 5, "F4-by-BF16 matrix multiply extension">; +def FeatureVendorXTHeadMMF4BF16B + : XTHeadAMEExtension<0, 5, "F4-by-F16 matrix multiply extension">; +def FeatureVendorXTHeadMMF4BF32B + : XTHeadAMEExtension<0, 5, "F4-by-F32 matrix multiply extension">; +def FeatureVendorXTHeadMMF8BBF16B + : XTHeadAMEExtension<0, 5, "F8-by-BF16 matrix multiply extension">; +def FeatureVendorXTHeadMMF8BF16B + : XTHeadAMEExtension<0, 5, "F8-by-F16 matrix multiply extension">; +def FeatureVendorXTHeadMMF8BF32B + : XTHeadAMEExtension<0, 5, "F8-by-F32 matrix multiply extension">; +def FeatureVendorXTHeadMMF16BF16B + : XTHeadAMEExtension<0, 5, "F16-by-BF16 matrix multiply extension">; +def FeatureVendorXTHeadMMF16BF32B + : XTHeadAMEExtension<0, 5, "F16-by-F32 matrix multiply extension">; +def FeatureVendorXTHeadMMF32BF32B + : XTHeadAMEExtension<0, 5, "F32-by-F32 matrix multiply extension">; +def FeatureVendorXTHeadMMF32BF64B + : XTHeadAMEExtension<0, 5, "F32-by-F64 matrix multiply extension">; +def FeatureVendorXTHeadMMF64BF64B + : XTHeadAMEExtension<0, 5, "F64-by-F64 matrix multiply extension">; +def FeatureVendorXTHeadMMI4BI32B + : XTHeadAMEExtension<0, 5, "I4-by-I32 matrix multiply extension">; +def FeatureVendorXTHeadMMI8BI32B + : XTHeadAMEExtension<0, 5, "I8-by-I32 matrix multiply extension">; +def FeatureVendorXTHeadMMMXF4B + : XTHeadAMEExtension<0, 5, "MXF4 matrix multiply extension">; +def FeatureVendorXTHeadMMMXF8B + : XTHeadAMEExtension<0, 5, "MXF8 matrix multiply extension">; +def FeatureVendorXTHeadMMMXF8BMXF4B + : XTHeadAMEExtension<0, 5, "MXF8-by-MXF4 matrix multiply extension">; +def FeatureVendorXTHeadMRED + : XTHeadAMEExtension<0, 5, "matrix reduction extension">; + +def FeatureVendorXTHeadAME + : SubtargetFeature<"xtheadame", "HasVendorXTHeadAME", "true", + "Enable all T-Head AME matrix extensions", + [FeatureVendorXTHeadMatrix, + FeatureVendorXTHeadMatrixMin, + FeatureVendorXTHeadMDMA, + FeatureVendorXTHeadMEW4B, + FeatureVendorXTHeadMEW8B, + FeatureVendorXTHeadMEW16B, + FeatureVendorXTHeadMEW32B, + FeatureVendorXTHeadMEW64B, + FeatureVendorXTHeadMFEW, + FeatureVendorXTHeadMFIC, + FeatureVendorXTHeadMHP, + FeatureVendorXTHeadMIEW, + FeatureVendorXTHeadMMBF16BBF16B, + FeatureVendorXTHeadMMBF16BF32B, + FeatureVendorXTHeadMMBF20BF32B, + FeatureVendorXTHeadMMF4BBF16B, + FeatureVendorXTHeadMMF4BF16B, + FeatureVendorXTHeadMMF4BF32B, + FeatureVendorXTHeadMMF8BBF16B, + FeatureVendorXTHeadMMF8BF16B, + FeatureVendorXTHeadMMF8BF32B, + FeatureVendorXTHeadMMF16BF16B, + FeatureVendorXTHeadMMF16BF32B, + FeatureVendorXTHeadMMF32BF32B, + FeatureVendorXTHeadMMF32BF64B, + FeatureVendorXTHeadMMF64BF64B, + FeatureVendorXTHeadMMI4BI32B, + FeatureVendorXTHeadMMI8BI32B, + FeatureVendorXTHeadMMMXF4B, + FeatureVendorXTHeadMMMXF8B, + FeatureVendorXTHeadMMMXF8BMXF4B, + FeatureVendorXTHeadMRED]>; +def HasVendorXTHeadAME : Predicate<"Subtarget->hasVendorXTHeadAME()">, + AssemblerPredicate<(all_of FeatureVendorXTHeadAME), + "'XTHeadAME' (T-Head AME matrix extensions)">; + // SiFive Extensions def FeatureVendorXSfvcp @@ -1864,6 +2042,13 @@ def HasVendorXSMTVDotII AssemblerPredicate<(all_of FeatureVendorXSMTVDotII), "'XSMTVDotII' (SpacemiT Vector Extension for Matrix 2.0)">; +def FeatureVendorXSMTIME + : RISCVExtension<1, 0, "SpacemiT IME extension">; +def HasVendorXSMTIME + : Predicate<"Subtarget->hasVendorXSMTIME()">, + AssemblerPredicate<(all_of FeatureVendorXSMTIME), + "'XSMTIME' (SpacemiT IME extension)">; + def FeatureVendorXAIFET : RISCVExtension<1, 0, "AI Foundry ET Extension", [FeatureStdExtF]>; def HasXAIFET : Predicate<"Subtarget->hasXAIFET()">, diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 50f548857a97b..931fc4ca1bd1a 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -12,6 +12,7 @@ #include "RISCVInstrInfo.h" #include "MCTargetDesc/RISCVBaseInfo.h" +#include "MCTargetDesc/RISCVMCTargetDesc.h" #include "MCTargetDesc/RISCVMatInt.h" #include "RISCV.h" #include "RISCVMachineFunctionInfo.h" @@ -3247,6 +3248,41 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI, return false; } break; + case RISCVOp::OPERAND_ZTT_MATRIX_REG: { + if (!MO.isImm()) { + ErrInfo = "Expected an immediate operand."; + return false; + } + int64_t Val = MO.getImm(); + const auto &Features = STI.getFeatureBits(); + if ((Features[RISCV::FeatureStdExtZttMatrixRegs32] && + (Val < 0 || Val >= 32)) || + (Features[RISCV::FeatureStdExtZttMatrixRegs16] && + (Val < 0 || Val >= 16))) { + ErrInfo = "matrix register index out of range for current subtarget"; + return false; + } + break; + } + + case RISCVOp::OPERAND_ZTT_ACC_REG: { + if (!MO.isImm()) { + ErrInfo = "Expected an immediate operand."; + return false; + } + int64_t Val = MO.getImm(); + const auto &Features = STI.getFeatureBits(); + if ((Features[RISCV::FeatureStdExtZttAccRegs4] && + (Val < 0 || Val >= 4)) || + (Features[RISCV::FeatureStdExtZttAccRegs2] && + (Val < 0 || Val >= 2)) || + (Features[RISCV::FeatureStdExtZttAccRegs1] && Val != 0)) { + ErrInfo = + "accumulator register index out of range for current subtarget"; + return false; + } + break; + } } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index e10ee7389db0e..83becdcbf4f79 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -2432,7 +2432,11 @@ include "RISCVInstrInfoXqccmt.td" include "RISCVInstrInfoXMips.td" include "RISCVInstrInfoXAndes.td" include "RISCVInstrInfoXSpacemiT.td" +include "RISCVInstrInfoXGemmini.td" +include "RISCVInstrInfoXBuckyballExt.td" +include "RISCVInstrInfoXSpacemiTIME.td" include "RISCVInstrInfoXAIF.td" +include "RISCVMatrixExt.td" //===----------------------------------------------------------------------===// // Global ISel diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXBBMeteor.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXBBMeteor.td new file mode 100644 index 0000000000000..eb2b589aac318 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXBBMeteor.td @@ -0,0 +1,30 @@ +//===- RISCVInstrInfoXBBMeteor.td -----------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// Buckyball Meteor Large Language Model Extension. +// +//===----------------------------------------------------------------------===// + +let isCodeGenOnly = 1, Predicates = [HasVendorXBBMeteor] in +def BB_BBFP_MUL : RVInstR<0b1000001, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_bbfp_mul", + "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXBBMeteor] in +def : Pat<(int_riscv_bb_bbfp_mul GPR:$rs1, GPR:$rs2), + (BB_BBFP_MUL GPR:$rs1, GPR:$rs2)>; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXBBPebble.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXBBPebble.td new file mode 100644 index 0000000000000..ccd8b1041846f --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXBBPebble.td @@ -0,0 +1,86 @@ +//===- RISCVInstrInfoXBBPebble.td -----------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// Buckyball Pebble CNN Extension. +// +//===----------------------------------------------------------------------===// + +let isCodeGenOnly = 1, Predicates = [HasVendorXBBPebble] in +def BB_MUL_WARP16 : RVInstR<0b1000000, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_mul_warp16", + "$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, + Predicates = [HasVendorXBBPebble] in +def BB_RELU : RVInstR<0b0110010, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_relu", "$rs1, $rs2"> { + let rd = 0; +} + +let isCodeGenOnly = 1, Predicates = [HasVendorXBBPebble] in +def BB_TRANSPOSE : RVInstR<0b0110001, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_transpose", + "$rs1, $rs2"> { + let rd = 0; +} + +let isCodeGenOnly = 1, Predicates = [HasVendorXBBPebble] in +def BB_IM2COL : RVInstR<0b0110000, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_im2col", + "$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, + Predicates = [HasVendorXBBPebble] in +def BB_FP2INT : RVInstR<0b0110011, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_quant", "$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, + Predicates = [HasVendorXBBPebble] in +def BB_INT2FP : RVInstR<0b0110100, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_dequant", + "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXBBPebble] in +def : Pat<(int_riscv_bb_mul_warp16 GPR:$rs1, GPR:$rs2), + (BB_MUL_WARP16 GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXBBPebble] in +def : Pat<(int_riscv_bb_relu GPR:$rs1, GPR:$rs2), + (BB_RELU GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXBBPebble] in +def : Pat<(int_riscv_bb_transpose GPR:$rs1, GPR:$rs2), + (BB_TRANSPOSE GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXBBPebble] in +def : Pat<(int_riscv_bb_im2col GPR:$rs1, GPR:$rs2), + (BB_IM2COL GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXBBPebble] in +def : Pat<(int_riscv_bb_fp2int GPR:$rs1, GPR:$rs2), + (BB_FP2INT GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXBBPebble] in +def : Pat<(int_riscv_bb_int2fp GPR:$rs1, GPR:$rs2), + (BB_INT2FP GPR:$rs1, GPR:$rs2)>; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXBOSCAME.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXBOSCAME.td new file mode 100644 index 0000000000000..8b6f48f460e27 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXBOSCAME.td @@ -0,0 +1,3301 @@ +//===------ RISCVInstrInfoXBOSCAME.td --------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// This is the instruction information file for the RISC-V BOSC AME extension. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// BOSC AME (RISC-V Matrix Extension) 32-bit Instructions +//===----------------------------------------------------------------------===// +// Reference: BOSC RISC-V Matrix Extension Specification +// 32-bit encoding format with prefix 1110111 at bits [6:0] +// +// Register Model: +// - BOSCAMETileReg (tr0-tr7): Input matrices A and B, MLEN bits each +// - BOSCAMEAccReg (acc0-acc7): Accumulator matrix C, MLEN×AMUL bits each +// +// Data Flow: +// Memory --[mlae/mlbe]--> BOSCAMETileReg --[mma]--> BOSCAMEAccReg --[msce]--> Memory +// +// Matrix Multiplication Instruction Format (32-bit): +// | 31:26 | 25 | 24 | 23:20 | 19 | 18:15 | 14:12 | 11 | 10:7 | 6:0 | +// | funct6 | fp | sa | ms2 | sn | ms1 | eew | ma | md | OP-M32 | +// +// suffix = 1110111 (BOSC AME prefix) +// +// Widening Instructions (required for AMUL > 1): +// - mqma.b.mm: int8 → int32 (quad-widen, AMUL ≥ 4, mmi8i32 MANDATORY) +// - mwma.h.mm: int16 → int32 (double-widen, AMUL ≥ 2) +// - mwma.w.mm: int32 → int64 (double-widen, AMUL ≥ 2) +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// AME Custom Operand Types for Tile Register Indices +//===----------------------------------------------------------------------===// +// These operand types allow intrinsics to pass immediate indices (0-7) +// for tile and accumulator registers. The AsmString of pseudo instructions +// hardcodes "acc" and "tr" prefixes so that indices 0-7 are printed as +// acc0-acc7 and tr0-tr7 respectively, without modifying LLVM submodule. + +// AsmOperandClass for tile register index (0-7) +def BOSCAMETileIndexAsmOperand : AsmOperandClass { + let Name = "BOSCAMETileIndex"; + let RenderMethod = "addImmOperands"; + let PredicateMethod = "isUImm3"; + let DiagnosticType = "InvalidBOSCAMETileIndex"; +} + +// AsmOperandClass for accumulator register index (0-7) +def BOSCAMEAccIndexAsmOperand : AsmOperandClass { + let Name = "BOSCAMEAccIndex"; + let RenderMethod = "addImmOperands"; + let PredicateMethod = "isUImm3"; + let DiagnosticType = "InvalidBOSCAMEAccIndex"; +} + +// Operand type for BOSCAMETileReg index (0-7), printed with "tr" prefix in AsmString +def BOSCAMETileIndex : RISCVOp { + let ParserMatchClass = BOSCAMETileIndexAsmOperand; + let DecoderMethod = "decodeUImmOperand<3>"; + let OperandType = "OPERAND_UIMM3"; +} + +// Operand type for BOSCAMEAccReg index (0-7), printed with "acc" prefix in AsmString +def BOSCAMEAccIndex : RISCVOp { + let ParserMatchClass = BOSCAMEAccIndexAsmOperand; + let DecoderMethod = "decodeUImmOperand<3>"; + let OperandType = "OPERAND_UIMM3"; +} + +//===----------------------------------------------------------------------===// +// BOSC AME 32-bit Instruction Format Base Class +//===----------------------------------------------------------------------===// + +// Base class for BOSC AME 32-bit matrix multiplication instructions +// Uses MatrixReg for inputs (ms1, ms2) and MatrixReg for output (md) +class RVInstBOSCAME32 + : RVInst { + bits<6> funct6; + bit fp; // 0 for integer, 1 for floating-point (if supported in future) + bit sa; // 0 for unsaturated, 1 for saturated + bits<4> ms2; + bit sn; // Whether the source operand is signed (for integers), 0 for floating-point + bits<4> ms1; + bits<3> eew; // 0=int8, 1=int16, 2=int32, 3=int64, 7=int4 + bit ma = 0b1; + bits<4> md; + bits<7> opcode = 0b1110111; + + let Inst{31-26} = funct6; + let Inst{25} = fp; + let Inst{24} = sa; + let Inst{23-20} = ms2; + let Inst{19} = sn; + let Inst{18-15} = ms1; + let Inst{14-12} = eew; + let Inst{11} = ma; + let Inst{10-7} = md; + let Inst{6-0} = 0b1110111; +} + +//===----------------------------------------------------------------------===// +// BOSC AME 32-bit Matrix Multiplication Instructions +//===----------------------------------------------------------------------===// +// Format: mma.{h|w|dw}.mm acc, tr, tr +// Semantics: acc = acc + tr1 * tr2 +// +// Data Flow: BOSCAMETileReg × BOSCAMETileReg → BOSCAMEAccReg (accumulate) +// - ms1: BOSCAMETileReg for matrix A +// - ms2: BOSCAMETileReg for matrix B +// - md: BOSCAMEAccReg for accumulation result C +// +// eew encoding: +// 000 = int8 (b), 001 = int16 (h), 010 = int32 (w), 011 = int64 (dw) +// 100 = use mtype.msew, 111 = int4 +// +// funct6 encoding: +// 001000 = no-widen +// 001001 = double-widen +// 001010 = quad-widen +// 001011 = oct-widen +//===----------------------------------------------------------------------===// + +// No-widen matrix multiply-accumulate: acc = acc + tr1 * tr2 +// Input and output have the same element width +class BOSC_AME_MMA_NO_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$md_in, BOSCAMETileReg:$ms1, BOSCAMETileReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let Constraints = "$md = $md_in"; // Accumulator constraint + let funct6 = 0b001000; // No widening + let fp = fp_val; + let sa = sa_val; // Matrix multiplication + let sn = sn_val; // (signed for integers, 0 for floating-point) + let eew = eew_val; + let ma = 0b1; // Accumulate + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // int 16 32 64 , output no-widen + def BOSC_AME_MMA_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b0, 0b1, 0b100, "mma.mm">; + def BOSC_AME_MMA_H_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b0, 0b1, 0b001, "mma.h.mm">; + def BOSC_AME_MMA_W_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b0, 0b1, 0b010, "mma.w.mm">; + def BOSC_AME_MMA_DW_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b0, 0b1, 0b011, "mma.dw.mm">; + + // int 16 32 64 , output no-widen and saturated + def BOSC_AME_MSMA_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b1, 0b1, 0b100, "msma.mm">; + def BOSC_AME_MSMA_H_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b1, 0b1, 0b001, "msma.h.mm">; + def BOSC_AME_MSMA_W_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b1, 0b1, 0b010, "msma.w.mm">; + def BOSC_AME_MSMA_DW_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b1, 0b1, 0b011, "msma.dw.mm">; + + // uint 16 32 64 , output no-widen + def BOSC_AME_MMAU_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b0, 0b0, 0b100, "mmau.mm">; + def BOSC_AME_MMAU_H_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b0, 0b0, 0b001, "mmau.h.mm">; + def BOSC_AME_MMAU_W_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b0, 0b0, 0b010, "mmau.w.mm">; + def BOSC_AME_MMAU_DW_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b0, 0b0, 0b011, "mmau.dw.mm">; + + // uint 16 32 64 , output no-widen and saturated + def BOSC_AME_MSMAU_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b1, 0b0, 0b100, "msmau.mm">; + def BOSC_AME_MSMAU_H_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b1, 0b0, 0b001, "msmau.h.mm">; + def BOSC_AME_MSMAU_W_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b1, 0b0, 0b010, "msmau.w.mm">; + def BOSC_AME_MSMAU_DW_MM : BOSC_AME_MMA_NO_WIDEN<0b0, 0b1, 0b0, 0b011, "msmau.dw.mm">; + + // float point 16 32 64 , output no-widen + def BOSC_AME_MFMA_MM : BOSC_AME_MMA_NO_WIDEN<0b1, 0b0, 0b0, 0b100, "mfma.mm">; + def BOSC_AME_MFMA_HF_MM : BOSC_AME_MMA_NO_WIDEN<0b1, 0b0, 0b0, 0b001, "mfma.hf.mm">; + def BOSC_AME_MFMA_F_MM : BOSC_AME_MMA_NO_WIDEN<0b1, 0b0, 0b0, 0b010, "mfma.f.mm">; + def BOSC_AME_MFMA_D_MM : BOSC_AME_MMA_NO_WIDEN<0b1, 0b0, 0b0, 0b011, "mfma.d.mm">; +} + +// double-widen matrix multiply-accumulate: acc = acc + tr1 * tr2 +class BOSC_AME_MMA_DOUBLE_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$md_in, BOSCAMETileReg:$ms1, BOSCAMETileReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let Constraints = "$md = $md_in"; + let funct6 = 0b001001; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b1; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // uint 16→32, 32→64 with double widening + def BOSC_AME_MWMAU_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b0, 0b0, 0b100, "mwmau.mm">; + def BOSC_AME_MWMAU_H_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b0, 0b0, 0b001, "mwmau.h.mm">; + def BOSC_AME_MWMAU_W_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b0, 0b0, 0b010, "mwmau.w.mm">; + + // uint 16→32, 32→64 with double widening and saturation + def BOSC_AME_MSWMAU_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b1, 0b0, 0b100, "mswmau.mm">; + def BOSC_AME_MSWMAU_H_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b1, 0b0, 0b001, "mswmau.h.mm">; + def BOSC_AME_MSWMAU_W_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b1, 0b0, 0b010, "mswmau.w.mm">; + + // int 16→32, 32→64 with double widening + def BOSC_AME_MWMA_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b0, 0b1, 0b100, "mwma.mm">; + def BOSC_AME_MWMA_H_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b0, 0b1, 0b001, "mwma.h.mm">; + def BOSC_AME_MWMA_W_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b0, 0b1, 0b010, "mwma.w.mm">; + + // int 16→32, 32→64 with double widening and saturation + def BOSC_AME_MSWMA_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b1, 0b1, 0b100, "mswma.mm">; + def BOSC_AME_MSWMA_H_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b1, 0b1, 0b001, "mswma.h.mm">; + def BOSC_AME_MSWMA_W_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b0, 0b1, 0b1, 0b010, "mswma.w.mm">; + + // float point 8→16, 16→32, 32→64 with double widening + def BOSC_AME_MFWMA_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b1, 0b0, 0b0, 0b100, "mfwma.mm">; + def BOSC_AME_MFWMA_CF_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b1, 0b0, 0b0, 0b000, "mfwma.cf.mm">; + def BOSC_AME_MFWMA_HF_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b1, 0b0, 0b0, 0b001, "mfwma.hf.mm">; + def BOSC_AME_MFWMA_F_MM : BOSC_AME_MMA_DOUBLE_WIDEN<0b1, 0b0, 0b0, 0b010, "mfwma.f.mm">; +} + +// quad-widen matrix multiply-accumulate: acc = acc + tr1 * tr2 +class BOSC_AME_MMA_QUAD_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$md_in, BOSCAMETileReg:$ms1, BOSCAMETileReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let Constraints = "$md = $md_in"; + let funct6 = 0b001010; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b1; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // uint 8→32 with quad widening + def BOSC_AME_MQMAU_MM : BOSC_AME_MMA_QUAD_WIDEN<0b0, 0b0, 0b0, 0b100, "mqmau.mm">; + def BOSC_AME_MQMAU_B_MM : BOSC_AME_MMA_QUAD_WIDEN<0b0, 0b0, 0b0, 0b000, "mqmau.b.mm">; + // uint 8→32 with quad widening and saturation + def BOSC_AME_MSQMAU_MM : BOSC_AME_MMA_QUAD_WIDEN<0b0, 0b1, 0b0, 0b100, "msqmau.mm">; + def BOSC_AME_MSQMAU_B_MM : BOSC_AME_MMA_QUAD_WIDEN<0b0, 0b1, 0b0, 0b000, "msqmau.b.mm">; + // int 8→32 with quad widening + def BOSC_AME_MQMA_MM : BOSC_AME_MMA_QUAD_WIDEN<0b0, 0b0, 0b1, 0b100, "mqma.mm">; + def BOSC_AME_MQMA_B_MM : BOSC_AME_MMA_QUAD_WIDEN<0b0, 0b0, 0b1, 0b000, "mqma.b.mm">; + // int 8→32 with quad widening and saturation + def BOSC_AME_MSQMA_MM : BOSC_AME_MMA_QUAD_WIDEN<0b0, 0b1, 0b1, 0b100, "msqma.mm">; + def BOSC_AME_MSQMA_B_MM : BOSC_AME_MMA_QUAD_WIDEN<0b0, 0b1, 0b1, 0b000, "msqma.b.mm">; + // float point 8→32 with quad widening + def BOSC_AME_MFQMA_MM : BOSC_AME_MMA_QUAD_WIDEN<0b1, 0b0, 0b1, 0b100, "mfqma.mm">; + def BOSC_AME_MFQMA_CF_MM : BOSC_AME_MMA_QUAD_WIDEN<0b1, 0b0, 0b1, 0b000, "mfqma.cf.mm">; +} + +// oct-widen matrix multiply-accumulate: acc = acc + tr1 * tr2 +class BOSC_AME_MMA_OCT_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$md_in, BOSCAMETileReg:$ms1, BOSCAMETileReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let Constraints = "$md = $md_in"; + let funct6 = 0b001011; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b1; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // uint 4→32 with oct widening + def BOSC_AME_MOMAU_MM : BOSC_AME_MMA_OCT_WIDEN<0b0, 0b0, 0b0, 0b100, "momau.mm">; + def BOSC_AME_MOMAU_HB_MM : BOSC_AME_MMA_OCT_WIDEN<0b0, 0b0, 0b0, 0b111, "momau.hb.mm">; + // uint 4→32 with oct widening and saturation + def BOSC_AME_MSOMAU_MM : BOSC_AME_MMA_OCT_WIDEN<0b0, 0b1, 0b0, 0b100, "msomau.mm">; + def BOSC_AME_MSOMAU_HB_MM : BOSC_AME_MMA_OCT_WIDEN<0b0, 0b1, 0b0, 0b111, "msomau.hb.mm">; + // int 4→32 with oct widening + def BOSC_AME_MOMA_MM : BOSC_AME_MMA_OCT_WIDEN<0b0, 0b0, 0b1, 0b100, "moma.mm">; + def BOSC_AME_MOMA_HB_MM : BOSC_AME_MMA_OCT_WIDEN<0b0, 0b0, 0b1, 0b111, "moma.hb.mm">; + // int 4→32 with oct widening and saturation + def BOSC_AME_MSOMA_MM : BOSC_AME_MMA_OCT_WIDEN<0b0, 0b1, 0b1, 0b100, "msoma.mm">; + def BOSC_AME_MSOMA_HB_MM : BOSC_AME_MMA_OCT_WIDEN<0b0, 0b1, 0b1, 0b111, "msoma.hb.mm">; +} + +//===----------------------------------------------------------------------===// +// BOSC AME Element-Wise Instructions +//===----------------------------------------------------------------------===// +// Matrix element-wise add/sub/multiply instructions. +class BOSC_AME_MADD_NO_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b001000; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // uint 4 8 16 32 64 , output no-widen + def BOSC_AME_MADDU_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b0, 0b100, "maddu.mm">; + def BOSC_AME_MADDU_HB_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b0, 0b111, "maddu.hb.mm">; + def BOSC_AME_MADDU_B_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b0, 0b000, "maddu.b.mm">; + def BOSC_AME_MADDU_H_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b0, 0b001, "maddu.h.mm">; + def BOSC_AME_MADDU_W_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b0, 0b010, "maddu.w.mm">; + def BOSC_AME_MADDU_DW_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b0, 0b011, "maddu.dw.mm">; + // uint 4 8 16 32 64 , output no-widen and saturated + def BOSC_AME_MSADDU_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b0, 0b100, "msaddu.mm">; + def BOSC_AME_MSADDU_HB_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b0, 0b111, "msaddu.hb.mm">; + def BOSC_AME_MSADDU_B_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b0, 0b000, "msaddu.b.mm">; + def BOSC_AME_MSADDU_H_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b0, 0b001, "msaddu.h.mm">; + def BOSC_AME_MSADDU_W_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b0, 0b010, "msaddu.w.mm">; + def BOSC_AME_MSADDU_DW_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b0, 0b011, "msaddu.dw.mm">; + // int 4 8 16 32 64 , output no-widen + def BOSC_AME_MADD_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b1, 0b100, "madd.mm">; + def BOSC_AME_MADD_HB_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b1, 0b111, "madd.hb.mm">; + def BOSC_AME_MADD_B_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b1, 0b000, "madd.b.mm">; + def BOSC_AME_MADD_H_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b1, 0b001, "madd.h.mm">; + def BOSC_AME_MADD_W_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b1, 0b010, "madd.w.mm">; + def BOSC_AME_MADD_DW_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b0, 0b1, 0b011, "madd.dw.mm">; + // int 4 8 16 32 64 , output no-widen and saturated + def BOSC_AME_MSADD_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b1, 0b100, "msadd.mm">; + def BOSC_AME_MSADD_HB_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b1, 0b111, "msadd.hb.mm">; + def BOSC_AME_MSADD_B_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b1, 0b000, "msadd.b.mm">; + def BOSC_AME_MSADD_H_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b1, 0b001, "msadd.h.mm">; + def BOSC_AME_MSADD_W_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b1, 0b010, "msadd.w.mm">; + def BOSC_AME_MSADD_DW_MM : BOSC_AME_MADD_NO_WIDEN<0b0, 0b1, 0b1, 0b011, "msadd.dw.mm">; + // float point 8 16 32 64 , output no-widen + def BOSC_AME_MFADD_MM : BOSC_AME_MADD_NO_WIDEN<0b1, 0b0, 0b1, 0b100, "mfadd.mm">; + def BOSC_AME_MFADD_CF_MM : BOSC_AME_MADD_NO_WIDEN<0b1, 0b0, 0b1, 0b000, "mfadd.cf.mm">; + def BOSC_AME_MFADD_HF_MM : BOSC_AME_MADD_NO_WIDEN<0b1, 0b0, 0b1, 0b001, "mfadd.hf.mm">; + def BOSC_AME_MFADD_F_MM : BOSC_AME_MADD_NO_WIDEN<0b1, 0b0, 0b1, 0b010, "mfadd.f.mm">; + def BOSC_AME_MFADD_D_MM : BOSC_AME_MADD_NO_WIDEN<0b1, 0b0, 0b1, 0b011, "mfadd.d.mm">; +} + +class BOSC_AME_MADD_DOUBLE_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b001001; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // uint 4 8 16 32 , output double-widen + def BOSC_AME_MWADDU_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0, 0b100, "mwaddu.mm">; + def BOSC_AME_MWADDU_HB_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0, 0b111, "mwaddu.hb.mm">; + def BOSC_AME_MWADDU_B_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0, 0b000, "mwaddu.b.mm">; + def BOSC_AME_MWADDU_H_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0, 0b001, "mwaddu.h.mm">; + def BOSC_AME_MWADDU_W_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0, 0b010, "mwaddu.w.mm">; + // int 4 8 16 32 , output double-widen + def BOSC_AME_MWADD_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0b1, 0b100, "mwadd.mm">; + def BOSC_AME_MWADD_HB_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0b1, 0b111, "mwadd.hb.mm">; + def BOSC_AME_MWADD_B_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0b1, 0b000, "mwadd.b.mm">; + def BOSC_AME_MWADD_H_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0b1, 0b001, "mwadd.h.mm">; + def BOSC_AME_MWADD_W_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b0, 0, 0b1, 0b010, "mwadd.w.mm">; + // float point 8 16 32 , output double-widen + def BOSC_AME_MFWADD_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b1, 0b0, 0b1, 0b100, "mfwadd.mm">; + def BOSC_AME_MFWADD_CF_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b1, 0b0, 0b1, 0b000, "mfwadd.cf.mm">; + def BOSC_AME_MFWADD_HF_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b1, 0b0, 0b1, 0b001, "mfwadd.hf.mm">; + def BOSC_AME_MFWADD_F_MM : BOSC_AME_MADD_DOUBLE_WIDEN<0b1, 0b0, 0b1, 0b010, "mfwadd.f.mm">; +} + +class BOSC_AME_MSUB_NO_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b001010; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MSUBU_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 0, 0b100, "msubu.mm">; + def BOSC_AME_MSUBU_HB_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 0, 0b111, "msubu.hb.mm">; + def BOSC_AME_MSUBU_B_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 0, 0b000, "msubu.b.mm">; + def BOSC_AME_MSUBU_H_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 0, 0b001, "msubu.h.mm">; + def BOSC_AME_MSUBU_W_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 0, 0b010, "msubu.w.mm">; + def BOSC_AME_MSUBU_DW_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 0, 0b011, "msubu.dw.mm">; + + def BOSC_AME_MSSUBU_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 0, 0b100, "mssubu.mm">; + def BOSC_AME_MSSUBU_HB_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 0, 0b111, "mssubu.hb.mm">; + def BOSC_AME_MSSUBU_B_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 0, 0b000, "mssubu.b.mm">; + def BOSC_AME_MSSUBU_H_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 0, 0b001, "mssubu.h.mm">; + def BOSC_AME_MSSUBU_W_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 0, 0b010, "mssubu.w.mm">; + def BOSC_AME_MSSUBU_DW_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 0, 0b011, "mssubu.dw.mm">; + + def BOSC_AME_MSUB_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 1, 0b100, "msub.mm">; + def BOSC_AME_MSUB_HB_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 1, 0b111, "msub.hb.mm">; + def BOSC_AME_MSUB_B_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 1, 0b000, "msub.b.mm">; + def BOSC_AME_MSUB_H_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 1, 0b001, "msub.h.mm">; + def BOSC_AME_MSUB_W_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 1, 0b010, "msub.w.mm">; + def BOSC_AME_MSUB_DW_MM : BOSC_AME_MSUB_NO_WIDEN<0, 0, 1, 0b011, "msub.dw.mm">; + + def BOSC_AME_MSSUB_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 1, 0b100, "mssub.mm">; + def BOSC_AME_MSSUB_HB_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 1, 0b111, "mssub.hb.mm">; + def BOSC_AME_MSSUB_B_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 1, 0b000, "mssub.b.mm">; + def BOSC_AME_MSSUB_H_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 1, 0b001, "mssub.h.mm">; + def BOSC_AME_MSSUB_W_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 1, 0b010, "mssub.w.mm">; + def BOSC_AME_MSSUB_DW_MM : BOSC_AME_MSUB_NO_WIDEN<0, 1, 1, 0b011, "mssub.dw.mm">; + + def BOSC_AME_MFSUB_MM : BOSC_AME_MSUB_NO_WIDEN<1, 0, 1, 0b100, "mfsub.mm">; + def BOSC_AME_MFSUB_CF_MM : BOSC_AME_MSUB_NO_WIDEN<1, 0, 1, 0b000, "mfsub.cf.mm">; + def BOSC_AME_MFSUB_HF_MM : BOSC_AME_MSUB_NO_WIDEN<1, 0, 1, 0b001, "mfsub.hf.mm">; + def BOSC_AME_MFSUB_F_MM : BOSC_AME_MSUB_NO_WIDEN<1, 0, 1, 0b010, "mfsub.f.mm">; + def BOSC_AME_MFSUB_D_MM : BOSC_AME_MSUB_NO_WIDEN<1, 0, 1, 0b011, "mfsub.d.mm">; +} + +class BOSC_AME_MSUB_DOUBLE_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b001011; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MWSUBU_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 0, 0b100, "mwsubu.mm">; + def BOSC_AME_MWSUBU_HB_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 0, 0b111, "mwsubu.hb.mm">; + def BOSC_AME_MWSUBU_B_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 0, 0b000, "mwsubu.b.mm">; + def BOSC_AME_MWSUBU_H_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 0, 0b001, "mwsubu.h.mm">; + def BOSC_AME_MWSUBU_W_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 0, 0b010, "mwsubu.w.mm">; + + def BOSC_AME_MWSUB_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 1, 0b100, "mwsub.mm">; + def BOSC_AME_MWSUB_HB_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 1, 0b111, "mwsub.hb.mm">; + def BOSC_AME_MWSUB_B_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 1, 0b000, "mwsub.b.mm">; + def BOSC_AME_MWSUB_H_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 1, 0b001, "mwsub.h.mm">; + def BOSC_AME_MWSUB_W_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<0, 0, 1, 0b010, "mwsub.w.mm">; + + def BOSC_AME_MFWSUB_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<1, 0, 1, 0b100, "mfwsub.mm">; + def BOSC_AME_MFWSUB_CF_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<1, 0, 1, 0b000, "mfwsub.cf.mm">; + def BOSC_AME_MFWSUB_HF_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<1, 0, 1, 0b001, "mfwsub.hf.mm">; + def BOSC_AME_MFWSUB_F_MM : BOSC_AME_MSUB_DOUBLE_WIDEN<1, 0, 1, 0b010, "mfwsub.f.mm">; +} + +class BOSC_AME_MMINMAX_NO_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b001100; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MMINU_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 0, 0b100, "mminu.mm">; + def BOSC_AME_MMINU_HB_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 0, 0b111, "mminu.hb.mm">; + def BOSC_AME_MMINU_B_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 0, 0b000, "mminu.b.mm">; + def BOSC_AME_MMINU_H_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 0, 0b001, "mminu.h.mm">; + def BOSC_AME_MMINU_W_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 0, 0b010, "mminu.w.mm">; + def BOSC_AME_MMINU_DW_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 0, 0b011, "mminu.dw.mm">; + + def BOSC_AME_MMAXU_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 0, 0b100, "mmaxu.mm">; + def BOSC_AME_MMAXU_HB_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 0, 0b111, "mmaxu.hb.mm">; + def BOSC_AME_MMAXU_B_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 0, 0b000, "mmaxu.b.mm">; + def BOSC_AME_MMAXU_H_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 0, 0b001, "mmaxu.h.mm">; + def BOSC_AME_MMAXU_W_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 0, 0b010, "mmaxu.w.mm">; + def BOSC_AME_MMAXU_DW_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 0, 0b011, "mmaxu.dw.mm">; + + def BOSC_AME_MMIN_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 1, 0b100, "mmin.mm">; + def BOSC_AME_MMIN_HB_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 1, 0b111, "mmin.hb.mm">; + def BOSC_AME_MMIN_B_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 1, 0b000, "mmin.b.mm">; + def BOSC_AME_MMIN_H_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 1, 0b001, "mmin.h.mm">; + def BOSC_AME_MMIN_W_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 1, 0b010, "mmin.w.mm">; + def BOSC_AME_MMIN_DW_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 0, 1, 0b011, "mmin.dw.mm">; + + def BOSC_AME_MMAX_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 1, 0b100, "mmax.mm">; + def BOSC_AME_MMAX_HB_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 1, 0b111, "mmax.hb.mm">; + def BOSC_AME_MMAX_B_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 1, 0b000, "mmax.b.mm">; + def BOSC_AME_MMAX_H_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 1, 0b001, "mmax.h.mm">; + def BOSC_AME_MMAX_W_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 1, 0b010, "mmax.w.mm">; + def BOSC_AME_MMAX_DW_MM : BOSC_AME_MMINMAX_NO_WIDEN<0, 1, 1, 0b011, "mmax.dw.mm">; + + def BOSC_AME_MFMIN_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 0, 1, 0b100, "mfmin.mm">; + def BOSC_AME_MFMIN_CF_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 0, 1, 0b000, "mfmin.cf.mm">; + def BOSC_AME_MFMIN_HF_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 0, 1, 0b001, "mfmin.hf.mm">; + def BOSC_AME_MFMIN_F_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 0, 1, 0b010, "mfmin.f.mm">; + def BOSC_AME_MFMIN_D_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 0, 1, 0b011, "mfmin.d.mm">; + + def BOSC_AME_MFMAX_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 1, 1, 0b100, "mfmax.mm">; + def BOSC_AME_MFMAX_CF_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 1, 1, 0b000, "mfmax.cf.mm">; + def BOSC_AME_MFMAX_HF_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 1, 1, 0b001, "mfmax.hf.mm">; + def BOSC_AME_MFMAX_F_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 1, 1, 0b010, "mfmax.f.mm">; + def BOSC_AME_MFMAX_D_MM : BOSC_AME_MMINMAX_NO_WIDEN<1, 1, 1, 0b011, "mfmax.d.mm">; +} + +class BOSC_AME_MMUL_NO_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b001101; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MSMULU_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 0, 0b100, "msmulu.mm">; + def BOSC_AME_MSMULU_HB_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 0, 0b111, "msmulu.hb.mm">; + def BOSC_AME_MSMULU_B_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 0, 0b000, "msmulu.b.mm">; + def BOSC_AME_MSMULU_H_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 0, 0b001, "msmulu.h.mm">; + def BOSC_AME_MSMULU_W_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 0, 0b010, "msmulu.w.mm">; + def BOSC_AME_MSMULU_DW_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 0, 0b011, "msmulu.dw.mm">; + + def BOSC_AME_MMUL_MM : BOSC_AME_MMUL_NO_WIDEN<0, 0, 1, 0b100, "mmul.mm">; + def BOSC_AME_MMUL_HB_MM : BOSC_AME_MMUL_NO_WIDEN<0, 0, 1, 0b111, "mmul.hb.mm">; + def BOSC_AME_MMUL_B_MM : BOSC_AME_MMUL_NO_WIDEN<0, 0, 1, 0b000, "mmul.b.mm">; + def BOSC_AME_MMUL_H_MM : BOSC_AME_MMUL_NO_WIDEN<0, 0, 1, 0b001, "mmul.h.mm">; + def BOSC_AME_MMUL_W_MM : BOSC_AME_MMUL_NO_WIDEN<0, 0, 1, 0b010, "mmul.w.mm">; + def BOSC_AME_MMUL_DW_MM : BOSC_AME_MMUL_NO_WIDEN<0, 0, 1, 0b011, "mmul.dw.mm">; + + def BOSC_AME_MSMUL_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 1, 0b100, "msmul.mm">; + def BOSC_AME_MSMUL_HB_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 1, 0b111, "msmul.hb.mm">; + def BOSC_AME_MSMUL_B_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 1, 0b000, "msmul.b.mm">; + def BOSC_AME_MSMUL_H_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 1, 0b001, "msmul.h.mm">; + def BOSC_AME_MSMUL_W_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 1, 0b010, "msmul.w.mm">; + def BOSC_AME_MSMUL_DW_MM : BOSC_AME_MMUL_NO_WIDEN<0, 1, 1, 0b011, "msmul.dw.mm">; + + def BOSC_AME_MFMUL_MM : BOSC_AME_MMUL_NO_WIDEN<1, 0, 1, 0b100, "mfmul.mm">; + def BOSC_AME_MFMUL_CF_MM : BOSC_AME_MMUL_NO_WIDEN<1, 0, 1, 0b000, "mfmul.cf.mm">; + def BOSC_AME_MFMUL_HF_MM : BOSC_AME_MMUL_NO_WIDEN<1, 0, 1, 0b001, "mfmul.hf.mm">; + def BOSC_AME_MFMUL_F_MM : BOSC_AME_MMUL_NO_WIDEN<1, 0, 1, 0b010, "mfmul.f.mm">; + def BOSC_AME_MFMUL_D_MM : BOSC_AME_MMUL_NO_WIDEN<1, 0, 1, 0b011, "mfmul.d.mm">; +} + +class BOSC_AME_MMULH_DIV_NO_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b001110; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MMULHU_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 0, 0b100, "mmulhu.mm">; + def BOSC_AME_MMULHU_HB_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 0, 0b111, "mmulhu.hb.mm">; + def BOSC_AME_MMULHU_B_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 0, 0b000, "mmulhu.b.mm">; + def BOSC_AME_MMULHU_H_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 0, 0b001, "mmulhu.h.mm">; + def BOSC_AME_MMULHU_W_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 0, 0b010, "mmulhu.w.mm">; + def BOSC_AME_MMULHU_DW_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 0, 0b011, "mmulhu.dw.mm">; + + def BOSC_AME_MMULH_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 1, 0b100, "mmulh.mm">; + def BOSC_AME_MMULH_HB_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 1, 0b111, "mmulh.hb.mm">; + def BOSC_AME_MMULH_B_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 1, 0b000, "mmulh.b.mm">; + def BOSC_AME_MMULH_H_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 1, 0b001, "mmulh.h.mm">; + def BOSC_AME_MMULH_W_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 1, 0b010, "mmulh.w.mm">; + def BOSC_AME_MMULH_DW_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 0, 1, 0b011, "mmulh.dw.mm">; + + def BOSC_AME_MMULHSU_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 0, 0b100, "mmulhsu.mm">; + def BOSC_AME_MMULHSU_HB_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 0, 0b111, "mmulhsu.hb.mm">; + def BOSC_AME_MMULHSU_B_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 0, 0b000, "mmulhsu.b.mm">; + def BOSC_AME_MMULHSU_H_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 0, 0b001, "mmulhsu.h.mm">; + def BOSC_AME_MMULHSU_W_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 0, 0b010, "mmulhsu.w.mm">; + def BOSC_AME_MMULHSU_DW_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 0, 0b011, "mmulhsu.dw.mm">; + + def BOSC_AME_MSMULSU_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 1, 0b100, "msmulsu.mm">; + def BOSC_AME_MSMULSU_HB_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 1, 0b111, "msmulsu.hb.mm">; + def BOSC_AME_MSMULSU_B_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 1, 0b000, "msmulsu.b.mm">; + def BOSC_AME_MSMULSU_H_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 1, 0b001, "msmulsu.h.mm">; + def BOSC_AME_MSMULSU_W_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 1, 0b010, "msmulsu.w.mm">; + def BOSC_AME_MSMULSU_DW_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<0, 1, 1, 0b011, "msmulsu.dw.mm">; + + def BOSC_AME_MFDIV_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<1, 0, 1, 0b100, "mfdiv.mm">; + def BOSC_AME_MFDIV_CF_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<1, 0, 1, 0b000, "mfdiv.cf.mm">; + def BOSC_AME_MFDIV_HF_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<1, 0, 1, 0b001, "mfdiv.hf.mm">; + def BOSC_AME_MFDIV_F_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<1, 0, 1, 0b010, "mfdiv.f.mm">; + def BOSC_AME_MFDIV_D_MM : BOSC_AME_MMULH_DIV_NO_WIDEN<1, 0, 1, 0b011, "mfdiv.d.mm">; +} + +class BOSC_AME_MMUL_DOUBLE_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b001111; + let fp = fp_val; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MWMULU_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 0, 0b100, "mwmulu.mm">; + def BOSC_AME_MWMULU_HB_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 0, 0b111, "mwmulu.hb.mm">; + def BOSC_AME_MWMULU_B_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 0, 0b000, "mwmulu.b.mm">; + def BOSC_AME_MWMULU_H_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 0, 0b001, "mwmulu.h.mm">; + def BOSC_AME_MWMULU_W_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 0, 0b010, "mwmulu.w.mm">; + + def BOSC_AME_MWMUL_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 1, 0b100, "mwmul.mm">; + def BOSC_AME_MWMUL_HB_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 1, 0b111, "mwmul.hb.mm">; + def BOSC_AME_MWMUL_B_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 1, 0b000, "mwmul.b.mm">; + def BOSC_AME_MWMUL_H_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 1, 0b001, "mwmul.h.mm">; + def BOSC_AME_MWMUL_W_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 0, 1, 0b010, "mwmul.w.mm">; + + def BOSC_AME_MWMULSU_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 1, 1, 0b100, "mwmulsu.mm">; + def BOSC_AME_MWMULSU_HB_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 1, 1, 0b111, "mwmulsu.hb.mm">; + def BOSC_AME_MWMULSU_B_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 1, 1, 0b000, "mwmulsu.b.mm">; + def BOSC_AME_MWMULSU_H_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 1, 1, 0b001, "mwmulsu.h.mm">; + def BOSC_AME_MWMULSU_W_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<0, 1, 1, 0b010, "mwmulsu.w.mm">; + + def BOSC_AME_MFWMUL_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<1, 0, 1, 0b100, "mfwmul.mm">; + def BOSC_AME_MFWMUL_CF_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<1, 0, 1, 0b000, "mfwmul.cf.mm">; + def BOSC_AME_MFWMUL_HF_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<1, 0, 1, 0b001, "mfwmul.hf.mm">; + def BOSC_AME_MFWMUL_F_MM : BOSC_AME_MMUL_DOUBLE_WIDEN<1, 0, 1, 0b010, "mfwmul.f.mm">; +} + +class BOSC_AME_LOGIC_NO_WIDEN + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b010000; + let fp = 0b0; + let sa = sa_val; + let sn = sn_val; + let eew = 0b100; + let ma = 0b0; + let opcode = 0b1110111; +} + +class BOSC_AME_SQRT_NO_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1), + opcodestr, "$md, $ms1"> { + let funct6 = 0b010000; + let fp = 0b1; + let sa = 0b0; + let ms2 = 0b0000; + let sn = 0b1; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MAND_MM : BOSC_AME_LOGIC_NO_WIDEN<0, 0, "mand.mm">; + def BOSC_AME_MOR_MM : BOSC_AME_LOGIC_NO_WIDEN<1, 0, "mor.mm">; + def BOSC_AME_MXOR_MM : BOSC_AME_LOGIC_NO_WIDEN<1, 1, "mxor.mm">; + + def BOSC_AME_MFSQRT_MM : BOSC_AME_SQRT_NO_WIDEN<0b100, "mfsqrt.mm">; + def BOSC_AME_MFSQRT_CF_MM : BOSC_AME_SQRT_NO_WIDEN<0b000, "mfsqrt.cf.mm">; + def BOSC_AME_MFSQRT_HF_MM : BOSC_AME_SQRT_NO_WIDEN<0b001, "mfsqrt.hf.mm">; + def BOSC_AME_MFSQRT_F_MM : BOSC_AME_SQRT_NO_WIDEN<0b010, "mfsqrt.f.mm">; + def BOSC_AME_MFSQRT_D_MM : BOSC_AME_SQRT_NO_WIDEN<0b011, "mfsqrt.d.mm">; +} + +class BOSC_AME_SHIFT_NO_WIDEN eew_val, string opcodestr> + : RVInstBOSCAME32<(outs BOSCAMEAccReg:$md), + (ins BOSCAMEAccReg:$ms1, BOSCAMEAccReg:$ms2), + opcodestr, "$md, $ms1, $ms2"> { + let funct6 = 0b010001; + let fp = 0b0; + let sa = sa_val; + let sn = sn_val; + let eew = eew_val; + let ma = 0b0; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MSLL_MM : BOSC_AME_SHIFT_NO_WIDEN<0, 0, 0b100, "msll.mm">; + def BOSC_AME_MSLL_HB_MM : BOSC_AME_SHIFT_NO_WIDEN<0, 0, 0b111, "msll.hb.mm">; + def BOSC_AME_MSLL_B_MM : BOSC_AME_SHIFT_NO_WIDEN<0, 0, 0b000, "msll.b.mm">; + def BOSC_AME_MSLL_H_MM : BOSC_AME_SHIFT_NO_WIDEN<0, 0, 0b001, "msll.h.mm">; + def BOSC_AME_MSLL_W_MM : BOSC_AME_SHIFT_NO_WIDEN<0, 0, 0b010, "msll.w.mm">; + def BOSC_AME_MSLL_DW_MM : BOSC_AME_SHIFT_NO_WIDEN<0, 0, 0b011, "msll.dw.mm">; + + def BOSC_AME_MSRL_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 0, 0b100, "msrl.mm">; + def BOSC_AME_MSRL_HB_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 0, 0b111, "msrl.hb.mm">; + def BOSC_AME_MSRL_B_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 0, 0b000, "msrl.b.mm">; + def BOSC_AME_MSRL_H_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 0, 0b001, "msrl.h.mm">; + def BOSC_AME_MSRL_W_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 0, 0b010, "msrl.w.mm">; + def BOSC_AME_MSRL_DW_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 0, 0b011, "msrl.dw.mm">; + + def BOSC_AME_MSRA_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 1, 0b100, "msra.mm">; + def BOSC_AME_MSRA_HB_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 1, 0b111, "msra.hb.mm">; + def BOSC_AME_MSRA_B_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 1, 0b000, "msra.b.mm">; + def BOSC_AME_MSRA_H_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 1, 0b001, "msra.h.mm">; + def BOSC_AME_MSRA_W_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 1, 0b010, "msra.w.mm">; + def BOSC_AME_MSRA_DW_MM : BOSC_AME_SHIFT_NO_WIDEN<1, 1, 0b011, "msra.dw.mm">; +} + +//===----------------------------------------------------------------------===// +// BOSC AME Configuration Instructions +//===----------------------------------------------------------------------===// +// Configuration instruction format (32-bit): +// Configuration Immediate Instructions: +// | 31:26 | 25 | 24:15 | 14:12 | 11:7 | 6:0 | +// | funct6 | im | imm10 | funct3 | rd | OP-M32 | +// Configuration Instructions: +// | 31:26 | 25 | 24:20 | 19:15 | 14:12 | 11:7 | 6:0 | +// | funct6 | im | 00000 | rs1 | funct3 | rd | OP-M32 | +//===----------------------------------------------------------------------===// + +class RVInstBOSCAMEConfig32 funct6_val, bits<3> funct3_val, string opcodestr> + : RVInst<(outs GPR:$rd), (ins GPR:$rs1), + opcodestr, "$rd, $rs1", [], InstFormatOther> { + bits<5> rs1; + bits<5> rd; + + let Inst{6-0} = 0b1110111; + let Inst{11-7} = rd; + let Inst{14-12} = funct3_val; + let Inst{19-15} = rs1; + let Inst{24-20} = 0b00000; + let Inst{25} = 0b0; // 0 for register source + let Inst{31-26} = funct6_val; // To be defined in subclasses +} + +class RVInstBOSCAMEConfigImm32 funct6_val, bits<3> funct3_val, string opcodestr> + : RVInst<(outs GPR:$rd), (ins uimm32:$imm), + opcodestr, "$rd, $imm", [], InstFormatOther> { + bits<5> rd; + bits<10> imm; // 10-bit immediate + + let Inst{6-0} = 0b1110111; + let Inst{11-7} = rd; + let Inst{14-12} = funct3_val; + let Inst{24-15} = imm; + let Inst{25} = 0b1; // 1 for immediate source + let Inst{31-26} = funct6_val; // To be defined in subclasses +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 1, mayLoad = 0, mayStore = 0 in { + // msettype - set the mtype CSR based on the value in a register + def BOSC_AME_MSETTYPE : RVInstBOSCAMEConfig32<0b000000, 0b100, "msettype">; + // msettilem - set tile M dimension from register + def BOSC_AME_MSETTILEM : RVInstBOSCAMEConfig32<0b000001, 0b101, "msettilem">; + // msettilen - set tile N dimension from register + def BOSC_AME_MSETTILEN : RVInstBOSCAMEConfig32<0b000001, 0b100, "msettilen">; + // msettilek - set tile K dimension from register + def BOSC_AME_MSETTILEK : RVInstBOSCAMEConfig32<0b000001, 0b110, "msettilek">; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 1, mayLoad = 0, mayStore = 0 in { + // msettypei - set the mtype CSR based on their arguments + def BOSC_AME_MSETTYPEI : RVInstBOSCAMEConfigImm32<0b000000, 0b100, "msettypei">; + // msettypehi + def BOSC_AME_MSETTYPEHI : RVInstBOSCAMEConfigImm32<0b000000, 0b101, "msettypehi">; + // msettilemi - set tile M dimension from immediate + def BOSC_AME_MSETTILEMI : RVInstBOSCAMEConfigImm32<0b000001, 0b101, "msettilemi">; + // msettileni - set tile N dimension from immediate + def BOSC_AME_MSETTILENI : RVInstBOSCAMEConfigImm32<0b000001, 0b100, "msettileni">; + // msettileki - set tile K dimension from immediate + def BOSC_AME_MSETTILEKI : RVInstBOSCAMEConfigImm32<0b000001, 0b110, "msettileki">; +} + +//===----------------------------------------------------------------------===// +// BOSC AME Load/Store Instructions +//===----------------------------------------------------------------------===// +// Load/Store instruction format (32-bit): +// | 31:26 | 25 | 24:20 | 19:15 | 14:12 | 11 | 10:7 | 6:0 | +// | funct6 | ls | rs2 | rs1 | eew | tr | ms3/td | OP-M32 | +// +// ls: 0 for load (mlae/mlbe), 1 for store (msce) +// tr: whether the operand in register is transposed +// eew: 000=int8, 001=int16, 010=int32, 011=int64, 111=int4 +// +// Data Flow Examples: +// mlae32.m tr0, (a0), a1 # Load matrix A into BOSCAMETileReg +// mlbe32.m tr1, (a0), a1 # Load matrix B into BOSCAMETileReg +// mqma.b.mm acc0, tr0, tr1 # Compute: BOSCAMEAccReg = BOSCAMEAccReg + BOSCAMETileReg × BOSCAMETileReg +// msce32.m acc0, (a0), a1 # Store BOSCAMEAccReg to memory +//===----------------------------------------------------------------------===// + +// Load into BOSCAMETileReg (for matrix A and B) +class RVInstBOSCAMELoadTile32 funct6_val, bits<3> eew_val, bit tr_val, string opcodestr> + : RVInst<(outs BOSCAMETileReg:$md), (ins GPR:$rs1, GPR:$rs2), + opcodestr, "$md, (${rs1}), $rs2", [], InstFormatOther> { + bits<5> rs2; + bits<5> rs1; + bits<4> md; // BOSCAMETileReg index for load + + let Inst{6-0} = 0b1110111; // AME prefix + let Inst{10-7} = md; + let Inst{11} = tr_val; + let Inst{14-12} = eew_val; + let Inst{19-15} = rs1; + let Inst{24-20} = rs2; + let Inst{25} = 0; // ls = 0 for load + let Inst{31-26} = funct6_val; // funct6 +} + +// Load into BOSCAMEAccReg (for accumulator C) +class RVInstBOSCAMELoadAcc32 funct6_val, bits<3> eew_val, bit tr_val, string opcodestr> + : RVInst<(outs BOSCAMEAccReg:$md), (ins GPR:$rs1, GPR:$rs2), + opcodestr, "$md, (${rs1}), $rs2", [], InstFormatOther> { + bits<5> rs2; + bits<5> rs1; + bits<4> md; // BOSCAMEAccReg index for load + + let Inst{6-0} = 0b1110111; // AME prefix + let Inst{10-7} = md; + let Inst{11} = tr_val; + let Inst{14-12} = eew_val; + let Inst{19-15} = rs1; + let Inst{24-20} = rs2; + let Inst{25} = 0; // ls = 0 for load + let Inst{31-26} = funct6_val; // funct6 +} + +// Store from BOSCAMETileReg (for matrix A and B) +class RVInstBOSCAMEStoreTile32 funct6_val, bits<3> eew_val, bit tr_val, string opcodestr> + : RVInst<(outs), (ins BOSCAMETileReg:$ms3, GPR:$rs1, GPR:$rs2), + opcodestr, "$ms3, (${rs1}), $rs2", [], InstFormatOther> { + bits<5> rs2; + bits<5> rs1; + bits<4> ms3; // BOSCAMETileReg index for store + + let Inst{6-0} = 0b1110111; // AME prefix + let Inst{10-7} = ms3; + let Inst{11} = tr_val; + let Inst{14-12} = eew_val; + let Inst{19-15} = rs1; + let Inst{24-20} = rs2; + let Inst{25} = 1; // ls = 1 for store + let Inst{31-26} = funct6_val; // funct6 +} + +// Store from BOSCAMEAccReg (for accumulator C) +class RVInstBOSCAMEStoreAcc32 funct6_val, bits<3> eew_val, bit tr_val, string opcodestr> + : RVInst<(outs), (ins BOSCAMEAccReg:$ms3, GPR:$rs1, GPR:$rs2), + opcodestr, "$ms3, (${rs1}), $rs2", [], InstFormatOther> { + bits<5> rs2; + bits<5> rs1; + bits<4> ms3; // BOSCAMEAccReg index for store + + let Inst{6-0} = 0b1110111; // AME prefix + let Inst{10-7} = ms3; + let Inst{11} = tr_val; + let Inst{14-12} = eew_val; + let Inst{19-15} = rs1; + let Inst{24-20} = rs2; + let Inst{25} = 1; // ls = 1 for store + let Inst{31-26} = funct6_val; // funct6 +} + +//===----------------------------------------------------------------------===// +// Load matrix A into BOSCAMETileReg - mlae*.m +// Syntax: mlae{8|16|32|64}.m md, (rs1), rs2 +// md: Destination BOSCAMETileReg +// rs1: Base address (GPR) +// rs2: Row stride in bytes (GPR) +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 1, mayStore = 0 in { + def BOSC_AME_MLAE8_M : RVInstBOSCAMELoadTile32<0b000001, 0b000, 0b0, "mlae8.m">; + def BOSC_AME_MLAE16_M : RVInstBOSCAMELoadTile32<0b000001, 0b001, 0b0, "mlae16.m">; + def BOSC_AME_MLAE32_M : RVInstBOSCAMELoadTile32<0b000001, 0b010, 0b0, "mlae32.m">; + def BOSC_AME_MLAE64_M : RVInstBOSCAMELoadTile32<0b000001, 0b011, 0b0, "mlae64.m">; + + def BOSC_AME_MLATE8_M : RVInstBOSCAMELoadTile32<0b000001, 0b000, 0b1, "mlate8.m">; + def BOSC_AME_MLATE16_M : RVInstBOSCAMELoadTile32<0b000001, 0b001, 0b1, "mlate16.m">; + def BOSC_AME_MLATE32_M : RVInstBOSCAMELoadTile32<0b000001, 0b010, 0b1, "mlate32.m">; + def BOSC_AME_MLATE64_M : RVInstBOSCAMELoadTile32<0b000001, 0b011, 0b1, "mlate64.m">; + + def BOSC_AME_MLTRE8_M : RVInstBOSCAMELoadTile32<0b000011, 0b000, 0b0, "mltre8.m">; + def BOSC_AME_MLTRE16_M : RVInstBOSCAMELoadTile32<0b000011, 0b001, 0b0, "mltre16.m">; + def BOSC_AME_MLTRE32_M : RVInstBOSCAMELoadTile32<0b000011, 0b010, 0b0, "mltre32.m">; + def BOSC_AME_MLTRE64_M : RVInstBOSCAMELoadTile32<0b000011, 0b011, 0b0, "mltre64.m">; +} + +//===----------------------------------------------------------------------===// +// Load matrix B into BOSCAMETileReg - mlbe*.m +// Syntax: mlbe{8|16|32|64}.m md, (rs1), rs2 +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 1, mayStore = 0 in { + def BOSC_AME_MLBE8_M : RVInstBOSCAMELoadTile32<0b000010, 0b000, 0b0, "mlbe8.m">; + def BOSC_AME_MLBE16_M : RVInstBOSCAMELoadTile32<0b000010, 0b001, 0b0, "mlbe16.m">; + def BOSC_AME_MLBE32_M : RVInstBOSCAMELoadTile32<0b000010, 0b010, 0b0, "mlbe32.m">; + def BOSC_AME_MLBE64_M : RVInstBOSCAMELoadTile32<0b000010, 0b011, 0b0, "mlbe64.m">; + + def BOSC_AME_MLBTE8_M : RVInstBOSCAMELoadTile32<0b000010, 0b000, 0b1, "mlbte8.m">; + def BOSC_AME_MLBTE16_M : RVInstBOSCAMELoadTile32<0b000010, 0b001, 0b1, "mlbte16.m">; + def BOSC_AME_MLBTE32_M : RVInstBOSCAMELoadTile32<0b000010, 0b010, 0b1, "mlbte32.m">; + def BOSC_AME_MLBTE64_M : RVInstBOSCAMELoadTile32<0b000010, 0b011, 0b1, "mlbte64.m">; +} + +//===----------------------------------------------------------------------===// +// Load matrix C (accumulator) into BOSCAMEAccReg - mlce*.m +// Syntax: mlce{8|16|32|64}.m md, (rs1), rs2 +// md: Destination BOSCAMEAccReg (MLEN×AMUL bits) +// Note: eew here refers to the output element width after widening +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 1, mayStore = 0 in { + def BOSC_AME_MLCE8_M : RVInstBOSCAMELoadAcc32<0b000000, 0b000, 0b0, "mlce8.m">; + def BOSC_AME_MLCE16_M : RVInstBOSCAMELoadAcc32<0b000000, 0b001, 0b0, "mlce16.m">; + def BOSC_AME_MLCE32_M : RVInstBOSCAMELoadAcc32<0b000000, 0b010, 0b0, "mlce32.m">; + def BOSC_AME_MLCE64_M : RVInstBOSCAMELoadAcc32<0b000000, 0b011, 0b0, "mlce64.m">; + + def BOSC_AME_MLCTE8_M : RVInstBOSCAMELoadAcc32<0b000000, 0b000, 0b1, "mlcte8.m">; + def BOSC_AME_MLCTE16_M : RVInstBOSCAMELoadAcc32<0b000000, 0b001, 0b1, "mlcte16.m">; + def BOSC_AME_MLCTE32_M : RVInstBOSCAMELoadAcc32<0b000000, 0b010, 0b1, "mlcte32.m">; + def BOSC_AME_MLCTE64_M : RVInstBOSCAMELoadAcc32<0b000000, 0b011, 0b1, "mlcte64.m">; + + def BOSC_AME_MLACCE8_M : RVInstBOSCAMELoadAcc32<0b000011, 0b000, 0b1, "mlacce8.m">; + def BOSC_AME_MLACCE16_M : RVInstBOSCAMELoadAcc32<0b000011, 0b001, 0b1, "mlacce16.m">; + def BOSC_AME_MLACCE32_M : RVInstBOSCAMELoadAcc32<0b000011, 0b010, 0b1, "mlacce32.m">; + def BOSC_AME_MLACCE64_M : RVInstBOSCAMELoadAcc32<0b000011, 0b011, 0b1, "mlacce64.m">; +} + +//===----------------------------------------------------------------------===// +// Store matrix A from BOSCAMETileReg - msae*.m +// Syntax: msae{8|16|32|64}.m ms3, (rs1), rs2 +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 1 in { + def BOSC_AME_MSAE8_M : RVInstBOSCAMEStoreTile32<0b000001, 0b000, 0b0, "msae8.m">; + def BOSC_AME_MSAE16_M : RVInstBOSCAMEStoreTile32<0b000001, 0b001, 0b0, "msae16.m">; + def BOSC_AME_MSAE32_M : RVInstBOSCAMEStoreTile32<0b000001, 0b010, 0b0, "msae32.m">; + def BOSC_AME_MSAE64_M : RVInstBOSCAMEStoreTile32<0b000001, 0b011, 0b0, "msae64.m">; + + def BOSC_AME_MSATE8_M : RVInstBOSCAMEStoreTile32<0b000001, 0b000, 0b1, "msate8.m">; + def BOSC_AME_MSATE16_M : RVInstBOSCAMEStoreTile32<0b000001, 0b001, 0b1, "msate16.m">; + def BOSC_AME_MSATE32_M : RVInstBOSCAMEStoreTile32<0b000001, 0b010, 0b1, "msate32.m">; + def BOSC_AME_MSATE64_M : RVInstBOSCAMEStoreTile32<0b000001, 0b011, 0b1, "msate64.m">; + + def BOSC_AME_MSTRE8_M : RVInstBOSCAMEStoreTile32<0b000011, 0b000, 0b0, "mstre8.m">; + def BOSC_AME_MSTRE16_M : RVInstBOSCAMEStoreTile32<0b000011, 0b001, 0b0, "mstre16.m">; + def BOSC_AME_MSTRE32_M : RVInstBOSCAMEStoreTile32<0b000011, 0b010, 0b0, "mstre32.m">; + def BOSC_AME_MSTRE64_M : RVInstBOSCAMEStoreTile32<0b000011, 0b011, 0b0, "mstre64.m">; +} + +//===----------------------------------------------------------------------===// +// Store matrix B from BOSCAMETileReg - msbe*.m +// Syntax: msbe{8|16|32|64}.m ms3, (rs1), rs2 +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 1 in { + def BOSC_AME_MSBE8_M : RVInstBOSCAMEStoreTile32<0b000010, 0b000, 0b0, "msbe8.m">; + def BOSC_AME_MSBE16_M : RVInstBOSCAMEStoreTile32<0b000010, 0b001, 0b0, "msbe16.m">; + def BOSC_AME_MSBE32_M : RVInstBOSCAMEStoreTile32<0b000010, 0b010, 0b0, "msbe32.m">; + def BOSC_AME_MSBE64_M : RVInstBOSCAMEStoreTile32<0b000010, 0b011, 0b0, "msbe64.m">; + + def BOSC_AME_MSBTE8_M : RVInstBOSCAMEStoreTile32<0b000010, 0b000, 0b1, "msbte8.m">; + def BOSC_AME_MSBTE16_M : RVInstBOSCAMEStoreTile32<0b000010, 0b001, 0b1, "msbte16.m">; + def BOSC_AME_MSBTE32_M : RVInstBOSCAMEStoreTile32<0b000010, 0b010, 0b1, "msbte32.m">; + def BOSC_AME_MSBTE64_M : RVInstBOSCAMEStoreTile32<0b000010, 0b011, 0b1, "msbte64.m">; +} + +//===----------------------------------------------------------------------===// +// Store matrix C (accumulator) from BOSCAMEAccReg - msce*.m +// Syntax: msce{8|16|32|64}.m ms3, (rs1), rs2 +// ms3: Source BOSCAMEAccReg (MLEN×AMUL bits) +// This is the primary store for computation results +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 1 in { + def BOSC_AME_MSCE8_M : RVInstBOSCAMEStoreAcc32<0b000000, 0b000, 0b0, "msce8.m">; + def BOSC_AME_MSCE16_M : RVInstBOSCAMEStoreAcc32<0b000000, 0b001, 0b0, "msce16.m">; + def BOSC_AME_MSCE32_M : RVInstBOSCAMEStoreAcc32<0b000000, 0b010, 0b0, "msce32.m">; + def BOSC_AME_MSCE64_M : RVInstBOSCAMEStoreAcc32<0b000000, 0b011, 0b0, "msce64.m">; + + def BOSC_AME_MSCTE8_M : RVInstBOSCAMEStoreAcc32<0b000000, 0b000, 0b1, "mscte8.m">; + def BOSC_AME_MSCTE16_M : RVInstBOSCAMEStoreAcc32<0b000000, 0b001, 0b1, "mscte16.m">; + def BOSC_AME_MSCTE32_M : RVInstBOSCAMEStoreAcc32<0b000000, 0b010, 0b1, "mscte32.m">; + def BOSC_AME_MSCTE64_M : RVInstBOSCAMEStoreAcc32<0b000000, 0b011, 0b1, "mscte64.m">; + + def BOSC_AME_MSACCE8_M : RVInstBOSCAMEStoreAcc32<0b000011, 0b000, 0b1, "msacce8.m">; + def BOSC_AME_MSACCE16_M : RVInstBOSCAMEStoreAcc32<0b000011, 0b001, 0b1, "msacce16.m">; + def BOSC_AME_MSACCE32_M : RVInstBOSCAMEStoreAcc32<0b000011, 0b010, 0b1, "msacce32.m">; + def BOSC_AME_MSACCE64_M : RVInstBOSCAMEStoreAcc32<0b000011, 0b011, 0b1, "msacce64.m">; +} + +//===----------------------------------------------------------------------===// +// BOSC AME Data Move Instructions +//===----------------------------------------------------------------------===// +// Data Move Instruction format (32-bit): +// | 31:26 | 25 | 24:20 | 19:15 | 14:12 | 11:7 | 6:0 | +// | funct6 | di | rs2 | rs1 | eew | rd | OP-M32 | +//===----------------------------------------------------------------------===// +class RVInstBOSCAMEMove + : RVInst { + bits<6> funct6; + bit di; + bits<5> rs2; + bits<5> rs1; + bits<3> eew; // 0=int8, 1=int16, 2=int32, 3=int64, 7=int4 + bits<5> rd; + bits<7> opcode = 0b1110111; + + let Inst{31-26} = funct6; + let Inst{25} = di; + let Inst{24-20} = rs2; + let Inst{19-15} = rs1; + let Inst{14-12} = eew; + let Inst{11-7} = rd; + let Inst{6-0} = 0b1110111; +} + +// Data Move Instructions between Matrix Registers +class RVInstBOSCAMEMove_T_T eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMETileReg:$md), (ins BOSCAMETileReg:$ms1), + opcodestr, "$md, $ms1"> { + bits<5> md; // BOSCAMETileReg index for store + bits<5> ms1; // BOSCAMETileReg index for load + + let funct6 = 0b000111; + let di = 0b0; + let rs2 = 0b00000; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +class RVInstBOSCAMEMove_A_A eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMEAccReg:$md), (ins BOSCAMEAccReg:$ms1), + opcodestr, "$md, $ms1"> { + bits<5> md; // BOSCAMEAccReg index for store + bits<5> ms1; // BOSCAMEAccReg index for load + + let funct6 = 0b000111; + let di = 0b0; + let rs2 = 0b00001; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +class RVInstBOSCAMEMove_A_T eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMEAccReg:$md), (ins BOSCAMETileReg:$ms1, GPR:$rs2), + opcodestr, "$md, $ms1, $rs2"> { + bits<5> md; // BOSCAMEAccReg index for store + bits<5> ms1; // BOSCAMETileReg index for load + + let funct6 = 0b000100; + let di = 0b0; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +class RVInstBOSCAMEMove_T_A eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMETileReg:$md), (ins BOSCAMEAccReg:$ms1, GPR:$rs2), + opcodestr, "$md, $ms1, $rs2"> { + bits<5> md; // BOSCAMETileReg index for load + bits<5> ms1; // BOSCAMEAccReg index for store + + let funct6 = 0b000100; + let di = 0b1; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +class RVInstBOSCAMEMoveImm_A_T eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMEAccReg:$md), (ins BOSCAMETileReg:$ms1, uimm5:$imm), + opcodestr, "$md, $ms1, $imm"> { + bits<5> imm; + bits<5> ms1; + bits<5> md; + + let funct6 = 0b000100; + let di = 0b0; + let rs2 = imm; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +class RVInstBOSCAMEMoveImm_T_A eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMETileReg:$md), (ins BOSCAMEAccReg:$ms1, uimm5:$imm), + opcodestr, "$md, $ms1, $imm"> { + bits<5> imm; + bits<5> ms1; + bits<5> md; + + let funct6 = 0b000100; + let di = 0b1; + let rs2 = imm; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MMVE8_T_T : RVInstBOSCAMEMove_T_T<0b000, "mmve8.t.t">; + def BOSC_AME_MMVE16_T_T : RVInstBOSCAMEMove_T_T<0b001, "mmve16.t.t">; + def BOSC_AME_MMVE32_T_T : RVInstBOSCAMEMove_T_T<0b010, "mmve32.t.t">; + def BOSC_AME_MMVE64_T_T : RVInstBOSCAMEMove_T_T<0b011, "mmve64.t.t">; + + def BOSC_AME_MMVE8_A_A : RVInstBOSCAMEMove_A_A<0b000, "mmve8.a.a">; + def BOSC_AME_MMVE16_A_A : RVInstBOSCAMEMove_A_A<0b001, "mmve16.a.a">; + def BOSC_AME_MMVE32_A_A : RVInstBOSCAMEMove_A_A<0b010, "mmve32.a.a">; + def BOSC_AME_MMVE64_A_A : RVInstBOSCAMEMove_A_A<0b011, "mmve64.a.a">; + + def BOSC_AME_MMVE8_A_T : RVInstBOSCAMEMove_A_T<0b000, "mmve8.a.t">; + def BOSC_AME_MMVE16_A_T : RVInstBOSCAMEMove_A_T<0b001, "mmve16.a.t">; + def BOSC_AME_MMVE32_A_T : RVInstBOSCAMEMove_A_T<0b010, "mmve32.a.t">; + def BOSC_AME_MMVE64_A_T : RVInstBOSCAMEMove_A_T<0b011, "mmve64.a.t">; + + def BOSC_AME_MMVE8_T_A : RVInstBOSCAMEMove_T_A<0b000, "mmve8.t.a">; + def BOSC_AME_MMVE16_T_A : RVInstBOSCAMEMove_T_A<0b001, "mmve16.t.a">; + def BOSC_AME_MMVE32_T_A : RVInstBOSCAMEMove_T_A<0b010, "mmve32.t.a">; + def BOSC_AME_MMVE64_T_A : RVInstBOSCAMEMove_T_A<0b011, "mmve64.t.a">; + + def BOSC_AME_MMVIE8_A_T : RVInstBOSCAMEMoveImm_A_T<0b100, "mmvie8.a.t">; + def BOSC_AME_MMVIE16_A_T : RVInstBOSCAMEMoveImm_A_T<0b101, "mmvie16.a.t">; + def BOSC_AME_MMVIE32_A_T : RVInstBOSCAMEMoveImm_A_T<0b110, "mmvie32.a.t">; + def BOSC_AME_MMVIE64_A_T : RVInstBOSCAMEMoveImm_A_T<0b111, "mmvie64.a.t">; + + def BOSC_AME_MMVIE8_T_A : RVInstBOSCAMEMoveImm_T_A<0b100, "mmvie8.t.a">; + def BOSC_AME_MMVIE16_T_A : RVInstBOSCAMEMoveImm_T_A<0b101, "mmvie16.t.a">; + def BOSC_AME_MMVIE32_T_A : RVInstBOSCAMEMoveImm_T_A<0b110, "mmvie32.t.a">; + def BOSC_AME_MMVIE64_T_A : RVInstBOSCAMEMoveImm_T_A<0b111, "mmvie64.t.a">; +} + +// Data Move Instructions between Matrix and Integer +class RVInstBOSCAMEMove_X_T eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs GPR:$rd), (ins BOSCAMETileReg:$ms1, GPR:$rs2), + opcodestr, "$rd, $ms1, $rs2"> { + bits<5> rs2; + bits<5> ms1; + bits<5> rd; + + let funct6 = 0b000101; + let di = 0b0; + let rs1 = ms1; + let eew = eew_val; +} + +class RVInstBOSCAMEMove_T_X eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMETileReg:$md), (ins GPR:$rs1, GPR:$rs2), + opcodestr, "$md, $rs1, $rs2"> { + bits<5> rs2; + bits<5> rs1; + bits<5> md; + + let funct6 = 0b000101; + let di = 0b1; + let eew = eew_val; + let rd = md; +} + +class RVInstBOSCAMEMove_X_A eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs GPR:$rd), (ins BOSCAMEAccReg:$ms1, GPR:$rs2), + opcodestr, "$rd, $ms1, $rs2"> { + bits<5> rs2; + bits<5> ms1; + bits<5> rd; + + let funct6 = 0b000101; + let di = 0b0; + let rs1 = ms1; + let eew = eew_val; +} + +class RVInstBOSCAMEMove_A_X eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMEAccReg:$md), (ins GPR:$rs1, GPR:$rs2), + opcodestr, "$md, $rs1, $rs2"> { + bits<5> rs2; + bits<5> rs1; + bits<5> md; + + let funct6 = 0b000101; + let di = 0b1; + let eew = eew_val; + let rd = md; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MMVE8_X_T : RVInstBOSCAMEMove_X_T<0b000, "mmve8.x.t">; + def BOSC_AME_MMVE16_X_T : RVInstBOSCAMEMove_X_T<0b001, "mmve16.x.t">; + def BOSC_AME_MMVE32_X_T : RVInstBOSCAMEMove_X_T<0b010, "mmve32.x.t">; + def BOSC_AME_MMVE64_X_T : RVInstBOSCAMEMove_X_T<0b011, "mmve64.x.t">; + + def BOSC_AME_MMVE8_T_X : RVInstBOSCAMEMove_T_X<0b000, "mmve8.t.x">; + def BOSC_AME_MMVE16_T_X : RVInstBOSCAMEMove_T_X<0b001, "mmve16.t.x">; + def BOSC_AME_MMVE32_T_X : RVInstBOSCAMEMove_T_X<0b010, "mmve32.t.x">; + def BOSC_AME_MMVE64_T_X : RVInstBOSCAMEMove_T_X<0b011, "mmve64.t.x">; + + def BOSC_AME_MMVE8_X_A : RVInstBOSCAMEMove_X_A<0b100, "mmve8.x.a">; + def BOSC_AME_MMVE16_X_A : RVInstBOSCAMEMove_X_A<0b101, "mmve16.x.a">; + def BOSC_AME_MMVE32_X_A : RVInstBOSCAMEMove_X_A<0b110, "mmve32.x.a">; + def BOSC_AME_MMVE64_X_A : RVInstBOSCAMEMove_X_A<0b111, "mmve64.x.a">; + + def BOSC_AME_MMVE8_A_X : RVInstBOSCAMEMove_A_X<0b100, "mmve8.a.x">; + def BOSC_AME_MMVE16_A_X : RVInstBOSCAMEMove_A_X<0b101, "mmve16.a.x">; + def BOSC_AME_MMVE32_A_X : RVInstBOSCAMEMove_A_X<0b110, "mmve32.a.x">; + def BOSC_AME_MMVE64_A_X : RVInstBOSCAMEMove_A_X<0b111, "mmve64.a.x">; +} + +// Data Move Instructions between Matrix and Float-point +class RVInstBOSCAMEFMove_F_T eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs GPR:$rd), (ins BOSCAMETileReg:$ms1, GPR:$rs2), + opcodestr, "$rd, $ms1, $rs2"> { + bits<5> rs2; + bits<5> ms1; + bits<5> rd; + + let funct6 = 0b000110; + let di = 0b0; + let rs1 = ms1; + let eew = eew_val; +} + +class RVInstBOSCAMEFMove_T_F eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMETileReg:$md), (ins GPR:$rs1, GPR:$rs2), + opcodestr, "$md, $rs1, $rs2"> { + bits<5> rs2; + bits<5> rs1; + bits<5> md; + + let funct6 = 0b000110; + let di = 0b1; + let eew = eew_val; + let rd = md; +} + +class RVInstBOSCAMEFMove_F_A eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs GPR:$rd), (ins BOSCAMEAccReg:$ms1, GPR:$rs2), + opcodestr, "$rd, $ms1, $rs2"> { + bits<5> rs2; + bits<5> ms1; + bits<5> rd; + + let funct6 = 0b000110; + let di = 0b0; + let rs1 = ms1; + let eew = eew_val; +} + +class RVInstBOSCAMEFMove_A_F eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMEAccReg:$md), (ins GPR:$rs1, GPR:$rs2), + opcodestr, "$md, $rs1, $rs2"> { + bits<5> rs2; + bits<5> rs1; + bits<5> md; + + let funct6 = 0b000110; + let di = 0b1; + let eew = eew_val; + let rd = md; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + + def BOSC_AME_MFMVE8_F_T : RVInstBOSCAMEFMove_F_T<0b000, "mfmve8.f.t">; + def BOSC_AME_MFMVE16_F_T : RVInstBOSCAMEFMove_F_T<0b001, "mfmve16.f.t">; + def BOSC_AME_MFMVE32_F_T : RVInstBOSCAMEFMove_F_T<0b010, "mfmve32.f.t">; + def BOSC_AME_MFMVE64_F_T : RVInstBOSCAMEFMove_F_T<0b011, "mfmve64.f.t">; + + def BOSC_AME_MFMVE8_T_F : RVInstBOSCAMEFMove_T_F<0b000, "mfmve8.t.f">; + def BOSC_AME_MFMVE16_T_F : RVInstBOSCAMEFMove_T_F<0b001, "mfmve16.t.f">; + def BOSC_AME_MFMVE32_T_F : RVInstBOSCAMEFMove_T_F<0b010, "mfmve32.t.f">; + def BOSC_AME_MFMVE64_T_F : RVInstBOSCAMEFMove_T_F<0b011, "mfmve64.t.f">; + + def BOSC_AME_MFMVE8_F_A : RVInstBOSCAMEFMove_F_A<0b100, "mfmve8.f.a">; + def BOSC_AME_MFMVE16_F_A : RVInstBOSCAMEFMove_F_A<0b101, "mfmve16.f.a">; + def BOSC_AME_MFMVE32_F_A : RVInstBOSCAMEFMove_F_A<0b110, "mfmve32.f.a">; + def BOSC_AME_MFMVE64_F_A : RVInstBOSCAMEFMove_F_A<0b111, "mfmve64.f.a">; + + def BOSC_AME_MFMVE8_A_F : RVInstBOSCAMEFMove_A_F<0b100, "mfmve8.a.f">; + def BOSC_AME_MFMVE16_A_F : RVInstBOSCAMEFMove_A_F<0b101, "mfmve16.a.f">; + def BOSC_AME_MFMVE32_A_F : RVInstBOSCAMEFMove_A_F<0b110, "mfmve32.a.f">; + def BOSC_AME_MFMVE64_A_F : RVInstBOSCAMEFMove_A_F<0b111, "mfmve64.a.f">; +} + +// Data Broadcast Instructions +class RVInstBOSCAMEBroadcastTile rs2_val, bits<3> eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMETileReg:$md), (ins BOSCAMETileReg:$ms1), + opcodestr, "$md, $ms1"> { + bits<5> md; + bits<5> ms1; + + let funct6 = 0b000111; + let di = 0b0; + let rs2 = rs2_val; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +class RVInstBOSCAMEBroadcastAcc rs2_val, bits<3> eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMEAccReg:$md), (ins BOSCAMEAccReg:$ms1), + opcodestr, "$md, $ms1"> { + bits<5> md; + bits<5> ms1; + + let funct6 = 0b000111; + let di = 0b0; + let rs2 = rs2_val; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // Row broadcast from BOSCAMETileReg/BOSCAMEAccReg + def BOSC_AME_MBCAR_M : RVInstBOSCAMEBroadcastTile<0b10001, 0b000, "mbcar.m">; + def BOSC_AME_MBCBR_M : RVInstBOSCAMEBroadcastTile<0b10010, 0b000, "mbcbr.m">; + def BOSC_AME_MBCCR_M : RVInstBOSCAMEBroadcastAcc<0b10000, 0b000, "mbccr.m">; + // Column broadcast from BOSCAMETileReg + def BOSC_AME_MBCACE8_M : RVInstBOSCAMEBroadcastTile<0b10101, 0b000, "mbcace8.m">; + def BOSC_AME_MBCACE16_M : RVInstBOSCAMEBroadcastTile<0b10101, 0b001, "mbcace16.m">; + def BOSC_AME_MBCACE32_M : RVInstBOSCAMEBroadcastTile<0b10101, 0b010, "mbcace32.m">; + def BOSC_AME_MBCACE64_M : RVInstBOSCAMEBroadcastTile<0b10101, 0b011, "mbcace64.m">; + // Column broadcast from BOSCAMETileReg + def BOSC_AME_MBCBCE8_M : RVInstBOSCAMEBroadcastTile<0b10110, 0b000, "mbcbce8.m">; + def BOSC_AME_MBCBCE16_M : RVInstBOSCAMEBroadcastTile<0b10110, 0b001, "mbcbce16.m">; + def BOSC_AME_MBCBCE32_M : RVInstBOSCAMEBroadcastTile<0b10110, 0b010, "mbcbce32.m">; + def BOSC_AME_MBCBCE64_M : RVInstBOSCAMEBroadcastTile<0b10110, 0b011, "mbcbce64.m">; + // Column broadcast from BOSCAMEAccReg + def BOSC_AME_MBCCCE8_M : RVInstBOSCAMEBroadcastAcc<0b10100, 0b000, "mbccce8.m">; + def BOSC_AME_MBCCCE16_M : RVInstBOSCAMEBroadcastAcc<0b10100, 0b001, "mbccce16.m">; + def BOSC_AME_MBCCCE32_M : RVInstBOSCAMEBroadcastAcc<0b10100, 0b010, "mbccce32.m">; + def BOSC_AME_MBCCCE64_M : RVInstBOSCAMEBroadcastAcc<0b10100, 0b011, "mbccce64.m">; + // Element broadcast from BOSCAMETileReg + def BOSC_AME_MBCAEE8_M : RVInstBOSCAMEBroadcastTile<0b11001, 0b000, "mbcaee8.m">; + def BOSC_AME_MBCAEE16_M : RVInstBOSCAMEBroadcastTile<0b11001, 0b001, "mbcaee16.m">; + def BOSC_AME_MBCAEE32_M : RVInstBOSCAMEBroadcastTile<0b11001, 0b010, "mbcaee32.m">; + def BOSC_AME_MBCAEE64_M : RVInstBOSCAMEBroadcastTile<0b11001, 0b011, "mbcaee64.m">; + // Element broadcast from BOSCAMETileReg + def BOSC_AME_MBCBEE8_M : RVInstBOSCAMEBroadcastTile<0b11010, 0b000, "mbcbee8.m">; + def BOSC_AME_MBCBEE16_M : RVInstBOSCAMEBroadcastTile<0b11010, 0b001, "mbcbee16.m">; + def BOSC_AME_MBCBEE32_M : RVInstBOSCAMEBroadcastTile<0b11010, 0b010, "mbcbee32.m">; + def BOSC_AME_MBCBEE64_M : RVInstBOSCAMEBroadcastTile<0b11010, 0b011, "mbcbee64.m">; + // Element broadcast from BOSCAMEAccReg + def BOSC_AME_MBCCEE8_M : RVInstBOSCAMEBroadcastAcc<0b11000, 0b000, "mbccee8.m">; + def BOSC_AME_MBCCEE16_M : RVInstBOSCAMEBroadcastAcc<0b11000, 0b001, "mbccee16.m">; + def BOSC_AME_MBCCEE32_M : RVInstBOSCAMEBroadcastAcc<0b11000, 0b010, "mbccee32.m">; + def BOSC_AME_MBCCEE64_M : RVInstBOSCAMEBroadcastAcc<0b11000, 0b011, "mbccee64.m">; +} + +// Matrix Transpose Instructions +class RVInstBOSCAMETransposeTile rs2_val, bits<3> eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMETileReg:$md), (ins BOSCAMETileReg:$ms1), + opcodestr, "$md, $ms1"> { + bits<5> md; + bits<5> ms1; + + let funct6 = 0b000111; + let di = 0b0; + let rs2 = rs2_val; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +class RVInstBOSCAMETransposeAcc rs2_val, bits<3> eew_val, string opcodestr> + : RVInstBOSCAMEMove<(outs BOSCAMEAccReg:$md), (ins BOSCAMEAccReg:$ms1), + opcodestr, "$md, $ms1"> { + bits<5> md; + bits<5> ms1; + + let funct6 = 0b000111; + let di = 0b0; + let rs2 = rs2_val; + let rs1 = ms1; + let eew = eew_val; + let rd = md; + let opcode = 0b1110111; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // Transpose square matrix of tile register + def BOSC_AME_MTAE8_M : RVInstBOSCAMETransposeTile<0b11101, 0b000, "mtae8.m">; + def BOSC_AME_MTAE16_M : RVInstBOSCAMETransposeTile<0b11101, 0b001, "mtae16.m">; + def BOSC_AME_MTAE32_M : RVInstBOSCAMETransposeTile<0b11101, 0b010, "mtae32.m">; + def BOSC_AME_MTAE64_M : RVInstBOSCAMETransposeTile<0b11101, 0b011, "mtae64.m">; + // Transpose square matrix of tile register + def BOSC_AME_MTBE8_M : RVInstBOSCAMETransposeTile<0b11110, 0b000, "mtbe8.m">; + def BOSC_AME_MTBE16_M : RVInstBOSCAMETransposeTile<0b11110, 0b001, "mtbe16.m">; + def BOSC_AME_MTBE32_M : RVInstBOSCAMETransposeTile<0b11110, 0b010, "mtbe32.m">; + def BOSC_AME_MTBE64_M : RVInstBOSCAMETransposeTile<0b11110, 0b011, "mtbe64.m">; + // Transpose square matrix of acc register + def BOSC_AME_MTCE8_M : RVInstBOSCAMETransposeAcc<0b11100, 0b000, "mtce8.m">; + def BOSC_AME_MTCE16_M : RVInstBOSCAMETransposeAcc<0b11100, 0b001, "mtce16.m">; + def BOSC_AME_MTCE32_M : RVInstBOSCAMETransposeAcc<0b11100, 0b010, "mtce32.m">; + def BOSC_AME_MTCE64_M : RVInstBOSCAMETransposeAcc<0b11100, 0b011, "mtce64.m">; +} + +//===----------------------------------------------------------------------===// +// Pseudo Instructions for Index-Based Operations +//===----------------------------------------------------------------------===// +// These pseudo instructions accept i64 indices and have AsmString for direct +// assembly output. This allows the pseudo instructions to be printed directly +// without needing complex expansion logic. +// +// For load/store/mma instructions, the AsmString hardcodes "acc" and "tr" +// prefixes so that indices 0-7 are printed as acc0-acc7 and tr0-tr7. +//===----------------------------------------------------------------------===// + +// Pseudo configuration instruction +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 1, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + // Register configuration instructions + def BOSC_AME_MSETTYPE_PSEUDO : Pseudo<(outs GPR:$rd), (ins GPR:$rs1), []> { + let AsmString = "msettype\t$rd, $rs1"; + } + def BOSC_AME_MSETTILEM_PSEUDO : Pseudo<(outs GPR:$rd), (ins GPR:$rs1), []> { + let AsmString = "msettilem\t$rd, $rs1"; + } + def BOSC_AME_MSETTILEN_PSEUDO : Pseudo<(outs GPR:$rd), (ins GPR:$rs1), []> { + let AsmString = "msettilen\t$rd, $rs1"; + } + def BOSC_AME_MSETTILEK_PSEUDO : Pseudo<(outs GPR:$rd), (ins GPR:$rs1), []> { + let AsmString = "msettilek\t$rd, $rs1"; + } + // Immediate configuration instructions + def BOSC_AME_MSETTYPEI_PSEUDO : Pseudo<(outs GPR:$rd), (ins i64imm:$imm), []> { + let AsmString = "msettypei\t$rd, $imm"; + } + def BOSC_AME_MSETTYPEHI_PSEUDO : Pseudo<(outs GPR:$rd), (ins i64imm:$imm), []> { + let AsmString = "msettypehi\t$rd, $imm"; + } + def BOSC_AME_MSETTILEMI_PSEUDO : Pseudo<(outs GPR:$rd), (ins i64imm:$imm), []> { + let AsmString = "msettilemi\t$rd, $imm"; + } + def BOSC_AME_MSETTILENI_PSEUDO : Pseudo<(outs GPR:$rd), (ins i64imm:$imm), []> { + let AsmString = "msettileni\t$rd, $imm"; + } + def BOSC_AME_MSETTILEKI_PSEUDO : Pseudo<(outs GPR:$rd), (ins i64imm:$imm), []> { + let AsmString = "msettileki\t$rd, $imm"; + } +} + +// Pseudo load instructions +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 1, mayLoad = 1, mayStore = 0, + isCodeGenOnly = 1 in { + def BOSC_AME_MLAE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlae8.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLAE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlae16.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLAE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlae32.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLAE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlae64.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLATE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlate8.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLATE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlate16.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLATE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlate32.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLATE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlate64.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLTRE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mltre8.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLTRE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mltre16.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLTRE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mltre32.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLTRE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mltre64.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLBE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlbe8.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLBE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlbe16.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLBE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlbe32.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLBE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlbe64.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLBTE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlbte8.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLBTE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlbte16.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLBTE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlbte32.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLBTE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlbte64.m\ttr$md, ($rs1), $rs2"; + } + def BOSC_AME_MLCE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlce8.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLCE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlce16.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLCE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlce32.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLCE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlce64.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLCTE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlcte8.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLCTE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlcte16.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLCTE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlcte32.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLCTE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlcte64.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLACCE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlacce8.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLACCE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlacce16.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLACCE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlacce32.m\tacc$md, ($rs1), $rs2"; + } + def BOSC_AME_MLACCE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mlacce64.m\tacc$md, ($rs1), $rs2"; + } +} + +// Pseudo store instructions +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 1, mayLoad = 0, mayStore = 1, + isCodeGenOnly = 1 in { + def BOSC_AME_MSAE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msae8.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSAE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msae16.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSAE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msae32.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSAE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msae64.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSATE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msate8.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSATE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msate16.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSATE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msate32.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSATE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msate64.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSTRE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mstre8.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSTRE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mstre16.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSTRE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mstre32.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSTRE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mstre64.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSBE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msbe8.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSBE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msbe16.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSBE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msbe32.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSBE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msbe64.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSBTE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msbte8.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSBTE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msbte16.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSBTE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msbte32.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSBTE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMETileIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msbte64.m\ttr$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSCE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msce8.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSCE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msce16.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSCE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msce32.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSCE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msce64.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSCTE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mscte8.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSCTE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mscte16.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSCTE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mscte32.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSCTE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "mscte64.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSACCE8_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msacce8.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSACCE16_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msacce16.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSACCE32_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msacce32.m\tacc$ms3, ($rs1), $rs2"; + } + def BOSC_AME_MSACCE64_M_PSEUDO : Pseudo<(outs), (ins BOSCAMEAccIndex:$ms3, GPR:$rs1, GPR:$rs2), []> { + let AsmString = "msacce64.m\tacc$ms3, ($rs1), $rs2"; + } +} + +// Pseudo data move instructions +// Matrix to Matrix (Pure) +class RVInstBOSCAMEMove_T_T_Pseudo + : Pseudo<(outs), (ins BOSCAMETileIndex:$md, BOSCAMETileIndex:$ms1), []> { + let AsmString = opcodestr # "\ttr$md, tr$ms1"; +} +class RVInstBOSCAMEMove_A_A_Pseudo + : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, BOSCAMEAccIndex:$ms1), []> { + let AsmString = opcodestr # "\tacc$md, acc$ms1"; +} + +// Matrix to Matrix (with GPR index) +class RVInstBOSCAMEMove_A_T_Pseudo + : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, BOSCAMETileIndex:$ms1, GPR:$rs2), []> { + let AsmString = opcodestr # "\tacc$md, tr$ms1, $rs2"; +} +class RVInstBOSCAMEMove_T_A_Pseudo + : Pseudo<(outs), (ins BOSCAMETileIndex:$md, BOSCAMEAccIndex:$ms1, GPR:$rs2), []> { + let AsmString = opcodestr # "\ttr$md, acc$ms1, $rs2"; +} + +// Matrix to Matrix (with Immediate index) +class RVInstBOSCAMEMoveImm_A_T_Pseudo + : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, BOSCAMETileIndex:$ms1, uimm5:$imm), []> { + let AsmString = opcodestr # "\tacc$md, tr$ms1, $imm"; +} +class RVInstBOSCAMEMoveImm_T_A_Pseudo + : Pseudo<(outs), (ins BOSCAMETileIndex:$md, BOSCAMEAccIndex:$ms1, uimm5:$imm), []> { + let AsmString = opcodestr # "\ttr$md, acc$ms1, $imm"; +} + +// Matrix to Scalar GPR (Notice: rd is a real output GPR) +class RVInstBOSCAMEMove_X_T_Pseudo + : Pseudo<(outs GPR:$rd), (ins BOSCAMETileIndex:$ms1, GPR:$rs2), []> { + let AsmString = opcodestr # "\t$rd, tr$ms1, $rs2"; +} +class RVInstBOSCAMEMove_X_A_Pseudo + : Pseudo<(outs GPR:$rd), (ins BOSCAMEAccIndex:$ms1, GPR:$rs2), []> { + let AsmString = opcodestr # "\t$rd, acc$ms1, $rs2"; +} + +// Scalar GPR to Matrix +class RVInstBOSCAMEMove_T_X_Pseudo + : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = opcodestr # "\ttr$md, $rs1, $rs2"; +} +class RVInstBOSCAMEMove_A_X_Pseudo + : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = opcodestr # "\tacc$md, $rs1, $rs2"; +} + +// Pseudo classes for Float-point Matrix moves +class RVInstBOSCAMEMove_F_T_Pseudo + : Pseudo<(outs GPR:$rd), (ins BOSCAMETileIndex:$ms1, GPR:$rs2), []> { + let AsmString = opcodestr # "\t$rd, tr$ms1, $rs2"; +} + +class RVInstBOSCAMEMove_T_F_Pseudo + : Pseudo<(outs), (ins BOSCAMETileIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = opcodestr # "\ttr$md, $rs1, $rs2"; +} + +class RVInstBOSCAMEMove_F_A_Pseudo + : Pseudo<(outs GPR:$rd), (ins BOSCAMEAccIndex:$ms1, GPR:$rs2), []> { + let AsmString = opcodestr # "\t$rd, acc$ms1, $rs2"; +} + +class RVInstBOSCAMEMove_A_F_Pseudo + : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, GPR:$rs1, GPR:$rs2), []> { + let AsmString = opcodestr # "\tacc$md, $rs1, $rs2"; +} + +class RVInstBOSCAMEBroadcastTile_Pseudo + : RVInstBOSCAMEMove_T_T_Pseudo; +class RVInstBOSCAMEBroadcastAcc_Pseudo + : RVInstBOSCAMEMove_A_A_Pseudo; +class RVInstBOSCAMETransposeTile_Pseudo + : RVInstBOSCAMEMove_T_T_Pseudo; +class RVInstBOSCAMETransposeAcc_Pseudo + : RVInstBOSCAMEMove_A_A_Pseudo; + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 1, isCodeGenOnly = 1 in { + //---------------------------------------------------------------------- + // Matrix-Matrix Move Pseudos + //---------------------------------------------------------------------- + def BOSC_AME_MMVE8_T_T_PSEUDO : RVInstBOSCAMEMove_T_T_Pseudo<"mmve8.t.t">; + def BOSC_AME_MMVE16_T_T_PSEUDO : RVInstBOSCAMEMove_T_T_Pseudo<"mmve16.t.t">; + def BOSC_AME_MMVE32_T_T_PSEUDO : RVInstBOSCAMEMove_T_T_Pseudo<"mmve32.t.t">; + def BOSC_AME_MMVE64_T_T_PSEUDO : RVInstBOSCAMEMove_T_T_Pseudo<"mmve64.t.t">; + + def BOSC_AME_MMVE8_A_A_PSEUDO : RVInstBOSCAMEMove_A_A_Pseudo<"mmve8.a.a">; + def BOSC_AME_MMVE16_A_A_PSEUDO : RVInstBOSCAMEMove_A_A_Pseudo<"mmve16.a.a">; + def BOSC_AME_MMVE32_A_A_PSEUDO : RVInstBOSCAMEMove_A_A_Pseudo<"mmve32.a.a">; + def BOSC_AME_MMVE64_A_A_PSEUDO : RVInstBOSCAMEMove_A_A_Pseudo<"mmve64.a.a">; + + def BOSC_AME_MMVE8_A_T_PSEUDO : RVInstBOSCAMEMove_A_T_Pseudo<"mmve8.a.t">; + def BOSC_AME_MMVE16_A_T_PSEUDO : RVInstBOSCAMEMove_A_T_Pseudo<"mmve16.a.t">; + def BOSC_AME_MMVE32_A_T_PSEUDO : RVInstBOSCAMEMove_A_T_Pseudo<"mmve32.a.t">; + def BOSC_AME_MMVE64_A_T_PSEUDO : RVInstBOSCAMEMove_A_T_Pseudo<"mmve64.a.t">; + + def BOSC_AME_MMVE8_T_A_PSEUDO : RVInstBOSCAMEMove_T_A_Pseudo<"mmve8.t.a">; + def BOSC_AME_MMVE16_T_A_PSEUDO : RVInstBOSCAMEMove_T_A_Pseudo<"mmve16.t.a">; + def BOSC_AME_MMVE32_T_A_PSEUDO : RVInstBOSCAMEMove_T_A_Pseudo<"mmve32.t.a">; + def BOSC_AME_MMVE64_T_A_PSEUDO : RVInstBOSCAMEMove_T_A_Pseudo<"mmve64.t.a">; + + def BOSC_AME_MMVIE8_A_T_PSEUDO : RVInstBOSCAMEMoveImm_A_T_Pseudo<"mmvie8.a.t">; + def BOSC_AME_MMVIE16_A_T_PSEUDO : RVInstBOSCAMEMoveImm_A_T_Pseudo<"mmvie16.a.t">; + def BOSC_AME_MMVIE32_A_T_PSEUDO : RVInstBOSCAMEMoveImm_A_T_Pseudo<"mmvie32.a.t">; + def BOSC_AME_MMVIE64_A_T_PSEUDO : RVInstBOSCAMEMoveImm_A_T_Pseudo<"mmvie64.a.t">; + + def BOSC_AME_MMVIE8_T_A_PSEUDO : RVInstBOSCAMEMoveImm_T_A_Pseudo<"mmvie8.t.a">; + def BOSC_AME_MMVIE16_T_A_PSEUDO : RVInstBOSCAMEMoveImm_T_A_Pseudo<"mmvie16.t.a">; + def BOSC_AME_MMVIE32_T_A_PSEUDO : RVInstBOSCAMEMoveImm_T_A_Pseudo<"mmvie32.t.a">; + def BOSC_AME_MMVIE64_T_A_PSEUDO : RVInstBOSCAMEMoveImm_T_A_Pseudo<"mmvie64.t.a">; + //---------------------------------------------------------------------- + // Int / Float Matrix-Scalar Move Pseudos + //---------------------------------------------------------------------- + def BOSC_AME_MMVE8_X_T_PSEUDO : RVInstBOSCAMEMove_X_T_Pseudo<"mmve8.x.t">; + def BOSC_AME_MMVE16_X_T_PSEUDO : RVInstBOSCAMEMove_X_T_Pseudo<"mmve16.x.t">; + def BOSC_AME_MMVE32_X_T_PSEUDO : RVInstBOSCAMEMove_X_T_Pseudo<"mmve32.x.t">; + def BOSC_AME_MMVE64_X_T_PSEUDO : RVInstBOSCAMEMove_X_T_Pseudo<"mmve64.x.t">; + + def BOSC_AME_MMVE8_T_X_PSEUDO : RVInstBOSCAMEMove_T_X_Pseudo<"mmve8.t.x">; + def BOSC_AME_MMVE16_T_X_PSEUDO : RVInstBOSCAMEMove_T_X_Pseudo<"mmve16.t.x">; + def BOSC_AME_MMVE32_T_X_PSEUDO : RVInstBOSCAMEMove_T_X_Pseudo<"mmve32.t.x">; + def BOSC_AME_MMVE64_T_X_PSEUDO : RVInstBOSCAMEMove_T_X_Pseudo<"mmve64.t.x">; + + def BOSC_AME_MMVE8_X_A_PSEUDO : RVInstBOSCAMEMove_X_A_Pseudo<"mmve8.x.a">; + def BOSC_AME_MMVE16_X_A_PSEUDO : RVInstBOSCAMEMove_X_A_Pseudo<"mmve16.x.a">; + def BOSC_AME_MMVE32_X_A_PSEUDO : RVInstBOSCAMEMove_X_A_Pseudo<"mmve32.x.a">; + def BOSC_AME_MMVE64_X_A_PSEUDO : RVInstBOSCAMEMove_X_A_Pseudo<"mmve64.x.a">; + + def BOSC_AME_MMVE8_A_X_PSEUDO : RVInstBOSCAMEMove_A_X_Pseudo<"mmve8.a.x">; + def BOSC_AME_MMVE16_A_X_PSEUDO : RVInstBOSCAMEMove_A_X_Pseudo<"mmve16.a.x">; + def BOSC_AME_MMVE32_A_X_PSEUDO : RVInstBOSCAMEMove_A_X_Pseudo<"mmve32.a.x">; + def BOSC_AME_MMVE64_A_X_PSEUDO : RVInstBOSCAMEMove_A_X_Pseudo<"mmve64.a.x">; + + def BOSC_AME_MFMVE8_F_T_PSEUDO : RVInstBOSCAMEMove_F_T_Pseudo<"mfmve8.f.t">; + def BOSC_AME_MFMVE16_F_T_PSEUDO : RVInstBOSCAMEMove_F_T_Pseudo<"mfmve16.f.t">; + def BOSC_AME_MFMVE32_F_T_PSEUDO : RVInstBOSCAMEMove_F_T_Pseudo<"mfmve32.f.t">; + def BOSC_AME_MFMVE64_F_T_PSEUDO : RVInstBOSCAMEMove_F_T_Pseudo<"mfmve64.f.t">; + + def BOSC_AME_MFMVE8_T_F_PSEUDO : RVInstBOSCAMEMove_T_F_Pseudo<"mfmve8.t.f">; + def BOSC_AME_MFMVE16_T_F_PSEUDO : RVInstBOSCAMEMove_T_F_Pseudo<"mfmve16.t.f">; + def BOSC_AME_MFMVE32_T_F_PSEUDO : RVInstBOSCAMEMove_T_F_Pseudo<"mfmve32.t.f">; + def BOSC_AME_MFMVE64_T_F_PSEUDO : RVInstBOSCAMEMove_T_F_Pseudo<"mfmve64.t.f">; + + def BOSC_AME_MFMVE8_F_A_PSEUDO : RVInstBOSCAMEMove_F_A_Pseudo<"mfmve8.f.a">; + def BOSC_AME_MFMVE16_F_A_PSEUDO : RVInstBOSCAMEMove_F_A_Pseudo<"mfmve16.f.a">; + def BOSC_AME_MFMVE32_F_A_PSEUDO : RVInstBOSCAMEMove_F_A_Pseudo<"mfmve32.f.a">; + def BOSC_AME_MFMVE64_F_A_PSEUDO : RVInstBOSCAMEMove_F_A_Pseudo<"mfmve64.f.a">; + + def BOSC_AME_MFMVE8_A_F_PSEUDO : RVInstBOSCAMEMove_A_F_Pseudo<"mfmve8.a.f">; + def BOSC_AME_MFMVE16_A_F_PSEUDO : RVInstBOSCAMEMove_A_F_Pseudo<"mfmve16.a.f">; + def BOSC_AME_MFMVE32_A_F_PSEUDO : RVInstBOSCAMEMove_A_F_Pseudo<"mfmve32.a.f">; + def BOSC_AME_MFMVE64_A_F_PSEUDO : RVInstBOSCAMEMove_A_F_Pseudo<"mfmve64.a.f">; + //---------------------------------------------------------------------- + // Broadcast Pseudos + //---------------------------------------------------------------------- + def BOSC_AME_MBCAR_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcar.m">; + def BOSC_AME_MBCBR_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbr.m">; + def BOSC_AME_MBCCR_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccr.m">; + + def BOSC_AME_MBCACE8_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcace8.m">; + def BOSC_AME_MBCACE16_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcace16.m">; + def BOSC_AME_MBCACE32_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcace32.m">; + def BOSC_AME_MBCACE64_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcace64.m">; + + def BOSC_AME_MBCBCE8_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbce8.m">; + def BOSC_AME_MBCBCE16_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbce16.m">; + def BOSC_AME_MBCBCE32_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbce32.m">; + def BOSC_AME_MBCBCE64_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbce64.m">; + + def BOSC_AME_MBCCCE8_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccce8.m">; + def BOSC_AME_MBCCCE16_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccce16.m">; + def BOSC_AME_MBCCCE32_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccce32.m">; + def BOSC_AME_MBCCCE64_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccce64.m">; + + def BOSC_AME_MBCAEE8_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcaee8.m">; + def BOSC_AME_MBCAEE16_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcaee16.m">; + def BOSC_AME_MBCAEE32_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcaee32.m">; + def BOSC_AME_MBCAEE64_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcaee64.m">; + + def BOSC_AME_MBCBEE8_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbee8.m">; + def BOSC_AME_MBCBEE16_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbee16.m">; + def BOSC_AME_MBCBEE32_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbee32.m">; + def BOSC_AME_MBCBEE64_M_PSEUDO : RVInstBOSCAMEBroadcastTile_Pseudo<"mbcbee64.m">; + + def BOSC_AME_MBCCEE8_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccee8.m">; + def BOSC_AME_MBCCEE16_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccee16.m">; + def BOSC_AME_MBCCEE32_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccee32.m">; + def BOSC_AME_MBCCEE64_M_PSEUDO : RVInstBOSCAMEBroadcastAcc_Pseudo<"mbccee64.m">; + //---------------------------------------------------------------------- + // Transpose Pseudos + //---------------------------------------------------------------------- + def BOSC_AME_MTAE8_M_PSEUDO : RVInstBOSCAMETransposeTile_Pseudo<"mtae8.m">; + def BOSC_AME_MTAE16_M_PSEUDO : RVInstBOSCAMETransposeTile_Pseudo<"mtae16.m">; + def BOSC_AME_MTAE32_M_PSEUDO : RVInstBOSCAMETransposeTile_Pseudo<"mtae32.m">; + def BOSC_AME_MTAE64_M_PSEUDO : RVInstBOSCAMETransposeTile_Pseudo<"mtae64.m">; + + def BOSC_AME_MTBE8_M_PSEUDO : RVInstBOSCAMETransposeTile_Pseudo<"mtbe8.m">; + def BOSC_AME_MTBE16_M_PSEUDO : RVInstBOSCAMETransposeTile_Pseudo<"mtbe16.m">; + def BOSC_AME_MTBE32_M_PSEUDO : RVInstBOSCAMETransposeTile_Pseudo<"mtbe32.m">; + def BOSC_AME_MTBE64_M_PSEUDO : RVInstBOSCAMETransposeTile_Pseudo<"mtbe64.m">; + + def BOSC_AME_MTCE8_M_PSEUDO : RVInstBOSCAMETransposeAcc_Pseudo<"mtce8.m">; + def BOSC_AME_MTCE16_M_PSEUDO : RVInstBOSCAMETransposeAcc_Pseudo<"mtce16.m">; + def BOSC_AME_MTCE32_M_PSEUDO : RVInstBOSCAMETransposeAcc_Pseudo<"mtce32.m">; + def BOSC_AME_MTCE64_M_PSEUDO : RVInstBOSCAMETransposeAcc_Pseudo<"mtce64.m">; +} + +// Pseudo element-wise instruction +// Base Pseudo class for 3-operand element-wise instructions (e.g., md, ms1, ms2) +class RVInstBOSCAMEEleWise3_Pseudo + : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, BOSCAMEAccIndex:$ms1, BOSCAMEAccIndex:$ms2), []> { + let AsmString = opcodestr # "\tacc$md, acc$ms1, acc$ms2"; +} + +// Base Pseudo class for 2-operand element-wise instructions (e.g., md, ms1 for sqrt) +class RVInstBOSCAMEEleWise2_Pseudo + : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, BOSCAMEAccIndex:$ms1), []> { + let AsmString = opcodestr # "\tacc$md, acc$ms1"; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 1, isCodeGenOnly = 1 in { + //===--------------------------------------------------------------------===// + // 1. ADD / SUB - No Widen + //===--------------------------------------------------------------------===// + def BOSC_AME_MADDU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"maddu.mm">; + def BOSC_AME_MADDU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"maddu.hb.mm">; + def BOSC_AME_MADDU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"maddu.b.mm">; + def BOSC_AME_MADDU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"maddu.h.mm">; + def BOSC_AME_MADDU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"maddu.w.mm">; + def BOSC_AME_MADDU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"maddu.dw.mm">; + + def BOSC_AME_MSADDU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msaddu.mm">; + def BOSC_AME_MSADDU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msaddu.hb.mm">; + def BOSC_AME_MSADDU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msaddu.b.mm">; + def BOSC_AME_MSADDU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msaddu.h.mm">; + def BOSC_AME_MSADDU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msaddu.w.mm">; + def BOSC_AME_MSADDU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msaddu.dw.mm">; + + def BOSC_AME_MADD_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"madd.mm">; + def BOSC_AME_MADD_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"madd.hb.mm">; + def BOSC_AME_MADD_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"madd.b.mm">; + def BOSC_AME_MADD_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"madd.h.mm">; + def BOSC_AME_MADD_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"madd.w.mm">; + def BOSC_AME_MADD_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"madd.dw.mm">; + + def BOSC_AME_MSADD_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msadd.mm">; + def BOSC_AME_MSADD_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msadd.hb.mm">; + def BOSC_AME_MSADD_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msadd.b.mm">; + def BOSC_AME_MSADD_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msadd.h.mm">; + def BOSC_AME_MSADD_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msadd.w.mm">; + def BOSC_AME_MSADD_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msadd.dw.mm">; + + def BOSC_AME_MFADD_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfadd.mm">; + def BOSC_AME_MFADD_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfadd.cf.mm">; + def BOSC_AME_MFADD_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfadd.hf.mm">; + def BOSC_AME_MFADD_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfadd.f.mm">; + def BOSC_AME_MFADD_D_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfadd.d.mm">; + + def BOSC_AME_MSUBU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msubu.mm">; + def BOSC_AME_MSUBU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msubu.hb.mm">; + def BOSC_AME_MSUBU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msubu.b.mm">; + def BOSC_AME_MSUBU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msubu.h.mm">; + def BOSC_AME_MSUBU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msubu.w.mm">; + def BOSC_AME_MSUBU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msubu.dw.mm">; + + def BOSC_AME_MSSUBU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssubu.mm">; + def BOSC_AME_MSSUBU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssubu.hb.mm">; + def BOSC_AME_MSSUBU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssubu.b.mm">; + def BOSC_AME_MSSUBU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssubu.h.mm">; + def BOSC_AME_MSSUBU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssubu.w.mm">; + def BOSC_AME_MSSUBU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssubu.dw.mm">; + + def BOSC_AME_MSUB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msub.mm">; + def BOSC_AME_MSUB_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msub.hb.mm">; + def BOSC_AME_MSUB_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msub.b.mm">; + def BOSC_AME_MSUB_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msub.h.mm">; + def BOSC_AME_MSUB_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msub.w.mm">; + def BOSC_AME_MSUB_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msub.dw.mm">; + + def BOSC_AME_MSSUB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssub.mm">; + def BOSC_AME_MSSUB_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssub.hb.mm">; + def BOSC_AME_MSSUB_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssub.b.mm">; + def BOSC_AME_MSSUB_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssub.h.mm">; + def BOSC_AME_MSSUB_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssub.w.mm">; + def BOSC_AME_MSSUB_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mssub.dw.mm">; + + def BOSC_AME_MFSUB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfsub.mm">; + def BOSC_AME_MFSUB_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfsub.cf.mm">; + def BOSC_AME_MFSUB_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfsub.hf.mm">; + def BOSC_AME_MFSUB_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfsub.f.mm">; + def BOSC_AME_MFSUB_D_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfsub.d.mm">; + //===--------------------------------------------------------------------===// + // 2. ADD / SUB - Double Widen + //===--------------------------------------------------------------------===// + def BOSC_AME_MWADDU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwaddu.mm">; + def BOSC_AME_MWADDU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwaddu.hb.mm">; + def BOSC_AME_MWADDU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwaddu.b.mm">; + def BOSC_AME_MWADDU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwaddu.h.mm">; + def BOSC_AME_MWADDU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwaddu.w.mm">; + + def BOSC_AME_MWADD_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwadd.mm">; + def BOSC_AME_MWADD_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwadd.hb.mm">; + def BOSC_AME_MWADD_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwadd.b.mm">; + def BOSC_AME_MWADD_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwadd.h.mm">; + def BOSC_AME_MWADD_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwadd.w.mm">; + + def BOSC_AME_MFWADD_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwadd.mm">; + def BOSC_AME_MFWADD_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwadd.cf.mm">; + def BOSC_AME_MFWADD_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwadd.hf.mm">; + def BOSC_AME_MFWADD_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwadd.f.mm">; + + def BOSC_AME_MWSUBU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsubu.mm">; + def BOSC_AME_MWSUBU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsubu.hb.mm">; + def BOSC_AME_MWSUBU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsubu.b.mm">; + def BOSC_AME_MWSUBU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsubu.h.mm">; + def BOSC_AME_MWSUBU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsubu.w.mm">; + + def BOSC_AME_MWSUB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsub.mm">; + def BOSC_AME_MWSUB_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsub.hb.mm">; + def BOSC_AME_MWSUB_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsub.b.mm">; + def BOSC_AME_MWSUB_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsub.h.mm">; + def BOSC_AME_MWSUB_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwsub.w.mm">; + + def BOSC_AME_MFWSUB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwsub.mm">; + def BOSC_AME_MFWSUB_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwsub.cf.mm">; + def BOSC_AME_MFWSUB_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwsub.hf.mm">; + def BOSC_AME_MFWSUB_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwsub.f.mm">; + //===--------------------------------------------------------------------===// + // 3. MIN / MAX - No Widen + //===--------------------------------------------------------------------===// + def BOSC_AME_MMINU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mminu.mm">; + def BOSC_AME_MMINU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mminu.hb.mm">; + def BOSC_AME_MMINU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mminu.b.mm">; + def BOSC_AME_MMINU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mminu.h.mm">; + def BOSC_AME_MMINU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mminu.w.mm">; + def BOSC_AME_MMINU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mminu.dw.mm">; + + def BOSC_AME_MMAXU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmaxu.mm">; + def BOSC_AME_MMAXU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmaxu.hb.mm">; + def BOSC_AME_MMAXU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmaxu.b.mm">; + def BOSC_AME_MMAXU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmaxu.h.mm">; + def BOSC_AME_MMAXU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmaxu.w.mm">; + def BOSC_AME_MMAXU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmaxu.dw.mm">; + + def BOSC_AME_MMIN_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmin.mm">; + def BOSC_AME_MMIN_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmin.hb.mm">; + def BOSC_AME_MMIN_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmin.b.mm">; + def BOSC_AME_MMIN_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmin.h.mm">; + def BOSC_AME_MMIN_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmin.w.mm">; + def BOSC_AME_MMIN_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmin.dw.mm">; + + def BOSC_AME_MMAX_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmax.mm">; + def BOSC_AME_MMAX_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmax.hb.mm">; + def BOSC_AME_MMAX_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmax.b.mm">; + def BOSC_AME_MMAX_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmax.h.mm">; + def BOSC_AME_MMAX_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmax.w.mm">; + def BOSC_AME_MMAX_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmax.dw.mm">; + + def BOSC_AME_MFMIN_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmin.mm">; + def BOSC_AME_MFMIN_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmin.cf.mm">; + def BOSC_AME_MFMIN_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmin.hf.mm">; + def BOSC_AME_MFMIN_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmin.f.mm">; + def BOSC_AME_MFMIN_D_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmin.d.mm">; + + def BOSC_AME_MFMAX_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmax.mm">; + def BOSC_AME_MFMAX_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmax.cf.mm">; + def BOSC_AME_MFMAX_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmax.hf.mm">; + def BOSC_AME_MFMAX_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmax.f.mm">; + def BOSC_AME_MFMAX_D_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmax.d.mm">; + //===--------------------------------------------------------------------===// + // 4. MUL / MULH / DIV - No Widen + //===--------------------------------------------------------------------===// + def BOSC_AME_MSMULU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulu.mm">; + def BOSC_AME_MSMULU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulu.hb.mm">; + def BOSC_AME_MSMULU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulu.b.mm">; + def BOSC_AME_MSMULU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulu.h.mm">; + def BOSC_AME_MSMULU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulu.w.mm">; + def BOSC_AME_MSMULU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulu.dw.mm">; + + def BOSC_AME_MMUL_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmul.mm">; + def BOSC_AME_MMUL_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmul.hb.mm">; + def BOSC_AME_MMUL_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmul.b.mm">; + def BOSC_AME_MMUL_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmul.h.mm">; + def BOSC_AME_MMUL_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmul.w.mm">; + def BOSC_AME_MMUL_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmul.dw.mm">; + + def BOSC_AME_MSMUL_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmul.mm">; + def BOSC_AME_MSMUL_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmul.hb.mm">; + def BOSC_AME_MSMUL_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmul.b.mm">; + def BOSC_AME_MSMUL_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmul.h.mm">; + def BOSC_AME_MSMUL_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmul.w.mm">; + def BOSC_AME_MSMUL_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmul.dw.mm">; + + def BOSC_AME_MFMUL_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmul.mm">; + def BOSC_AME_MFMUL_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmul.cf.mm">; + def BOSC_AME_MFMUL_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmul.hf.mm">; + def BOSC_AME_MFMUL_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmul.f.mm">; + def BOSC_AME_MFMUL_D_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfmul.d.mm">; + + def BOSC_AME_MMULHU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhu.mm">; + def BOSC_AME_MMULHU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhu.hb.mm">; + def BOSC_AME_MMULHU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhu.b.mm">; + def BOSC_AME_MMULHU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhu.h.mm">; + def BOSC_AME_MMULHU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhu.w.mm">; + def BOSC_AME_MMULHU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhu.dw.mm">; + + def BOSC_AME_MMULH_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulh.mm">; + def BOSC_AME_MMULH_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulh.hb.mm">; + def BOSC_AME_MMULH_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulh.b.mm">; + def BOSC_AME_MMULH_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulh.h.mm">; + def BOSC_AME_MMULH_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulh.w.mm">; + def BOSC_AME_MMULH_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulh.dw.mm">; + + def BOSC_AME_MMULHSU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhsu.mm">; + def BOSC_AME_MMULHSU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhsu.hb.mm">; + def BOSC_AME_MMULHSU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhsu.b.mm">; + def BOSC_AME_MMULHSU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhsu.h.mm">; + def BOSC_AME_MMULHSU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhsu.w.mm">; + def BOSC_AME_MMULHSU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mmulhsu.dw.mm">; + + def BOSC_AME_MSMULSU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulsu.mm">; + def BOSC_AME_MSMULSU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulsu.hb.mm">; + def BOSC_AME_MSMULSU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulsu.b.mm">; + def BOSC_AME_MSMULSU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulsu.h.mm">; + def BOSC_AME_MSMULSU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulsu.w.mm">; + def BOSC_AME_MSMULSU_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msmulsu.dw.mm">; + + def BOSC_AME_MFDIV_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfdiv.mm">; + def BOSC_AME_MFDIV_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfdiv.cf.mm">; + def BOSC_AME_MFDIV_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfdiv.hf.mm">; + def BOSC_AME_MFDIV_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfdiv.f.mm">; + def BOSC_AME_MFDIV_D_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfdiv.d.mm">; + //===--------------------------------------------------------------------===// + // 5. MUL - Double Widen + //===--------------------------------------------------------------------===// + def BOSC_AME_MWMULU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulu.mm">; + def BOSC_AME_MWMULU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulu.hb.mm">; + def BOSC_AME_MWMULU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulu.b.mm">; + def BOSC_AME_MWMULU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulu.h.mm">; + def BOSC_AME_MWMULU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulu.w.mm">; + + def BOSC_AME_MWMUL_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmul.mm">; + def BOSC_AME_MWMUL_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmul.hb.mm">; + def BOSC_AME_MWMUL_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmul.b.mm">; + def BOSC_AME_MWMUL_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmul.h.mm">; + def BOSC_AME_MWMUL_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmul.w.mm">; + + def BOSC_AME_MWMULSU_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulsu.mm">; + def BOSC_AME_MWMULSU_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulsu.hb.mm">; + def BOSC_AME_MWMULSU_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulsu.b.mm">; + def BOSC_AME_MWMULSU_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulsu.h.mm">; + def BOSC_AME_MWMULSU_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mwmulsu.w.mm">; + + def BOSC_AME_MFWMUL_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwmul.mm">; + def BOSC_AME_MFWMUL_CF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwmul.cf.mm">; + def BOSC_AME_MFWMUL_HF_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwmul.hf.mm">; + def BOSC_AME_MFWMUL_F_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mfwmul.f.mm">; + //===--------------------------------------------------------------------===// + // 6. LOGIC & SHIFT - No Widen + //===--------------------------------------------------------------------===// + def BOSC_AME_MAND_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mand.mm">; + def BOSC_AME_MOR_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mor.mm">; + def BOSC_AME_MXOR_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"mxor.mm">; + + def BOSC_AME_MSLL_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msll.mm">; + def BOSC_AME_MSLL_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msll.hb.mm">; + def BOSC_AME_MSLL_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msll.b.mm">; + def BOSC_AME_MSLL_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msll.h.mm">; + def BOSC_AME_MSLL_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msll.w.mm">; + def BOSC_AME_MSLL_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msll.dw.mm">; + + def BOSC_AME_MSRL_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msrl.mm">; + def BOSC_AME_MSRL_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msrl.hb.mm">; + def BOSC_AME_MSRL_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msrl.b.mm">; + def BOSC_AME_MSRL_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msrl.h.mm">; + def BOSC_AME_MSRL_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msrl.w.mm">; + def BOSC_AME_MSRL_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msrl.dw.mm">; + + def BOSC_AME_MSRA_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msra.mm">; + def BOSC_AME_MSRA_HB_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msra.hb.mm">; + def BOSC_AME_MSRA_B_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msra.b.mm">; + def BOSC_AME_MSRA_H_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msra.h.mm">; + def BOSC_AME_MSRA_W_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msra.w.mm">; + def BOSC_AME_MSRA_DW_MM_PSEUDO : RVInstBOSCAMEEleWise3_Pseudo<"msra.dw.mm">; + //===--------------------------------------------------------------------===// + // 7. SQRT - No Widen (Unary, 2-Operand) + //===--------------------------------------------------------------------===// + def BOSC_AME_MFSQRT_MM_PSEUDO : RVInstBOSCAMEEleWise2_Pseudo<"mfsqrt.mm">; + def BOSC_AME_MFSQRT_CF_MM_PSEUDO : RVInstBOSCAMEEleWise2_Pseudo<"mfsqrt.cf.mm">; + def BOSC_AME_MFSQRT_HF_MM_PSEUDO : RVInstBOSCAMEEleWise2_Pseudo<"mfsqrt.hf.mm">; + def BOSC_AME_MFSQRT_F_MM_PSEUDO : RVInstBOSCAMEEleWise2_Pseudo<"mfsqrt.f.mm">; + def BOSC_AME_MFSQRT_D_MM_PSEUDO : RVInstBOSCAMEEleWise2_Pseudo<"mfsqrt.d.mm">; +} + +// Pseudo matrix multiply instruction +// md: BOSCAMEAccReg index (0-7), ms1/ms2: BOSCAMETileReg indices (0-7) +// Base Pseudo class for all MMA instructions +class RVInstBOSCAMEMMA_Pseudo + : Pseudo<(outs), (ins BOSCAMEAccIndex:$md, BOSCAMETileIndex:$ms1, BOSCAMETileIndex:$ms2), []> { + let AsmString = opcodestr # "\tacc$md, tr$ms1, tr$ms2"; +} + +let Predicates = [HasVendorXBOSCAME], hasSideEffects = 1, isCodeGenOnly = 1 in { + //===--------------------------------------------------------------------===// + // 1. No-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // int + def BOSC_AME_MMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mma.mm">; + def BOSC_AME_MMA_H_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mma.h.mm">; + def BOSC_AME_MMA_W_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mma.w.mm">; + def BOSC_AME_MMA_DW_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mma.dw.mm">; + // int, saturated + def BOSC_AME_MSMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msma.mm">; + def BOSC_AME_MSMA_H_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msma.h.mm">; + def BOSC_AME_MSMA_W_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msma.w.mm">; + def BOSC_AME_MSMA_DW_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msma.dw.mm">; + // uint + def BOSC_AME_MMAU_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mmau.mm">; + def BOSC_AME_MMAU_H_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mmau.h.mm">; + def BOSC_AME_MMAU_W_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mmau.w.mm">; + def BOSC_AME_MMAU_DW_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mmau.dw.mm">; + // uint, saturated + def BOSC_AME_MSMAU_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msmau.mm">; + def BOSC_AME_MSMAU_H_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msmau.h.mm">; + def BOSC_AME_MSMAU_W_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msmau.w.mm">; + def BOSC_AME_MSMAU_DW_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msmau.dw.mm">; + // float point + def BOSC_AME_MFMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfma.mm">; + def BOSC_AME_MFMA_HF_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfma.hf.mm">; + def BOSC_AME_MFMA_F_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfma.f.mm">; + def BOSC_AME_MFMA_D_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfma.d.mm">; + //===--------------------------------------------------------------------===// + // 2. Double-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def BOSC_AME_MWMAU_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mwmau.mm">; + def BOSC_AME_MWMAU_H_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mwmau.h.mm">; + def BOSC_AME_MWMAU_W_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mwmau.w.mm">; + // uint, saturated + def BOSC_AME_MSWMAU_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mswmau.mm">; + def BOSC_AME_MSWMAU_H_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mswmau.h.mm">; + def BOSC_AME_MSWMAU_W_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mswmau.w.mm">; + // int + def BOSC_AME_MWMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mwma.mm">; + def BOSC_AME_MWMA_H_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mwma.h.mm">; + def BOSC_AME_MWMA_W_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mwma.w.mm">; + // int, saturated + def BOSC_AME_MSWMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mswma.mm">; + def BOSC_AME_MSWMA_H_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mswma.h.mm">; + def BOSC_AME_MSWMA_W_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mswma.w.mm">; + // float point + def BOSC_AME_MFWMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfwma.mm">; + def BOSC_AME_MFWMA_CF_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfwma.cf.mm">; + def BOSC_AME_MFWMA_HF_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfwma.hf.mm">; + def BOSC_AME_MFWMA_F_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfwma.f.mm">; + //===--------------------------------------------------------------------===// + // 3. Quad-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def BOSC_AME_MQMAU_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mqmau.mm">; + def BOSC_AME_MQMAU_B_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mqmau.b.mm">; + // uint, saturated + def BOSC_AME_MSQMAU_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msqmau.mm">; + def BOSC_AME_MSQMAU_B_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msqmau.b.mm">; + // int + def BOSC_AME_MQMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mqma.mm">; + def BOSC_AME_MQMA_B_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mqma.b.mm">; + // int, saturated + def BOSC_AME_MSQMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msqma.mm">; + def BOSC_AME_MSQMA_B_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msqma.b.mm">; + // float point + def BOSC_AME_MFQMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfqma.mm">; + def BOSC_AME_MFQMA_CF_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"mfqma.cf.mm">; + //===--------------------------------------------------------------------===// + // 4. Oct-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def BOSC_AME_MOMAU_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"momau.mm">; + def BOSC_AME_MOMAU_HB_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"momau.hb.mm">; + // uint, saturated + def BOSC_AME_MSOMAU_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msomau.mm">; + def BOSC_AME_MSOMAU_HB_MM_PSEUDO: RVInstBOSCAMEMMA_Pseudo<"msomau.hb.mm">; + // int + def BOSC_AME_MOMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"moma.mm">; + def BOSC_AME_MOMA_HB_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"moma.hb.mm">; + // int, saturated + def BOSC_AME_MSOMA_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msoma.mm">; + def BOSC_AME_MSOMA_HB_MM_PSEUDO : RVInstBOSCAMEMMA_Pseudo<"msoma.hb.mm">; +} + +//===----------------------------------------------------------------------===// +// Pattern Matching for 32-bit New Instructions +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXBOSCAME] in { + // Configuration immediate instructions + def : Pat<(i64 (int_riscv_bosc_msettypei timm:$imm)), + (BOSC_AME_MSETTYPEI_PSEUDO timm:$imm)>; + def : Pat<(i64 (int_riscv_bosc_msettypehi timm:$imm)), + (BOSC_AME_MSETTYPEHI_PSEUDO timm:$imm)>; + def : Pat<(i64 (int_riscv_bosc_msettilemi timm:$imm)), + (BOSC_AME_MSETTILEMI_PSEUDO timm:$imm)>; + def : Pat<(i64 (int_riscv_bosc_msettileni timm:$imm)), + (BOSC_AME_MSETTILENI_PSEUDO timm:$imm)>; + def : Pat<(i64 (int_riscv_bosc_msettileki timm:$imm)), + (BOSC_AME_MSETTILEKI_PSEUDO timm:$imm)>; + // Register versions + def : Pat<(i64 (int_riscv_bosc_msettype GPR:$rs1)), + (BOSC_AME_MSETTYPE_PSEUDO GPR:$rs1)>; + def : Pat<(i64 (int_riscv_bosc_msettilem GPR:$rs1)), + (BOSC_AME_MSETTILEM_PSEUDO GPR:$rs1)>; + def : Pat<(i64 (int_riscv_bosc_msettilen GPR:$rs1)), + (BOSC_AME_MSETTILEN_PSEUDO GPR:$rs1)>; + def : Pat<(i64 (int_riscv_bosc_msettilek GPR:$rs1)), + (BOSC_AME_MSETTILEK_PSEUDO GPR:$rs1)>; + + // Load instructions + // Matrix A + def : Pat<(int_riscv_bosc_mlae8_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLAE8_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlae16_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLAE16_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlae32_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLAE32_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlae64_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLAE64_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlate8_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLATE8_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlate16_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLATE16_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlate32_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLATE32_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlate64_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLATE64_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mltre8_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLTRE8_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mltre16_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLTRE16_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mltre32_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLTRE32_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mltre64_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLTRE64_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + // Matrix B + def : Pat<(int_riscv_bosc_mlbe8_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLBE8_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlbe16_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLBE16_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlbe32_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLBE32_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlbe64_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLBE64_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlbte8_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLBTE8_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlbte16_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLBTE16_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlbte32_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLBTE32_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlbte64_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLBTE64_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + // Matrix C (Accumulator) + def : Pat<(int_riscv_bosc_mlce8_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLCE8_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlce16_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLCE16_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlce32_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLCE32_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlce64_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLCE64_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlcte8_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLCTE8_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlcte16_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLCTE16_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlcte32_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLCTE32_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlcte64_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLCTE64_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlacce8_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLACCE8_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlacce16_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLACCE16_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlacce32_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLACCE32_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mlacce64_m timm:$md, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MLACCE64_M_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + + // Store instruction + // Matrix A + def : Pat<(int_riscv_bosc_msae8_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSAE8_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msae16_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSAE16_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msae32_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSAE32_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msae64_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSAE64_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msate8_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSATE8_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msate16_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSATE16_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msate32_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSATE32_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msate64_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSATE64_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mstre8_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSTRE8_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mstre16_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSTRE16_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mstre32_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSTRE32_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mstre64_m timm:$ms1, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSTRE64_M_PSEUDO timm:$ms1, GPR:$rs1, GPR:$rs2)>; + // Matrix B + def : Pat<(int_riscv_bosc_msbe8_m timm:$ms2, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSBE8_M_PSEUDO timm:$ms2, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msbe16_m timm:$ms2, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSBE16_M_PSEUDO timm:$ms2, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msbe32_m timm:$ms2, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSBE32_M_PSEUDO timm:$ms2, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msbe64_m timm:$ms2, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSBE64_M_PSEUDO timm:$ms2, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msbte8_m timm:$ms2, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSBTE8_M_PSEUDO timm:$ms2, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msbte16_m timm:$ms2, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSBTE16_M_PSEUDO timm:$ms2, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msbte32_m timm:$ms2, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSBTE32_M_PSEUDO timm:$ms2, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msbte64_m timm:$ms2, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSBTE64_M_PSEUDO timm:$ms2, GPR:$rs1, GPR:$rs2)>; + // Matrix C (Accumulator) + def : Pat<(int_riscv_bosc_msce8_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSCE8_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msce16_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSCE16_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msce32_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSCE32_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msce64_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSCE64_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mscte8_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSCTE8_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mscte16_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSCTE16_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mscte32_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSCTE32_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mscte64_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSCTE64_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msacce8_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSACCE8_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msacce16_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSACCE16_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msacce32_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSACCE32_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_msacce64_m timm:$ms3, iPTR:$rs1, i64:$rs2), + (BOSC_AME_MSACCE64_M_PSEUDO timm:$ms3, GPR:$rs1, GPR:$rs2)>; + + // Data move instructions + //---------------------------------------------------------------------- + // 1. Matrix-Matrix Move Pseudos (Pure Matrix) + //---------------------------------------------------------------------- + def : Pat<(int_riscv_bosc_mmve8_t_t timm:$md, timm:$ms1), + (BOSC_AME_MMVE8_T_T_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mmve16_t_t timm:$md, timm:$ms1), + (BOSC_AME_MMVE16_T_T_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mmve32_t_t timm:$md, timm:$ms1), + (BOSC_AME_MMVE32_T_T_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mmve64_t_t timm:$md, timm:$ms1), + (BOSC_AME_MMVE64_T_T_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mmve8_a_a timm:$md, timm:$ms1), + (BOSC_AME_MMVE8_A_A_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mmve16_a_a timm:$md, timm:$ms1), + (BOSC_AME_MMVE16_A_A_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mmve32_a_a timm:$md, timm:$ms1), + (BOSC_AME_MMVE32_A_A_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mmve64_a_a timm:$md, timm:$ms1), + (BOSC_AME_MMVE64_A_A_PSEUDO timm:$md, timm:$ms1)>; + //---------------------------------------------------------------------- + // 2. Matrix-Matrix Move (with GPR Index) + //---------------------------------------------------------------------- + def : Pat<(int_riscv_bosc_mmve8_a_t timm:$md, timm:$ms1, i64:$rs2), + (BOSC_AME_MMVE8_A_T_PSEUDO timm:$md, timm:$ms1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve16_a_t timm:$md, timm:$ms1, i64:$rs2), + (BOSC_AME_MMVE16_A_T_PSEUDO timm:$md, timm:$ms1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve32_a_t timm:$md, timm:$ms1, i64:$rs2), + (BOSC_AME_MMVE32_A_T_PSEUDO timm:$md, timm:$ms1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve64_a_t timm:$md, timm:$ms1, i64:$rs2), + (BOSC_AME_MMVE64_A_T_PSEUDO timm:$md, timm:$ms1, GPR:$rs2)>; + + def : Pat<(int_riscv_bosc_mmve8_t_a timm:$md, timm:$ms1, i64:$rs2), + (BOSC_AME_MMVE8_T_A_PSEUDO timm:$md, timm:$ms1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve16_t_a timm:$md, timm:$ms1, i64:$rs2), + (BOSC_AME_MMVE16_T_A_PSEUDO timm:$md, timm:$ms1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve32_t_a timm:$md, timm:$ms1, i64:$rs2), + (BOSC_AME_MMVE32_T_A_PSEUDO timm:$md, timm:$ms1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve64_t_a timm:$md, timm:$ms1, i64:$rs2), + (BOSC_AME_MMVE64_T_A_PSEUDO timm:$md, timm:$ms1, GPR:$rs2)>; + //---------------------------------------------------------------------- + // 3. Matrix-Matrix Move (with Immediate Index) + //---------------------------------------------------------------------- + def : Pat<(int_riscv_bosc_mmvie8_a_t timm:$md, timm:$ms1, imm:$imm), + (BOSC_AME_MMVIE8_A_T_PSEUDO timm:$md, timm:$ms1, imm:$imm)>; + def : Pat<(int_riscv_bosc_mmvie16_a_t timm:$md, timm:$ms1, imm:$imm), + (BOSC_AME_MMVIE16_A_T_PSEUDO timm:$md, timm:$ms1, imm:$imm)>; + def : Pat<(int_riscv_bosc_mmvie32_a_t timm:$md, timm:$ms1, imm:$imm), + (BOSC_AME_MMVIE32_A_T_PSEUDO timm:$md, timm:$ms1, imm:$imm)>; + def : Pat<(int_riscv_bosc_mmvie64_a_t timm:$md, timm:$ms1, imm:$imm), + (BOSC_AME_MMVIE64_A_T_PSEUDO timm:$md, timm:$ms1, imm:$imm)>; + + def : Pat<(int_riscv_bosc_mmvie8_t_a timm:$md, timm:$ms1, imm:$imm), + (BOSC_AME_MMVIE8_T_A_PSEUDO timm:$md, timm:$ms1, imm:$imm)>; + def : Pat<(int_riscv_bosc_mmvie16_t_a timm:$md, timm:$ms1, imm:$imm), + (BOSC_AME_MMVIE16_T_A_PSEUDO timm:$md, timm:$ms1, imm:$imm)>; + def : Pat<(int_riscv_bosc_mmvie32_t_a timm:$md, timm:$ms1, imm:$imm), + (BOSC_AME_MMVIE32_T_A_PSEUDO timm:$md, timm:$ms1, imm:$imm)>; + def : Pat<(int_riscv_bosc_mmvie64_t_a timm:$md, timm:$ms1, imm:$imm), + (BOSC_AME_MMVIE64_T_A_PSEUDO timm:$md, timm:$ms1, imm:$imm)>; + //---------------------------------------------------------------------- + // 4. Matrix to Scalar Move (Returns GPR, matches i64 in IR) + //---------------------------------------------------------------------- + def : Pat<(i64 (int_riscv_bosc_mmve8_x_t timm:$ms1, i64:$rs2)), + (BOSC_AME_MMVE8_X_T_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mmve16_x_t timm:$ms1, i64:$rs2)), + (BOSC_AME_MMVE16_X_T_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mmve32_x_t timm:$ms1, i64:$rs2)), + (BOSC_AME_MMVE32_X_T_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mmve64_x_t timm:$ms1, i64:$rs2)), + (BOSC_AME_MMVE64_X_T_PSEUDO timm:$ms1, GPR:$rs2)>; + + def : Pat<(i64 (int_riscv_bosc_mmve8_x_a timm:$ms1, i64:$rs2)), + (BOSC_AME_MMVE8_X_A_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mmve16_x_a timm:$ms1, i64:$rs2)), + (BOSC_AME_MMVE16_X_A_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mmve32_x_a timm:$ms1, i64:$rs2)), + (BOSC_AME_MMVE32_X_A_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mmve64_x_a timm:$ms1, i64:$rs2)), + (BOSC_AME_MMVE64_X_A_PSEUDO timm:$ms1, GPR:$rs2)>; + + def : Pat<(i64 (int_riscv_bosc_mfmve8_f_t timm:$ms1, i64:$rs2)), + (BOSC_AME_MFMVE8_F_T_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mfmve16_f_t timm:$ms1, i64:$rs2)), + (BOSC_AME_MFMVE16_F_T_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mfmve32_f_t timm:$ms1, i64:$rs2)), + (BOSC_AME_MFMVE32_F_T_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mfmve64_f_t timm:$ms1, i64:$rs2)), + (BOSC_AME_MFMVE64_F_T_PSEUDO timm:$ms1, GPR:$rs2)>; + + def : Pat<(i64 (int_riscv_bosc_mfmve8_f_a timm:$ms1, i64:$rs2)), + (BOSC_AME_MFMVE8_F_A_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mfmve16_f_a timm:$ms1, i64:$rs2)), + (BOSC_AME_MFMVE16_F_A_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mfmve32_f_a timm:$ms1, i64:$rs2)), + (BOSC_AME_MFMVE32_F_A_PSEUDO timm:$ms1, GPR:$rs2)>; + def : Pat<(i64 (int_riscv_bosc_mfmve64_f_a timm:$ms1, i64:$rs2)), + (BOSC_AME_MFMVE64_F_A_PSEUDO timm:$ms1, GPR:$rs2)>; + //---------------------------------------------------------------------- + // 5. Scalar GPR to Matrix Move + //---------------------------------------------------------------------- + def : Pat<(int_riscv_bosc_mmve8_t_x timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MMVE8_T_X_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve16_t_x timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MMVE16_T_X_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve32_t_x timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MMVE32_T_X_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve64_t_x timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MMVE64_T_X_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + + def : Pat<(int_riscv_bosc_mmve8_a_x timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MMVE8_A_X_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve16_a_x timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MMVE16_A_X_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve32_a_x timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MMVE32_A_X_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mmve64_a_x timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MMVE64_A_X_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + + def : Pat<(int_riscv_bosc_mfmve8_t_f timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MFMVE8_T_F_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mfmve16_t_f timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MFMVE16_T_F_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mfmve32_t_f timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MFMVE32_T_F_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mfmve64_t_f timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MFMVE64_T_F_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + + def : Pat<(int_riscv_bosc_mfmve8_a_f timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MFMVE8_A_F_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mfmve16_a_f timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MFMVE16_A_F_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mfmve32_a_f timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MFMVE32_A_F_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + def : Pat<(int_riscv_bosc_mfmve64_a_f timm:$md, i64:$rs1, i64:$rs2), + (BOSC_AME_MFMVE64_A_F_PSEUDO timm:$md, GPR:$rs1, GPR:$rs2)>; + //---------------------------------------------------------------------- + // 6. Broadcast Pseudos + //---------------------------------------------------------------------- + def : Pat<(int_riscv_bosc_mbcar_m timm:$md, timm:$ms1), + (BOSC_AME_MBCAR_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcbr_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBR_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbccr_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCR_M_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mbcace8_m timm:$md, timm:$ms1), + (BOSC_AME_MBCACE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcace16_m timm:$md, timm:$ms1), + (BOSC_AME_MBCACE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcace32_m timm:$md, timm:$ms1), + (BOSC_AME_MBCACE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcace64_m timm:$md, timm:$ms1), + (BOSC_AME_MBCACE64_M_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mbcbce8_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBCE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcbce16_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBCE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcbce32_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBCE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcbce64_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBCE64_M_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mbccce8_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCCE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbccce16_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCCE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbccce32_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCCE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbccce64_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCCE64_M_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mbcaee8_m timm:$md, timm:$ms1), + (BOSC_AME_MBCAEE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcaee16_m timm:$md, timm:$ms1), + (BOSC_AME_MBCAEE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcaee32_m timm:$md, timm:$ms1), + (BOSC_AME_MBCAEE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcaee64_m timm:$md, timm:$ms1), + (BOSC_AME_MBCAEE64_M_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mbcbee8_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBEE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcbee16_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBEE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcbee32_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBEE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbcbee64_m timm:$md, timm:$ms1), + (BOSC_AME_MBCBEE64_M_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mbccee8_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCEE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbccee16_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCEE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbccee32_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCEE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mbccee64_m timm:$md, timm:$ms1), + (BOSC_AME_MBCCEE64_M_PSEUDO timm:$md, timm:$ms1)>; + //---------------------------------------------------------------------- + // 7. Transpose Pseudos + //---------------------------------------------------------------------- + def : Pat<(int_riscv_bosc_mtae8_m timm:$md, timm:$ms1), + (BOSC_AME_MTAE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtae16_m timm:$md, timm:$ms1), + (BOSC_AME_MTAE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtae32_m timm:$md, timm:$ms1), + (BOSC_AME_MTAE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtae64_m timm:$md, timm:$ms1), + (BOSC_AME_MTAE64_M_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mtbe8_m timm:$md, timm:$ms1), + (BOSC_AME_MTBE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtbe16_m timm:$md, timm:$ms1), + (BOSC_AME_MTBE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtbe32_m timm:$md, timm:$ms1), + (BOSC_AME_MTBE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtbe64_m timm:$md, timm:$ms1), + (BOSC_AME_MTBE64_M_PSEUDO timm:$md, timm:$ms1)>; + + def : Pat<(int_riscv_bosc_mtce8_m timm:$md, timm:$ms1), + (BOSC_AME_MTCE8_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtce16_m timm:$md, timm:$ms1), + (BOSC_AME_MTCE16_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtce32_m timm:$md, timm:$ms1), + (BOSC_AME_MTCE32_M_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mtce64_m timm:$md, timm:$ms1), + (BOSC_AME_MTCE64_M_PSEUDO timm:$md, timm:$ms1)>; + + // Element-wise subtract instruction + //===--------------------------------------------------------------------===// + // 1. ADD / SUB - No Widen + //===--------------------------------------------------------------------===// + def : Pat<(int_riscv_bosc_maddu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADDU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_maddu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADDU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_maddu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADDU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_maddu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADDU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_maddu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADDU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_maddu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADDU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msaddu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADDU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msaddu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADDU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msaddu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADDU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msaddu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADDU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msaddu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADDU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msaddu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADDU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_madd_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADD_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_madd_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADD_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_madd_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADD_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_madd_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADD_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_madd_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADD_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_madd_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MADD_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msadd_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADD_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msadd_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADD_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msadd_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADD_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msadd_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADD_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msadd_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADD_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msadd_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSADD_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfadd_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFADD_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfadd_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFADD_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfadd_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFADD_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfadd_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFADD_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfadd_d_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFADD_D_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msubu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUBU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msubu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUBU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msubu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUBU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msubu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUBU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msubu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUBU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msubu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUBU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mssubu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUBU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssubu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUBU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssubu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUBU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssubu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUBU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssubu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUBU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssubu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUBU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msub_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msub_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUB_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msub_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUB_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msub_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUB_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msub_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUB_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msub_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSUB_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mssub_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssub_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUB_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssub_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUB_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssub_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUB_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssub_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUB_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mssub_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSSUB_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfsub_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFSUB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfsub_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFSUB_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfsub_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFSUB_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfsub_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFSUB_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfsub_d_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFSUB_D_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 2. ADD / SUB - Double Widen + //===--------------------------------------------------------------------===// + def : Pat<(int_riscv_bosc_mwaddu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADDU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwaddu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADDU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwaddu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADDU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwaddu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADDU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwaddu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADDU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mwadd_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADD_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwadd_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADD_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwadd_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADD_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwadd_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADD_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwadd_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWADD_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfwadd_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWADD_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwadd_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWADD_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwadd_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWADD_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwadd_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWADD_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mwsubu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUBU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwsubu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUBU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwsubu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUBU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwsubu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUBU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwsubu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUBU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mwsub_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwsub_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUB_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwsub_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUB_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwsub_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUB_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwsub_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWSUB_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfwsub_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWSUB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwsub_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWSUB_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwsub_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWSUB_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwsub_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWSUB_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 3. MIN / MAX - No Widen + //===--------------------------------------------------------------------===// + def : Pat<(int_riscv_bosc_mminu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMINU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mminu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMINU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mminu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMINU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mminu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMINU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mminu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMINU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mminu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMINU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mmaxu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAXU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmaxu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAXU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmaxu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAXU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmaxu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAXU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmaxu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAXU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmaxu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAXU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mmin_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMIN_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmin_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMIN_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmin_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMIN_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmin_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMIN_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmin_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMIN_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmin_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMIN_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mmax_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAX_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmax_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAX_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmax_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAX_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmax_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAX_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmax_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAX_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmax_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAX_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfmin_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMIN_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmin_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMIN_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmin_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMIN_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmin_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMIN_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmin_d_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMIN_D_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfmax_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMAX_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmax_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMAX_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmax_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMAX_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmax_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMAX_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmax_d_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMAX_D_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 4. MUL / MULH / DIV - No Widen + //===--------------------------------------------------------------------===// + def : Pat<(int_riscv_bosc_msmulu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mmul_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMUL_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmul_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMUL_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmul_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMUL_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmul_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMUL_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmul_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMUL_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmul_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMUL_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msmul_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMUL_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmul_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMUL_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmul_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMUL_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmul_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMUL_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmul_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMUL_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmul_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMUL_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfmul_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMUL_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmul_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMUL_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmul_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMUL_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmul_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMUL_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfmul_d_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMUL_D_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mmulhu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mmulh_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULH_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulh_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULH_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulh_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULH_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulh_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULH_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulh_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULH_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulh_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULH_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mmulhsu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHSU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhsu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHSU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhsu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHSU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhsu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHSU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhsu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHSU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmulhsu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMULHSU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msmulsu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULSU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulsu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULSU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulsu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULSU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulsu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULSU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulsu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULSU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmulsu_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMULSU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfdiv_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFDIV_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfdiv_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFDIV_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfdiv_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFDIV_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfdiv_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFDIV_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfdiv_d_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFDIV_D_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 5. MUL - Double Widen + //===--------------------------------------------------------------------===// + def : Pat<(int_riscv_bosc_mwmulu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmulu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmulu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmulu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmulu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mwmul_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMUL_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmul_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMUL_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmul_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMUL_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmul_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMUL_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmul_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMUL_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mwmulsu_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULSU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmulsu_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULSU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmulsu_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULSU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmulsu_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULSU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmulsu_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMULSU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_mfwmul_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWMUL_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwmul_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWMUL_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwmul_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWMUL_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwmul_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWMUL_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 6. LOGIC & SHIFT - No Widen + //===--------------------------------------------------------------------===// + def : Pat<(int_riscv_bosc_mand_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MAND_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mor_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MOR_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mxor_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MXOR_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msll_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSLL_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msll_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSLL_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msll_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSLL_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msll_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSLL_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msll_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSLL_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msll_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSLL_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msrl_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRL_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msrl_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRL_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msrl_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRL_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msrl_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRL_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msrl_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRL_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msrl_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRL_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + + def : Pat<(int_riscv_bosc_msra_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msra_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRA_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msra_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRA_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msra_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRA_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msra_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRA_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msra_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSRA_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 7. SQRT - No Widen (Unary, 2-Operand) + //===--------------------------------------------------------------------===// + def : Pat<(int_riscv_bosc_mfsqrt_mm timm:$md, timm:$ms1), + (BOSC_AME_MFSQRT_MM_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mfsqrt_cf_mm timm:$md, timm:$ms1), + (BOSC_AME_MFSQRT_CF_MM_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mfsqrt_hf_mm timm:$md, timm:$ms1), + (BOSC_AME_MFSQRT_HF_MM_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mfsqrt_f_mm timm:$md, timm:$ms1), + (BOSC_AME_MFSQRT_F_MM_PSEUDO timm:$md, timm:$ms1)>; + def : Pat<(int_riscv_bosc_mfsqrt_d_mm timm:$md, timm:$ms1), + (BOSC_AME_MFSQRT_D_MM_PSEUDO timm:$md, timm:$ms1)>; + + // Matrix multiply instruction + //===--------------------------------------------------------------------===// + // 1. No-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // int + def : Pat<(int_riscv_bosc_mma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mma_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMA_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mma_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMA_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mma_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMA_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // int, saturated + def : Pat<(int_riscv_bosc_msma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msma_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMA_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msma_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMA_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msma_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMA_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // uint + def : Pat<(int_riscv_bosc_mmau_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmau_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmau_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mmau_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MMAU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // uint, saturated + def : Pat<(int_riscv_bosc_msmau_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMAU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmau_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMAU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmau_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMAU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msmau_dw_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSMAU_DW_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // float point + def : Pat<(int_riscv_bosc_mfma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfma_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMA_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfma_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMA_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfma_d_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFMA_D_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 2. Double-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def : Pat<(int_riscv_bosc_mwmau_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMAU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmau_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMAU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwmau_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMAU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // uint, saturated + def : Pat<(int_riscv_bosc_mswmau_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSWMAU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mswmau_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSWMAU_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mswmau_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSWMAU_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // int + def : Pat<(int_riscv_bosc_mwma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwma_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMA_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mwma_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MWMA_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // int, saturated + def : Pat<(int_riscv_bosc_mswma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSWMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mswma_h_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSWMA_H_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mswma_w_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSWMA_W_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // float point + def : Pat<(int_riscv_bosc_mfwma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwma_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWMA_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwma_hf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWMA_HF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfwma_f_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFWMA_F_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 3. Quad-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def : Pat<(int_riscv_bosc_mqmau_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MQMAU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mqmau_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MQMAU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // uint, saturated + def : Pat<(int_riscv_bosc_msqmau_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSQMAU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msqmau_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSQMAU_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // int + def : Pat<(int_riscv_bosc_mqma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MQMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mqma_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MQMA_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // int, saturated + def : Pat<(int_riscv_bosc_msqma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSQMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msqma_b_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSQMA_B_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // float point + def : Pat<(int_riscv_bosc_mfqma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFQMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_mfqma_cf_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MFQMA_CF_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + //===--------------------------------------------------------------------===// + // 4. Oct-widen matrix multiply-accumulate + //===--------------------------------------------------------------------===// + // uint + def : Pat<(int_riscv_bosc_momau_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MOMAU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_momau_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MOMAU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // uint, saturated + def : Pat<(int_riscv_bosc_msomau_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSOMAU_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msomau_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSOMAU_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // int + def : Pat<(int_riscv_bosc_moma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MOMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_moma_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MOMA_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + // int, saturated + def : Pat<(int_riscv_bosc_msoma_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSOMA_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; + def : Pat<(int_riscv_bosc_msoma_hb_mm timm:$md, timm:$ms1, timm:$ms2), + (BOSC_AME_MSOMA_HB_MM_PSEUDO timm:$md, timm:$ms1, timm:$ms2)>; +} diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXBuckyballExt.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXBuckyballExt.td new file mode 100644 index 0000000000000..853adaa4dd101 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXBuckyballExt.td @@ -0,0 +1,68 @@ +//===- RISCVInstrInfoXBuckyballExt.td -------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +// Buckyball base instructions shared by derived kernels. +// +// Buckyball uses different values as funct7. +// Buckyball uses 0x3 (0b011) as funct3. +// Buckyball uses OPC_CUSTOM_3. + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, + Predicates = [HasVendorXBuckyballBase] in +def BB_MVIN : RVInstR<0b0100001, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_mvin", "$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, + Predicates = [HasVendorXBuckyballBase] in +def BB_MVOUT : RVInstR<0b0010000, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_mvout", "$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, + Predicates = [HasVendorXBuckyballBase] in +def BB_MSET : RVInstR<0b0100000, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_mset", "$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, + Predicates = [HasVendorXBuckyballBase] in +def BB_FENCE : RVInstR<0b0000000, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "bb_fence", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXBuckyballBase] in +def : Pat<(int_riscv_bb_mvin GPR:$rs1, GPR:$rs2), + (BB_MVIN GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXBuckyballBase] in +def : Pat<(int_riscv_bb_mvout GPR:$rs1, GPR:$rs2), + (BB_MVOUT GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXBuckyballBase] in +def : Pat<(int_riscv_bb_mset GPR:$rs1, GPR:$rs2), + (BB_MSET GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXBuckyballBase] in +def : Pat<(int_riscv_bb_fence GPR:$rs1, GPR:$rs2), + (BB_FENCE GPR:$rs1, GPR:$rs2)>; + +include "RISCVInstrInfoXBBPebble.td" +include "RISCVInstrInfoXBBMeteor.td" diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXGemmini.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXGemmini.td new file mode 100644 index 0000000000000..f5aa5c4470714 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXGemmini.td @@ -0,0 +1,226 @@ +//===- RISCVInstrInfoXGemmini.td --------------------------------------*- tablegen -*-===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// Gemmini RoCC-style extension instructions. +// +//===----------------------------------------------------------------------===// + +// Gemmini defines different values as func7 +// - https://github.com/ucb-bar/gemmini-rocc-tests/blob/e326e7c43457ff08669fe88edcaa395d846474d8/include/gemmini.h#L25 + +// Gemmini uses 0x3 (0b011) as func3 +// - https://github.com/IBM/rocc-software/blob/fddb795a0b52e82f8f4ce9ead9b1428440a62ab0/src/xcustom.h#L147 + +// Gemmini uses OPC_CUSTOM_3 +// - https://github.com/IBM/rocc-software/blob/fddb795a0b52e82f8f4ce9ead9b1428440a62ab0/src/xcustom.h#L123 + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, Predicates = [HasVendorXGemmini] in +def MVIN : RVInstR<0b0000010, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "mvin","$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, Predicates = [HasVendorXGemmini] in +def MVIN2 : RVInstR<0b0000001, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "mvin2","$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, Predicates = [HasVendorXGemmini] in +def MVIN3 : RVInstR<0b0001110, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "mvin3","$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, Predicates = [HasVendorXGemmini] in +def MVOUT : RVInstR<0b0000011, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "mvout","$rs1, $rs2">{ + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore = 1, Predicates = [HasVendorXGemmini] in +def FLUSH : RVInstR<0b0000111, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "flush", "$rs1"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def CONFIG : RVInstR<0b0000000, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "config", "$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore =1, Predicates = [HasVendorXGemmini] in +def PRELOAD : RVInstR<0b0000110, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "preload", "$rs1, $rs2">{ + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def COMPUTE_PRELOADED : RVInstR<0b0000100, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "compute_preloaded", "$rs1, $rs2">{ + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def COMPUTE_ACCUMULATED : RVInstR<0b0000101, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "compute_accumulated", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_WS_CONFIG_BOUNDS : RVInstR<0b0001001, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_ws_config_bounds","$rs1, $rs2">{ + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_WS_CONFIG_ADDRS_AB : RVInstR<0b0001010, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_ws_config_addrs_ab", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_WS_CONFIG_ADDRS_DC : RVInstR<0b0001011, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_ws_config_addrs_dc", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_WS_CONFIG_STRIDES_AB : RVInstR<0b0001100, 0b011, OPC_CUSTOM_3,(outs), + (ins GPR:$rs1, GPR:$rs2), "loop_ws_config_strides_ab", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_WS_CONFIG_STRIDES_DC : RVInstR<0b0001101, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_ws_config_strides_dc", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_WS : RVInstR<0b0001000, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_ws", "$rs1, $rs2"> { + let rd = 0; +} + +let hasSideEffects = 1, mayLoad = 1, mayStore =1, Predicates = [HasVendorXGemmini] in +def LOOP_CONV_WS : RVInstR<0b0001111, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_conv_ws", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_CONV_WS_CONFIG1 : RVInstR<0b0010000, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_conv_ws_config1", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_CONV_WS_CONFIG2 : RVInstR<0b0010001, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_conv_ws_config2", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_CONV_WS_CONFIG3 : RVInstR<0b0010010, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_conv_ws_config3", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_CONV_WS_CONFIG4 : RVInstR<0b0010011, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_conv_ws_config4", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_CONV_WS_CONFIG5 : RVInstR<0b0010100, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_conv_ws_config5", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def LOOP_CONV_WS_CONFIG6 : RVInstR<0b0010101, 0b011, OPC_CUSTOM_3, (outs), + (ins GPR:$rs1, GPR:$rs2), "loop_conv_ws_config6", "$rs1, $rs2"> { + let rd = 0; +} + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_mvin GPR:$rs1, GPR:$rs2), (MVIN GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_mvin2 GPR:$rs1, GPR:$rs2), (MVIN2 GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_mvin3 GPR:$rs1, GPR:$rs2), (MVIN3 GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_mvout GPR:$rs1, GPR:$rs2), (MVOUT GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_flush GPR:$rs1, GPR:$rs2), (FLUSH GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_config GPR:$rs1, GPR:$rs2), (CONFIG GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_preload GPR:$rs1, GPR:$rs2), (PRELOAD GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_compute_preloaded GPR:$rs1, GPR:$rs2), (COMPUTE_PRELOADED GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_compute_accumulated GPR:$rs1, GPR:$rs2), (COMPUTE_ACCUMULATED GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_ws_config_bounds GPR:$rs1, GPR:$rs2), (LOOP_WS_CONFIG_BOUNDS GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_ws_config_addrs_ab GPR:$rs1, GPR:$rs2), (LOOP_WS_CONFIG_ADDRS_AB GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_ws_config_addrs_dc GPR:$rs1, GPR:$rs2), (LOOP_WS_CONFIG_ADDRS_DC GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_ws_config_strides_ab GPR:$rs1, GPR:$rs2), (LOOP_WS_CONFIG_STRIDES_AB GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_ws_config_strides_dc GPR:$rs1, GPR:$rs2), (LOOP_WS_CONFIG_STRIDES_DC GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_ws GPR:$rs1, GPR:$rs2), (LOOP_WS GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_conv_ws GPR:$rs1, GPR:$rs2), (LOOP_CONV_WS GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_conv_ws_config1 GPR:$rs1, GPR:$rs2), (LOOP_CONV_WS_CONFIG1 GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_conv_ws_config2 GPR:$rs1, GPR:$rs2), (LOOP_CONV_WS_CONFIG2 GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_conv_ws_config3 GPR:$rs1, GPR:$rs2), (LOOP_CONV_WS_CONFIG3 GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_conv_ws_config4 GPR:$rs1, GPR:$rs2), (LOOP_CONV_WS_CONFIG4 GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_conv_ws_config5 GPR:$rs1, GPR:$rs2), (LOOP_CONV_WS_CONFIG5 GPR:$rs1, GPR:$rs2)>; + +let Predicates = [HasVendorXGemmini] in +def : Pat<(int_riscv_loop_conv_ws_config6 GPR:$rs1, GPR:$rs2), (LOOP_CONV_WS_CONFIG6 GPR:$rs1, GPR:$rs2)>; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSpacemiTIME.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSpacemiTIME.td new file mode 100644 index 0000000000000..98843fc4a448c --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSpacemiTIME.td @@ -0,0 +1,406 @@ +//===- RISCVInstrInfoXSpacemiTIME.td -----------------------------------*- tablegen -*-===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// SpacemiT IME vector dot-product extension instructions. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// IME Extension Instructions +//===----------------------------------------------------------------------===// + +class RVInstIME funct7, bits<3> funct3, dag outs, dag ins, + string opcodestr, string argstr> + : RVInst { + bits<5> vs2; + bits<5> vs1; + bits<5> vd; + + let Inst{31-25} = funct7; + let Inst{24-20} = vs2; + let Inst{19-15} = vs1; + let Inst{14-12} = funct3; + let Inst{11-7} = vd; + let Inst{6-0} = OPC_CUSTOM_1.Value; + + let Uses = [VTYPE, VL]; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT : RVInstIME<0b1110001, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM4:$vs1, VRM4:$vs2), + "vmadot", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOTU : RVInstIME<0b1110001, 0b011, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM4:$vs1, VRM4:$vs2), + "vmadotu", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOTSU : RVInstIME<0b1110001, 0b001, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM4:$vs1, VRM4:$vs2), + "vmadotsu", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOTUS : RVInstIME<0b1110001, 0b010, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM4:$vs1, VRM4:$vs2), + "vmadotus", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VFMADOT : RVInstIME<0b1110101, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM4:$vs1, VRM4:$vs2), + "vfmadot", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +//===----------------------------------------------------------------------===// +// IME Sliding-Window Instructions +//===----------------------------------------------------------------------===// + +// Integer slide-1 instructions +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT1 : RVInstIME<0b1110010, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot1", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT1U : RVInstIME<0b1110010, 0b011, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot1u", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT1SU : RVInstIME<0b1110010, 0b001, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot1su", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT1US : RVInstIME<0b1110010, 0b010, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot1us", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +// Integer slide-2 instructions +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT2 : RVInstIME<0b1110011, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot2", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT2U : RVInstIME<0b1110011, 0b011, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot2u", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT2SU : RVInstIME<0b1110011, 0b001, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot2su", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT2US : RVInstIME<0b1110011, 0b010, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot2us", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +// Integer slide-3 instructions +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT3 : RVInstIME<0b1110100, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot3", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT3U : RVInstIME<0b1110100, 0b011, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot3u", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT3SU : RVInstIME<0b1110100, 0b001, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot3su", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VMADOT3US : RVInstIME<0b1110100, 0b010, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vmadot3us", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +// Floating-point slide instructions +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VFMADOT1 : RVInstIME<0b1110110, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vfmadot1", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VFMADOT2 : RVInstIME<0b1110111, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vfmadot2", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def IME_VFMADOT3 : RVInstIME<0b1111000, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2), + "vfmadot3", "$vd, $vs1, $vs2"> { + let Constraints = "$vd = $vd_in"; +} + +//===----------------------------------------------------------------------===// +// IME Extension Patterns +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot nxv8i32:$vd, nxv32i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT VRM4:$vd, VRM4:$vs1, VRM4:$vs2)>; +} + +// int16 vmadot patterns +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot nxv8i32:$vd, nxv16i16:$vs1, nxv16i16:$vs2)), + (IME_VMADOT VRM4:$vd, VRM4:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadotu nxv8i32:$vd, nxv32i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOTU VRM4:$vd, VRM4:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadotsu nxv8i32:$vd, nxv32i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOTSU VRM4:$vd, VRM4:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadotus nxv8i32:$vd, nxv32i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOTUS VRM4:$vd, VRM4:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv16f16 (int_riscv_ime_vfmadot nxv16f16:$vd, nxv16f16:$vs1, nxv16f16:$vs2)), + (IME_VFMADOT VRM4:$vd, VRM4:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot1 nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT1 VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot1u nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT1U VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot1su nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT1SU VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot1us nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT1US VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot2 nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT2 VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot2u nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT2U VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot2su nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT2SU VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot2us nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT2US VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot3 nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT3 VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot3u nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT3U VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot3su nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT3SU VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadot3us nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2)), + (IME_VMADOT3US VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv16f16 (int_riscv_ime_vfmadot1 nxv16f16:$vd, nxv32f16:$vs1, nxv16f16:$vs2)), + (IME_VFMADOT1 VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv16f16 (int_riscv_ime_vfmadot2 nxv16f16:$vd, nxv32f16:$vs1, nxv16f16:$vs2)), + (IME_VFMADOT2 VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv16f16 (int_riscv_ime_vfmadot3 nxv16f16:$vd, nxv32f16:$vs1, nxv16f16:$vs2)), + (IME_VFMADOT3 VRM4:$vd, VRM8:$vs1, VRM4:$vs2)>; +} + +class RVInstIMEN funct7, bits<3> funct3, dag outs, dag ins, + string opcodestr, string argstr> + : RVInst { + bits<5> vs2; + bits<5> vs1; + bits<5> rs1; // GPR for dynamic slide value + bits<5> vd; + + let Inst{31-25} = funct7; + let Inst{24-20} = vs2; + let Inst{19-15} = rs1; + let Inst{14-12} = funct3; + let Inst{11-7} = vd; + let Inst{6-0} = OPC_CUSTOM_1.Value; + + let Uses = [VTYPE, VL]; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in +def IME_VMADOTN : RVInstIMEN<0b1111001, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2, GPR:$rs1), + "vmadotn", "$vd, $vs1, $vs2, $rs1"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in +def IME_VMADOTNU : RVInstIMEN<0b1111001, 0b011, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2, GPR:$rs1), + "vmadotnu", "$vd, $vs1, $vs2, $rs1"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in +def IME_VMADOTNSU : RVInstIMEN<0b1111001, 0b001, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2, GPR:$rs1), + "vmadotnsu", "$vd, $vs1, $vs2, $rs1"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in +def IME_VMADOTNUS : RVInstIMEN<0b1111001, 0b010, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2, GPR:$rs1), + "vmadotnus", "$vd, $vs1, $vs2, $rs1"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME], hasSideEffects = 0, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in +def IME_VFMADOTN : RVInstIMEN<0b1111010, 0b000, + (outs VRM4:$vd), + (ins VRM4:$vd_in, VRM8:$vs1, VRM4:$vs2, GPR:$rs1), + "vfmadotn", "$vd, $vs1, $vs2, $rs1"> { + let Constraints = "$vd = $vd_in"; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadotn nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2, GPR:$rs1)), + (IME_VMADOTN VRM4:$vd, VRM8:$vs1, VRM4:$vs2, GPR:$rs1)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadotnu nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2, GPR:$rs1)), + (IME_VMADOTNU VRM4:$vd, VRM8:$vs1, VRM4:$vs2, GPR:$rs1)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadotnsu nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2, GPR:$rs1)), + (IME_VMADOTNSU VRM4:$vd, VRM8:$vs1, VRM4:$vs2, GPR:$rs1)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv8i32 (int_riscv_ime_vmadotnus nxv8i32:$vd, nxv64i8:$vs1, nxv32i8:$vs2, GPR:$rs1)), + (IME_VMADOTNUS VRM4:$vd, VRM8:$vs1, VRM4:$vs2, GPR:$rs1)>; +} + +let Predicates = [HasVendorXSMTIME] in { + def : Pat<(nxv16f16 (int_riscv_ime_vfmadotn nxv16f16:$vd, nxv32f16:$vs1, nxv16f16:$vs2, GPR:$rs1)), + (IME_VFMADOTN VRM4:$vd, VRM8:$vs1, VRM4:$vs2, GPR:$rs1)>; +} diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHeadAME.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHeadAME.td new file mode 100644 index 0000000000000..b09e2026205e1 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHeadAME.td @@ -0,0 +1,1234 @@ +//===- RISCVInstrInfoXTHeadAME.td --------------------------------------*- tablegen -*-===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// T-Head AME 32-bit matrix extension instructions. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// XTAME (RISC-V Matrix Extension) 32-bit Instructions +//===----------------------------------------------------------------------===// +// Reference: RISC-V Matrix Extension Specification + +def THeadAMEMatrixIndexAsmOperand : AsmOperandClass { + let Name = "THeadAMEMatrixIndex"; + let RenderMethod = "addImmOperands"; + let PredicateMethod = "isUImm3"; + let DiagnosticType = "InvalidTHeadAMEMatrixIndex"; +} + +// Operand type for THeadAMEMatrixReg index (0-7), printed with "m" prefix in AsmString +def THeadAMEMatrixIndex : RISCVOp { + let ParserMatchClass = THeadAMEMatrixIndexAsmOperand; + let DecoderMethod = "decodeUImmOperand<3>"; + let OperandType = "OPERAND_UIMM3"; +} + +def THeadAMEMatrixRegAsmOperand : AsmOperandClass { + let Name = "THeadAMEMatrixRegOp"; + let ParserMethod = "parseTHeadAMEMatrixReg"; + let RenderMethod = "addRegOperands"; + let PredicateMethod = "isTHeadAMEMatrixReg"; +} + +def THeadAMEMatrixRegOp : RegisterOperand { + let ParserMatchClass = THeadAMEMatrixRegAsmOperand; + let PrintMethod = "printTHeadAMEMatrixReg"; +} + +//===----------------------------------------------------------------------===// +// XTAME 32-bit Instruction Format Base Class +//===----------------------------------------------------------------------===// + +// Base class for XTAME 32-bit matrix multiplication instructions +// Uses THeadAMEMatrixReg for inputs (ms1, ms2) and THeadAMEMatrixReg for output (md) +class RVInstXTAME32 + : RVInst { + bits<4> func4; + bits<3> uop = 0b000; // usually 0 + bit size; + bits<3> ms2; + bits<3> ms1; + bits<3> md; + bits<3> func3 = 0b000; // usually 0 + bits<2> size_w; + bits<3> type; + bits<7> opcode = 0b0101011; + + let Inst{31-28} = func4; + let Inst{27-25} = uop; + let Inst{24} = size; + let Inst{23-21} = ms2; + let Inst{20-18} = ms1; + let Inst{17-15} = md; + let Inst{14-12} = func3; + let Inst{11-10} = size_w; + let Inst{9-7} = type; + let Inst{6-0} = opcode; +} + +//===----------------------------------------------------------------------===// +// AME Matrix Multiplication Instructions +//===----------------------------------------------------------------------===// +class TH_MMACC type_val, string opcodestr> + : RVInstXTAME32<(outs THeadAMEMatrixRegOp:$md), + (ins THeadAMEMatrixRegOp:$md_in, THeadAMEMatrixRegOp:$ms2, THeadAMEMatrixRegOp:$ms1), + opcodestr, "$md, $ms2, $ms1"> { + let Constraints = "$md = $md_in"; + let func4 = 0b0010; + let uop = 0b000; + let size = 0; + let func3 = 0b000; + let size_w = 0b00; + let type = type_val; + let opcode = 0b0101011; +} + +// th.mmacc.w.b - int8 × int8 → int32 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MMACC_W_B : TH_MMACC<0b000, "th.mmacc.w.b">; +// th.mmaccu.w.b - uint8 × uint8 → int32 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MMACCU_W_B : TH_MMACC<0b001, "th.mmaccu.w.b">; +// th.mmaccus.w.b - uint8 × int8 → int32 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MMACCUS_W_B : TH_MMACC<0b010, "th.mmaccus.w.b">; +// th.mmaccsu.w.b - int8 × uint8 → int32 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MMACCSU_W_B : TH_MMACC<0b011, "th.mmaccsu.w.b">; + +//===----------------------------------------------------------------------===// +// AME Matrix Multiplication Instructions (float) +//===----------------------------------------------------------------------===// +class TH_MFMACC size_w_val, bits<3> type_val, string opcodestr> + : RVInstXTAME32<(outs THeadAMEMatrixRegOp:$md), + (ins THeadAMEMatrixRegOp:$md_in, THeadAMEMatrixRegOp:$ms2, THeadAMEMatrixRegOp:$ms1), + opcodestr, "$md, $ms2, $ms1"> { + let Constraints = "$md = $md_in"; + let func4 = 0b0001; + let uop = 0b000; + let size = size_val; + let func3 = 0b000; + let size_w = size_w_val; + let type = type_val; + let opcode = 0b0101011; +} + +// th.mfmacc.h - float16 × float16 → float16 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_H : TH_MFMACC<0, 0b01, 0b000, "th.mfmacc.h">; +// th.mfmacc.bf16 - bfloat16 × bfloat16 → bfloat16 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_BF16 : TH_MFMACC<0, 0b01, 0b001, "th.mfmacc.bf16">; +// th.mfmacc.s - float32 × float32 → float32 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_S : TH_MFMACC<0b0, 0b10, 0b000, "th.mfmacc.s">; +// th.mfmacc.d - float64 × float64 → float64 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_D : TH_MFMACC<0b0, 0b11, 0b000, "th.mfmacc.d">; +// th.mfmacc.h.e4m3 - float8 x float8 → float16 accumulate with E4M3 format +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_H_E4M3 : TH_MFMACC<0b1, 0b00, 0b001, "th.mfmacc.h.e4m3">; +// th.mfmacc.h.e5m2 - float8 x float8 → float16 accumulate with E5M2 format +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_H_E5M2 : TH_MFMACC<0b1, 0b00, 0b000, "th.mfmacc.h.e5m2">; +// th.mfmacc.bf16.e4m3 - bfloat8 x bfloat8 → bfloat16 accumulate with E4M3 format +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_BF16_E4M3 : TH_MFMACC<0b1, 0b00, 0b011, "th.mfmacc.bf16.e4m3">; +// th.mfmacc.bf16.e5m2 - bfloat8 x bfloat8 → bfloat16 accumulate with E5M2 format +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_BF16_E5M2 : TH_MFMACC<0b1, 0b00, 0b010, "th.mfmacc.bf16.e5m2">; +// th.mfmacc.s.h - float16 × float16 → float32 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_S_H : TH_MFMACC<0b1, 0b01, 0b000, "th.mfmacc.s.h">; +// th.mfmacc.s.bf16 - bfloat16 × bfloat16 → float32 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_S_BF16 : TH_MFMACC<0b1, 0b01, 0b100, "th.mfmacc.s.bf16">; +// th.mfmacc.d.s - float32 × float32 → float64 accumulate +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_D_S : TH_MFMACC<0b1, 0b10, 0b000, "th.mfmacc.d.s">; +// th.mfmacc.s.e4m3 - float8 x float8 → float32 accumulate with E4M3 format +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_S_E4M3 : TH_MFMACC<0b1, 0b00, 0b101, "th.mfmacc.s.e4m3">; +// th.mfmacc.s.e5m2 - float8 x float8 → float32 accumulate with E5M2 format +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MFMACC_S_E5M2 : TH_MFMACC<0b1, 0b00, 0b100, "th.mfmacc.s.e5m2">; + +//===----------------------------------------------------------------------===// +// AME 32-bit Configuration Instructions (Register Type) +// Reference: RISC-V Matrix Specification v0.5.0, Section 4.1, Table 7 +// Format: +// | 31 | 30:28 | 27:25 | 24:20 | 19:15 | 14:12 | 11:9 | 8 | 7 | 6:0 | +// | 1 | func | 111 | 00000 | rs1 | 000 | 000 | 0 | 0 | 0101011 | +//===----------------------------------------------------------------------===// + +// Base class for 32-bit register-type configuration instructions +class RVInstXTAMEConfig32Reg func_val> + : RVInst { + bits<5> rs1; + + let Inst{31} = 1; // register type + let Inst{30-28}= func_val; + let Inst{27-25}= 0b111; // uop fixed to 111 + let Inst{24-20}= 0b00000; // reserved + let Inst{19-15}= rs1; + let Inst{14-12}= 0b000; // func3 fixed to 000 + let Inst{11-9} = 0b000; // nop field + let Inst{8} = 0; // mxb = 0 + let Inst{7} = 0; // mxa = 0 + let Inst{6-0} = 0b0101011; // custom-1 opcode +} + +//===----------------------------------------------------------------------===// +// AME 32-bit Configuration Instructions (Immediate Type) +// Reference: RISC-V Matrix Specification v0.5.0, Section 4.1, Table 7 +// Format: +// | 31 | 30:28 | 27:25 | 24:18 | 17:15 | 14:12 | 11:9 | 8 | 7 | 6:0 | +// | 0 | func | 111 | uimm7 | 000 | 000 | 000 | 0 | 0 | 0101011 | +//===----------------------------------------------------------------------===// + +// Base class for 32-bit immediate-type configuration instructions +class RVInstXTAMEConfig32Imm func_val> + : RVInst { + bits<7> imm; + + let Inst{31} = 0; // immediate type + let Inst{30-28}= func_val; + let Inst{27-25}= 0b111; // uop fixed to 111 + let Inst{24-18}= imm{6-0}; // imm[6:0] + let Inst{17-15}= 0b000; // reserved + let Inst{14-12}= 0b000; // func3 fixed to 000 + let Inst{11-9} = 0b000; // nop field + let Inst{8} = 0; // mxb = 0 + let Inst{7} = 0; // mxa = 0 + let Inst{6-0} = 0b0101011; // custom-1 opcode +} + +//===----------------------------------------------------------------------===// +// Concrete Instructions +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // th.mcfgk - configure tile K dimension (func=000) + def TH_MCFGK : RVInstXTAMEConfig32Reg<(outs), (ins GPR:$rs1), + "th.mcfgk", "$rs1", 0b000>; + + // th.mcfgm - configure tile M dimension (func=001) + def TH_MCFGM : RVInstXTAMEConfig32Reg<(outs), (ins GPR:$rs1), + "th.mcfgm", "$rs1", 0b001>; + + // th.mcfgn - configure tile N dimension (func=010) + def TH_MCFGN : RVInstXTAMEConfig32Reg<(outs), (ins GPR:$rs1), + "th.mcfgn", "$rs1", 0b010>; + + // th.mcfg - configure entire tile size (func=111) + def TH_MCFG : RVInstXTAMEConfig32Reg<(outs), (ins GPR:$rs1), + "th.mcfg", "$rs1", 0b111>; +} + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // th.mcfgki - configure tile K dimension from immediate (func=000) + def TH_MCFGKI : RVInstXTAMEConfig32Imm<(outs), (ins uimm32:$imm), + "th.mcfgki", "$imm", 0b000>; + + // th.mcfgmi - configure tile M dimension from immediate (func=001) + def TH_MCFGMI : RVInstXTAMEConfig32Imm<(outs), (ins uimm32:$imm), + "th.mcfgmi", "$imm", 0b001>; + + // th.mcfgni - configure tile N dimension from immediate (func=010) + def TH_MCFGNI : RVInstXTAMEConfig32Imm<(outs), (ins uimm32:$imm), + "th.mcfgni", "$imm", 0b010>; +} + +//===----------------------------------------------------------------------===// +// XTAME 32-bit Load/Store Instructions +// Reference: RISC-V Matrix Specification v0.5.0, Section 4.4, Table 10 +// Format: +// | 31:28 | 27:25 | 24:20 | 19:15 | 14:12 | 11:10 | 9:7 | 6:0 | +// | func4 | uop | rs2 | rs1 | func3 | size | md/ms3| opcode | +// func3 = 000, opcode = 0b0101011 (custom-1) +//===----------------------------------------------------------------------===// + +// Base class for all 32-bit load instructions (uop = 100) +class RVInstXTAMELoad32 func4_val, string opcodestr> + : RVInst<(outs THeadAMEMatrixRegOp:$md), (ins GPR:$rs2, GPR:$rs1), + opcodestr, "$md, $rs2, $rs1", [], InstFormatOther> { + bits<5> rs2; + bits<5> rs1; + bits<3> md; + bits<2> size; + + let Inst{31-28} = func4_val; + let Inst{27-25} = 0b100; // uop = 100 for load + let Inst{24-20} = rs2; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b000; // func3 = 000 + let Inst{11-10} = size; // size to be set in derived classes + let Inst{9-7} = md; + let Inst{6-0} = 0b0101011; // custom-1 opcode +} + +class RVInstXTAMEPrefetch32 func4_val, string opcodestr> + : RVInst<(outs), (ins GPR:$rs2, GPR:$rs1), + opcodestr, "$rs2, $rs1", [], InstFormatOther> { + bits<5> rs2; + bits<5> rs1; + bits<2> size; + + let Inst{31-28} = func4_val; + let Inst{27-25} = 0b100; // uop = 100 for prefetch + let Inst{24-20} = rs2; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b000; // func3 = 000 + let Inst{11-10} = size; // size to be set in derived classes + let Inst{9-7} = 0b000; + let Inst{6-0} = 0b0101011; // custom-1 opcode +} + +// Base class for all 32-bit store instructions (uop = 101) +class RVInstXTAMEStore32 func4_val, string opcodestr> + : RVInst<(outs), (ins THeadAMEMatrixRegOp:$ms3, GPR:$rs1, GPR:$rs2), + opcodestr, "$ms3, $rs2, $rs1", [], InstFormatOther> { + bits<5> rs2; + bits<5> rs1; + bits<3> ms3; + bits<2> size; + + let Inst{31-28} = func4_val; + let Inst{27-25} = 0b101; // uop = 101 for store + let Inst{24-20} = rs2; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b000; // func3 = 000 + let Inst{11-10} = size; // size to be set in derived classes + let Inst{9-7} = ms3; + let Inst{6-0} = 0b0101011; // custom-1 opcode +} + +//===----------------------------------------------------------------------===// +// Normal Load Instructions (func4 = 0000) +// th.mld.e8, th.mld.e16, th.mld.e32, th.mld.e64 +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXTHeadAME], mayLoad = 1, mayStore = 0, hasSideEffects = 0 in { + def TH_MLDE8 : RVInstXTAMELoad32<0b0000, "th.mlde8"> { let size = 0b00; } + def TH_MLDE16 : RVInstXTAMELoad32<0b0000, "th.mlde16"> { let size = 0b01; } + def TH_MLDE32 : RVInstXTAMELoad32<0b0000, "th.mlde32"> { let size = 0b10; } + def TH_MLDE64 : RVInstXTAMELoad32<0b0000, "th.mlde64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Normal Store Instructions (func4 = 0000) +// th.mst.e8, th.mst.e16, th.mst.e32, th.mst.e64 +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXTHeadAME], mayLoad = 0, mayStore = 1, hasSideEffects = 0 in { + def TH_MSTE8 : RVInstXTAMEStore32<0b0000, "th.mste8"> { let size = 0b00; } + def TH_MSTE16 : RVInstXTAMEStore32<0b0000, "th.mste16"> { let size = 0b01; } + def TH_MSTE32 : RVInstXTAMEStore32<0b0000, "th.mste32"> { let size = 0b10; } + def TH_MSTE64 : RVInstXTAMEStore32<0b0000, "th.mste64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Transposed Load Instructions (func4 = 0011) +// th.mldt.e8, th.mldt.e16, th.mldt.e32, th.mldt.e64 +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXTHeadAME], mayLoad = 1, mayStore = 0, hasSideEffects = 0 in { + def TH_MLDTE8 : RVInstXTAMELoad32<0b0011, "th.mldte8"> { let size = 0b00; } + def TH_MLDTE16 : RVInstXTAMELoad32<0b0011, "th.mldte16"> { let size = 0b01; } + def TH_MLDTE32 : RVInstXTAMELoad32<0b0011, "th.mldte32"> { let size = 0b10; } + def TH_MLDTE64 : RVInstXTAMELoad32<0b0011, "th.mldte64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Transposed Store Instructions (func4 = 0011) +// th.mstte8, th.mstte16, th.mstte32, th.mstte64 +//===----------------------------------------------------------------------===// +let Predicates = [HasVendorXTHeadAME], mayLoad = 0, mayStore = 1, hasSideEffects = 0 in { + def TH_MSTTE8 : RVInstXTAMEStore32<0b0011, "th.mstte8"> { let size = 0b00; } + def TH_MSTTE16 : RVInstXTAMEStore32<0b0011, "th.mstte16"> { let size = 0b01; } + def TH_MSTTE32 : RVInstXTAMEStore32<0b0011, "th.mstte32"> { let size = 0b10; } + def TH_MSTTE64 : RVInstXTAMEStore32<0b0011, "th.mstte64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Stream Load Instructions (func4 = 0001) +// th.mslde8, th.mslde16, th.mslde32, th.mslde64 +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME], mayLoad = 1, mayStore = 0, hasSideEffects = 0 in { + def TH_MSLDE8 : RVInstXTAMELoad32<0b0001, "th.mslde8"> { let size = 0b00; } + def TH_MSLDE16 : RVInstXTAMELoad32<0b0001, "th.mslde16"> { let size = 0b01; } + def TH_MSLDE32 : RVInstXTAMELoad32<0b0001, "th.mslde32"> { let size = 0b10; } + def TH_MSLDE64 : RVInstXTAMELoad32<0b0001, "th.mslde64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Stream Store Instructions (func4 = 0001) +// th.msste8, th.msste16, th.msste32, th.msste64 +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME], mayLoad = 0, mayStore = 1, hasSideEffects = 0 in { + def TH_MSSTE8 : RVInstXTAMEStore32<0b0001, "th.msste8"> { let size = 0b00; } + def TH_MSSTE16 : RVInstXTAMEStore32<0b0001, "th.msste16"> { let size = 0b01; } + def TH_MSSTE32 : RVInstXTAMEStore32<0b0001, "th.msste32"> { let size = 0b10; } + def TH_MSSTE64 : RVInstXTAMEStore32<0b0001, "th.msste64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Transposed Stream Load Instructions (func4 = 0100) +// th.msldte8, th.msldte16, th.msldte32, th.msldte64 +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME], mayLoad = 1, mayStore = 0, hasSideEffects = 0 in { + def TH_MSLDTE8 : RVInstXTAMELoad32<0b0100, "th.msldte8"> { let size = 0b00; } + def TH_MSLDTE16 : RVInstXTAMELoad32<0b0100, "th.msldte16"> { let size = 0b01; } + def TH_MSLDTE32 : RVInstXTAMELoad32<0b0100, "th.msldte32"> { let size = 0b10; } + def TH_MSLDTE64 : RVInstXTAMELoad32<0b0100, "th.msldte64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Transposed Stream Store Instructions (func4 = 0100) +// th.msstte8, th.msstte16, th.msstte32, th.msstte64 +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME], mayLoad = 0, mayStore = 1, hasSideEffects = 0 in { + def TH_MSSTTE8 : RVInstXTAMEStore32<0b0100, "th.msstte8"> { let size = 0b00; } + def TH_MSSTTE16 : RVInstXTAMEStore32<0b0100, "th.msstte16"> { let size = 0b01; } + def TH_MSSTTE32 : RVInstXTAMEStore32<0b0100, "th.msstte32"> { let size = 0b10; } + def TH_MSSTTE64 : RVInstXTAMEStore32<0b0100, "th.msstte64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Prefetch Load Instructions (func4 = 0101) +// th.mplde8, th.mplde16, th.mplde32, th.mplde64 +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME], mayLoad = 1, mayStore = 0, hasSideEffects = 1 in { + def TH_MPLDE8 : RVInstXTAMEPrefetch32<0b0101, "th.mplde8"> { let size = 0b00; } + def TH_MPLDE16 : RVInstXTAMEPrefetch32<0b0101, "th.mplde16"> { let size = 0b01; } + def TH_MPLDE32 : RVInstXTAMEPrefetch32<0b0101, "th.mplde32"> { let size = 0b10; } + def TH_MPLDE64 : RVInstXTAMEPrefetch32<0b0101, "th.mplde64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// Transposed Prefetch Load Instructions (func4 = 0110) +// th.mpldte8, th.mpldte16, th.mpldte32, th.mpldte64 +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME], mayLoad = 1, mayStore = 0, hasSideEffects = 1 in { + def TH_MPLDTE8 : RVInstXTAMEPrefetch32<0b0110, "th.mpldte8"> { let size = 0b00; } + def TH_MPLDTE16 : RVInstXTAMEPrefetch32<0b0110, "th.mpldte16"> { let size = 0b01; } + def TH_MPLDTE32 : RVInstXTAMEPrefetch32<0b0110, "th.mpldte32"> { let size = 0b10; } + def TH_MPLDTE64 : RVInstXTAMEPrefetch32<0b0110, "th.mpldte64"> { let size = 0b11; } +} + +//===----------------------------------------------------------------------===// +// MISC Instructions (format "others") - Base class +//===----------------------------------------------------------------------===// + +class RVInstXTAMEMISC func4_val, bits<3> uop_val> + : RVInst { + bits<4> func4 = func4_val; + bits<3> uop = uop_val; + bit ctrl = 0; // bit 24 must be 0 for mzero + bits<3> ms2 = 0b000; // fixed to 000 + bits<3> ms1 = 0b000; // fixed to 000 + bits<3> md; // destination base matrix register + bits<3> func3 = 0b000; // fixed to 000 + bits<2> size = 0b00; // size field (00 for mzero) + bits<3> uimm3; // immediate for register count encoding + bits<7> opcode = 0b0101011; // custom-1 + + let Inst{31-28} = func4; + let Inst{27-25} = uop; + let Inst{24} = ctrl; + let Inst{23-21} = ms2; + let Inst{20-18} = ms1; + let Inst{17-15} = md; + let Inst{14-12} = func3; + let Inst{11-10} = size; + let Inst{9-7} = uimm3; + let Inst{6-0} = opcode; +} + +//===----------------------------------------------------------------------===// +// th.mzero variants (different uimm3 encodings) +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, mayLoad = 0, mayStore = 0 in { + // Zero 1 matrix register (uimm3 = 000) + def TH_MZERO : RVInstXTAMEMISC<(outs), (ins THeadAMEMatrixRegOp:$md), + "th.mzero", "$md", + 0b1010, 0b000> { + let uimm3 = 0b000; + } + + // Zero 2 matrix registers (uimm3 = 001) + def TH_MZERO2R : RVInstXTAMEMISC<(outs), (ins THeadAMEMatrixRegOp:$md), + "th.mzero2r", "$md", + 0b1010, 0b000> { + let uimm3 = 0b001; + } + + // Zero 4 matrix registers (uimm3 = 011) + def TH_MZERO4R : RVInstXTAMEMISC<(outs), (ins THeadAMEMatrixRegOp:$md), + "th.mzero4r", "$md", + 0b1010, 0b000> { + let uimm3 = 0b011; + } + + // Zero 8 matrix registers (uimm3 = 111) + def TH_MZERO8R : RVInstXTAMEMISC<(outs), (ins THeadAMEMatrixRegOp:$md), + "th.mzero8r", "$md", + 0b1010, 0b000> { + let uimm3 = 0b111; + } +} + +//===----------------------------------------------------------------------===// +// MISC Data Move Instructions +//===----------------------------------------------------------------------===// + +// Data Move Instructions between Matrix Registers +class RVInstXTAMEMMOV + : RVInst { + bits<4> func4 = 0b0000; + bits<3> uop; + bit ctrl = 0b0; + bits<3> ms2 = 0b000; + bits<3> ms1; + bits<3> md; + bits<3> func3 = 0b000; + bits<2> size = 0b00; + bits<3> type; + bits<7> opcode = 0b0101011; + + let Inst{31-28} = func4; + let Inst{27-25} = uop; + let Inst{24} = ctrl; + let Inst{23-21} = ms2; + let Inst{20-18} = ms1; + let Inst{17-15} = md; + let Inst{14-12} = func3; + let Inst{11-10} = size; + let Inst{9-7} = type; + let Inst{6-0} = opcode; +} + +// th.mmov.mm - move between matrix registers (uop = 000) +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in +def TH_MMOV_MM : RVInstXTAMEMMOV<(outs THeadAMEMatrixReg:$md), (ins THeadAMEMatrixReg:$ms1), + "th.mmov.mm", "$md, $ms1"> { + let uop = 0b000; + let type = 0b001; +} + +// Data Move Instructions between Integer and Matrix +// matrix-scalar move instructions (uop = 110) +class RVInstXTAMESMMOVE func4_val, + dag outs, dag ins, string opcodestr, string argstr> + : RVInst { + bits<3> uop = 0b110; + bits<5> rs2; + bits<5> rs1; + bits<3> func3 = 0b000; + bits<2> size; + bits<3> md; + bits<7> opcode = 0b0101011; + + let Inst{31-28} = func4_val; + let Inst{27-25} = uop; + let Inst{24-20} = rs2; + let Inst{19-15} = rs1; + let Inst{14-12} = func3; + let Inst{11-10} = size; + let Inst{9-7} = md; + let Inst{6-0} = opcode; +} + +// matrix-scalar mov with duplicate +class TH_MDUP sz, string opcodestr> + : RVInstXTAMESMMOVE<0b0001, (outs THeadAMEMatrixReg:$md), (ins GPR:$rs2), + opcodestr, "$md, $rs2"> { + let rs1 = 0b00000; + let size = sz; +} + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // th.mdupb.m.x - duplicate byte from GPR to all elements in a matrix register + def TH_MDUPB_M_X : TH_MDUP<0b00, "th.mdupb.m.x">; + // th.mduph.m.x - duplicate halfword from GPR to all elements in a matrix register + def TH_MDUPH_M_X : TH_MDUP<0b01, "th.mduph.m.x">; + // th.mdupw.m.x - duplicate word from GPR to all elements in a matrix register + def TH_MDUPW_M_X : TH_MDUP<0b10, "th.mdupw.m.x">; + // th.mdupd.m.x - duplicate doubleword from GPR to all elements in a matrix register + def TH_MDUPD_M_X : TH_MDUP<0b11, "th.mdupd.m.x">; +} + +// matrix-scalar move +class TH_MMOV sz, string opcodestr> + : RVInstXTAMESMMOVE<0b0010, (outs THeadAMEMatrixReg:$md), (ins GPR:$rs2, GPR:$rs1), + opcodestr, "$md, $rs2, $rs1"> { + let size = sz; +} + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // th.mmovb.m.x + def TH_MMOVB_M_X : TH_MMOV<0b00, "th.mmovb.m.x">; + // th.mmovh.m.x + def TH_MMOVH_M_X : TH_MMOV<0b01, "th.mmovh.m.x">; + // th.mmovw.m.x + def TH_MMOVW_M_X : TH_MMOV<0b10, "th.mmovw.m.x">; + // th.mmovd.m.x + def TH_MMOVD_M_X : TH_MMOV<0b11, "th.mmovd.m.x">; +} + +// scalar-matrix move instructions (uop = 110) +class RVInstXTAMEMSMOVE sz, string opcodestr> + : RVInst<(outs GPR:$rd), (ins THeadAMEMatrixReg:$ms2, GPR:$rs1), + opcodestr, "$rd, $ms2, $rs1", [], InstFormatOther> { + bits<5> rd; + bits<3> ms2; + bits<5> rs1; + + let Inst{31-28} = 0b0000; + let Inst{27-25} = 0b110; + let Inst{24} = sz{1}; + let Inst{23-21} = ms2; + let Inst{20} = sz{0}; + let Inst{19-15} = rs1; + let Inst{14-12} = 0b000; + let Inst{11-7} = rd; + let Inst{6-0} = 0b0101011; +} + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // b (00): bit 24 = 0, bit 20 = 0 -> {0, rs1} + def TH_MMOVB_X_M : RVInstXTAMEMSMOVE<0b00, "th.mmovb.x.m">; + + // h (01): bit 24 = 0, bit 20 = 1 -> {1, rs1} + def TH_MMOVH_X_M : RVInstXTAMEMSMOVE<0b01, "th.mmovh.x.m">; + + // w (10): bit 24 = 1, bit 20 = 0 -> {0, rs1} + def TH_MMOVW_X_M : RVInstXTAMEMSMOVE<0b10, "th.mmovw.x.m">; + + // d (11): bit 24 = 1, bit 20 = 1 -> {1, rs1} + def TH_MMOVD_X_M : RVInstXTAMEMSMOVE<0b11, "th.mmovd.x.m">; +} + +// Data Broadcast Instructions +class RVInstXTAMEDB func4_val, bits<3> uop_val, bits<2> sz, string opcodestr> + : RVInstXTAME32<(outs THeadAMEMatrixReg:$md), (ins THeadAMEMatrixReg:$ms1, uimm3:$uimm3), + opcodestr, "$md, $ms1[$uimm3]"> { + bits<3> uimm3; + + let func4 = func4_val; + let uop = uop_val; + let size = 0b0; + let ms2 = 0b000; + let func3 = 0b000; + let size_w = sz; + let type = uimm3; + let opcode = 0b0101011; +} + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // 1. th.mmov.mv.i , size_w = 00 + // func4 = 0000, uop = 010 + def TH_MMOV_MV_I : RVInstXTAMEDB<0b0000, 0b010, 0b00, "th.mmov.mv.i">; +} + +class TH_MCMOV sz, string opcodestr> + : RVInstXTAMEDB<0b0101, 0b110, sz, opcodestr>; + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + def TH_MCMOVB_MV_I : TH_MCMOV<0b00, "th.mcmovb.mv.i">; + def TH_MCMOVH_MV_I : TH_MCMOV<0b01, "th.mcmovh.mv.i">; + def TH_MCMOVW_MV_I : TH_MCMOV<0b10, "th.mcmovw.mv.i">; + def TH_MCMOVD_MV_I : TH_MCMOV<0b11, "th.mcmovd.mv.i">; +} + +// Matrix Pack Instructions +class TH_MPACK type_val, string opcodestr> + : RVInstXTAME32<(outs THeadAMEMatrixReg:$md), (ins THeadAMEMatrixReg:$ms2, THeadAMEMatrixReg:$ms1), + opcodestr, "$md, $ms2, $ms1"> { + let func4 = 0b0011; + let uop = 0b110; + let size = 0b0; + let func3 = 0b000; + let size_w = 0b00; + let type = type_val; + let opcode = 0b0101011; +} + +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 0, mayLoad = 0, mayStore = 0 in { + // th.mpack.mm: pack the low part of ms2 and low part of ms1 into md + def TH_MPACK_MM : TH_MPACK<0b000, "th.mpack.mm">; + // th.mpackhl.mm: pack the high part of ms2 and low part of ms1 into md + def TH_MPACKHL_MM : TH_MPACK<0b010, "th.mpackhl.mm">; + // th.mpackhh.mm: pack the high part of ms2 and high part of ms1 into md + def TH_MPACKHH_MM : TH_MPACK<0b011, "th.mpackhh.mm">; +} + +//===----------------------------------------------------------------------===// +// Pseudo Instructions for 32-bit New Instructions +//===----------------------------------------------------------------------===// + +// Configuration immediate instructions (no matrix register) +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def TH_MCFGMI_PSEUDO : Pseudo<(outs), (ins i64imm:$imm), []> { + let AsmString = "th.mcfgmi\t$imm"; + } + def TH_MCFGNI_PSEUDO : Pseudo<(outs), (ins i64imm:$imm), []> { + let AsmString = "th.mcfgni\t$imm"; + } + def TH_MCFGKI_PSEUDO : Pseudo<(outs), (ins i64imm:$imm), []> { + let AsmString = "th.mcfgki\t$imm"; + } +} + +// Configured Register instructions (no matrix register) +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def TH_MCFGM_PSEUDO : Pseudo<(outs), (ins GPR:$rs1), []> { + let AsmString = "th.mcfgm\t$rs1"; + } + def TH_MCFGN_PSEUDO : Pseudo<(outs), (ins GPR:$rs1), []> { + let AsmString = "th.mcfgn\t$rs1"; + } + def TH_MCFGK_PSEUDO : Pseudo<(outs), (ins GPR:$rs1), []> { + let AsmString = "th.mcfgk\t$rs1"; + } + def TH_MCFG_PSEUDO : Pseudo<(outs), (ins GPR:$rs1), []> { + let AsmString = "th.mcfg\t$rs1"; + } +} + +// Zero instruction (single matrix register) +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + def TH_MZERO_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md), []> { + let AsmString = "th.mzero\tm$md"; + } + def TH_MZERO2R_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md), []> { + let AsmString = "th.mzero2r\tm$md"; + } + def TH_MZERO4R_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md), []> { + let AsmString = "th.mzero4r\tm$md"; + } + def TH_MZERO8R_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md), []> { + let AsmString = "th.mzero8r\tm$md"; + } +} + +// MISC Instructions +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, mayLoad = 0, mayStore = 0, + isCodeGenOnly = 1 in { + // 1. Data Move Instructions between Matrix Registers + def TH_MMOV_MM_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mmov.mm\tm$md, m$ms1"; + } + // 2. Data Move Instructions between Integer and Matrix (Duplicate) + def TH_MDUPB_M_X_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2), []> { + let AsmString = "th.mdupb.m.x\tm$md, $rs2"; + } + def TH_MDUPH_M_X_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2), []> { + let AsmString = "th.mduph.m.x\tm$md, $rs2"; + } + def TH_MDUPW_M_X_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2), []> { + let AsmString = "th.mdupw.m.x\tm$md, $rs2"; + } + def TH_MDUPD_M_X_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2), []> { + let AsmString = "th.mdupd.m.x\tm$md, $rs2"; + } + // 3. Data Move Instructions between Integer and Matrix (Scalar to Matrix) + def TH_MMOVB_M_X_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mmovb.m.x\tm$md, $rs2, $rs1"; + } + def TH_MMOVH_M_X_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mmovh.m.x\tm$md, $rs2, $rs1"; + } + def TH_MMOVW_M_X_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mmovw.m.x\tm$md, $rs2, $rs1"; + } + def TH_MMOVD_M_X_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mmovd.m.x\tm$md, $rs2, $rs1"; + } + // 4. Data Move Instructions between Integer and Matrix (Matrix to Scalar) + def TH_MMOVB_X_M_PSEUDO : Pseudo<(outs GPR:$rd), (ins THeadAMEMatrixIndex:$ms2, GPR:$rs1), []> { + let AsmString = "th.mmovb.x.m\t$rd, m$ms2, $rs1"; + } + def TH_MMOVH_X_M_PSEUDO : Pseudo<(outs GPR:$rd), (ins THeadAMEMatrixIndex:$ms2, GPR:$rs1), []> { + let AsmString = "th.mmovh.x.m\t$rd, m$ms2, $rs1"; + } + def TH_MMOVW_X_M_PSEUDO : Pseudo<(outs GPR:$rd), (ins THeadAMEMatrixIndex:$ms2, GPR:$rs1), []> { + let AsmString = "th.mmovw.x.m\t$rd, m$ms2, $rs1"; + } + def TH_MMOVD_X_M_PSEUDO : Pseudo<(outs GPR:$rd), (ins THeadAMEMatrixIndex:$ms2, GPR:$rs1), []> { + let AsmString = "th.mmovd.x.m\t$rd, m$ms2, $rs1"; + } + // 5. Data Broadcast Instructions + def TH_MMOV_MV_I_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms1, uimm3:$uimm3), []> { + let AsmString = "th.mmov.mv.i\tm$md, m$ms1[$uimm3]"; + } + def TH_MCMOVB_MV_I_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms1, uimm3:$uimm3), []> { + let AsmString = "th.mcmovb.mv.i\tm$md, m$ms1[$uimm3]"; + } + def TH_MCMOVH_MV_I_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms1, uimm3:$uimm3), []> { + let AsmString = "th.mcmovh.mv.i\tm$md, m$ms1[$uimm3]"; + } + def TH_MCMOVW_MV_I_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms1, uimm3:$uimm3), []> { + let AsmString = "th.mcmovw.mv.i\tm$md, m$ms1[$uimm3]"; + } + def TH_MCMOVD_MV_I_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms1, uimm3:$uimm3), []> { + let AsmString = "th.mcmovd.mv.i\tm$md, m$ms1[$uimm3]"; + } + // 6. Matrix Pack Instructions + def TH_MPACK_MM_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mpack.mm\tm$md, m$ms2, m$ms1"; + } + def TH_MPACKHL_MM_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mpackhl.mm\tm$md, m$ms2, m$ms1"; + } + def TH_MPACKHH_MM_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mpackhh.mm\tm$md, m$ms2, m$ms1"; + } +} + +// Load instructions (normal and transposed) +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, mayLoad = 1, mayStore = 0, + isCodeGenOnly = 1 in { + def TH_MLDE8_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mlde8\tm$md, $rs2, ($rs1)"; + } + def TH_MLDE16_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mlde16\tm$md, $rs2, ($rs1)"; + } + def TH_MLDE32_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mlde32\tm$md, $rs2, ($rs1)"; + } + def TH_MLDE64_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mlde64\tm$md, $rs2, ($rs1)"; + } + def TH_MLDTE8_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mldte8\tm$md, $rs2, ($rs1)"; + } + def TH_MLDTE16_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mldte16\tm$md, $rs2, ($rs1)"; + } + def TH_MLDTE32_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mldte32\tm$md, $rs2, ($rs1)"; + } + def TH_MLDTE64_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mldte64\tm$md, $rs2, ($rs1)"; + } + def TH_MSLDE8_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mslde8\tm$md, $rs2, ($rs1)"; + } + def TH_MSLDE16_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mslde16\tm$md, $rs2, ($rs1)"; + } + def TH_MSLDE32_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mslde32\tm$md, $rs2, ($rs1)"; + } + def TH_MSLDE64_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mslde64\tm$md, $rs2, ($rs1)"; + } + def TH_MSLDTE8_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msldte8\tm$md, $rs2, ($rs1)"; + } + def TH_MSLDTE16_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msldte16\tm$md, $rs2, ($rs1)"; + } + def TH_MSLDTE32_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msldte32\tm$md, $rs2, ($rs1)"; + } + def TH_MSLDTE64_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$md, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msldte64\tm$md, $rs2, ($rs1)"; + } +} + +// Prefetch instructions (normal and transposed) +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, mayLoad = 1, mayStore = 0, + isCodeGenOnly = 1 in { + def TH_MPLDE8_PSEUDO : Pseudo<(outs), (ins GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mplde8\t$rs2, ($rs1)"; + } + def TH_MPLDE16_PSEUDO : Pseudo<(outs), (ins GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mplde16\t$rs2, ($rs1)"; + } + def TH_MPLDE32_PSEUDO : Pseudo<(outs), (ins GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mplde32\t$rs2, ($rs1)"; + } + def TH_MPLDE64_PSEUDO : Pseudo<(outs), (ins GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mplde64\t$rs2, ($rs1)"; + } + def TH_MPLDTE8_PSEUDO : Pseudo<(outs), (ins GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mpldte8\t$rs2, ($rs1)"; + } + def TH_MPLDTE16_PSEUDO : Pseudo<(outs), (ins GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mpldte16\t$rs2, ($rs1)"; + } + def TH_MPLDTE32_PSEUDO : Pseudo<(outs), (ins GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mpldte32\t$rs2, ($rs1)"; + } + def TH_MPLDTE64_PSEUDO : Pseudo<(outs), (ins GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mpldte64\t$rs2, ($rs1)"; + } +} + +// Store instruction +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, mayLoad = 0, mayStore = 1, + isCodeGenOnly = 1 in { + def TH_MSTE8_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mste8\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSTE16_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mste16\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSTE32_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mste32\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSTE64_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mste64\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSTTE8_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mstte8\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSTTE16_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mstte16\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSTTE32_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mstte32\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSTTE64_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.mstte64\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSSTE8_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msste8\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSSTE16_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msste16\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSSTE32_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msste32\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSSTE64_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msste64\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSSTTE8_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msstte8\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSSTTE16_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msstte16\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSSTTE32_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msstte32\tm$ms3, $rs2, ($rs1)"; + } + def TH_MSSTTE64_PSEUDO : Pseudo<(outs), (ins THeadAMEMatrixIndex:$ms3, GPR:$rs2, GPR:$rs1), []> { + let AsmString = "th.msstte64\tm$ms3, $rs2, ($rs1)"; + } +} + +// Matrix multiply instruction +let Predicates = [HasVendorXTHeadAME], hasSideEffects = 1, + isCodeGenOnly = 1 in { + def TH_MMACC_W_B_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mmacc.w.b\tm$md, m$ms2, m$ms1"; + } + def TH_MMACCU_W_B_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mmaccu.w.b\tm$md, m$ms2, m$ms1"; + } + def TH_MMACCUS_W_B_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mmaccus.w.b\tm$md, m$ms2, m$ms1"; + } + def TH_MMACCSU_W_B_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mmaccsu.w.b\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_H_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.h\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_BF16_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.bf16\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_S_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.s\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_D_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.d\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_H_E4M3_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.h.e4m3\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_H_E5M2_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.h.e5m2\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_BF16_E4M3_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.bf16.e4m3\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_BF16_E5M2_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.bf16.e5m2\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_S_H_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.s.h\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_S_BF16_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.s.bf16\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_D_S_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.d.s\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_S_E4M3_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.s.e4m3\tm$md, m$ms2, m$ms1"; + } + def TH_MFMACC_S_E5M2_PSEUDO : Pseudo<(outs), + (ins THeadAMEMatrixIndex:$md, THeadAMEMatrixIndex:$ms2, THeadAMEMatrixIndex:$ms1), []> { + let AsmString = "th.mfmacc.s.e5m2\tm$md, m$ms2, m$ms1"; + } +} + +//===----------------------------------------------------------------------===// +// Pattern Matching for 32-bit New Instructions +//===----------------------------------------------------------------------===// + +let Predicates = [HasVendorXTHeadAME] in { + + // Configuration immediate instructions + def : Pat<(int_riscv_th_mcfgmi timm:$imm), + (TH_MCFGMI_PSEUDO timm:$imm)>; + def : Pat<(int_riscv_th_mcfgni timm:$imm), + (TH_MCFGNI_PSEUDO timm:$imm)>; + def : Pat<(int_riscv_th_mcfgki timm:$imm), + (TH_MCFGKI_PSEUDO timm:$imm)>; + + // Configured Register instructions + def : Pat<(int_riscv_th_mcfgm i64:$rs1), + (TH_MCFGM_PSEUDO GPR:$rs1)>; + + def : Pat<(int_riscv_th_mcfgn i64:$rs1), + (TH_MCFGN_PSEUDO GPR:$rs1)>; + + def : Pat<(int_riscv_th_mcfgk i64:$rs1), + (TH_MCFGK_PSEUDO GPR:$rs1)>; + + def : Pat<(int_riscv_th_mcfg i64:$rs1), + (TH_MCFG_PSEUDO GPR:$rs1)>; + + // Zero instruction + def : Pat<(int_riscv_th_mzero timm:$md), + (TH_MZERO_PSEUDO timm:$md)>; + def : Pat<(int_riscv_th_mzero2r timm:$md), + (TH_MZERO2R_PSEUDO timm:$md)>; + def : Pat<(int_riscv_th_mzero4r timm:$md), + (TH_MZERO4R_PSEUDO timm:$md)>; + def : Pat<(int_riscv_th_mzero8r timm:$md), + (TH_MZERO8R_PSEUDO timm:$md)>; + + // MISC instructions + // 1. Data Move Instructions between Matrix Registers + def : Pat<(int_riscv_th_mmov_mm timm:$md, timm:$ms1), + (TH_MMOV_MM_PSEUDO timm:$md, timm:$ms1)>; + + // 2. Data Move Instructions between Integer and Matrix (Duplicate) + def : Pat<(int_riscv_th_mdupb_m_x timm:$md, i64:$rs2), + (TH_MDUPB_M_X_PSEUDO timm:$md, GPR:$rs2)>; + def : Pat<(int_riscv_th_mduph_m_x timm:$md, i64:$rs2), + (TH_MDUPH_M_X_PSEUDO timm:$md, GPR:$rs2)>; + def : Pat<(int_riscv_th_mdupw_m_x timm:$md, i64:$rs2), + (TH_MDUPW_M_X_PSEUDO timm:$md, GPR:$rs2)>; + def : Pat<(int_riscv_th_mdupd_m_x timm:$md, i64:$rs2), + (TH_MDUPD_M_X_PSEUDO timm:$md, GPR:$rs2)>; + + // 3. Data Move Instructions between Integer and Matrix (Scalar to Matrix) + def : Pat<(int_riscv_th_mmovb_m_x timm:$md, i64:$rs2, i64:$rs1), + (TH_MMOVB_M_X_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mmovh_m_x timm:$md, i64:$rs2, i64:$rs1), + (TH_MMOVH_M_X_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mmovw_m_x timm:$md, i64:$rs2, i64:$rs1), + (TH_MMOVW_M_X_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mmovd_m_x timm:$md, i64:$rs2, i64:$rs1), + (TH_MMOVD_M_X_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + + // 4. Data Move Instructions between Integer and Matrix (Matrix to Scalar) + def : Pat<(int_riscv_th_mmovb_x_m timm:$ms2, i64:$rs1), + (TH_MMOVB_X_M_PSEUDO timm:$ms2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mmovh_x_m timm:$ms2, i64:$rs1), + (TH_MMOVH_X_M_PSEUDO timm:$ms2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mmovw_x_m timm:$ms2, i64:$rs1), + (TH_MMOVW_X_M_PSEUDO timm:$ms2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mmovd_x_m timm:$ms2, i64:$rs1), + (TH_MMOVD_X_M_PSEUDO timm:$ms2, GPR:$rs1)>; + + // 5. Data Broadcast Instructions + def : Pat<(int_riscv_th_mmov_mv_i timm:$md, timm:$ms1, imm:$uimm3), + (TH_MMOV_MV_I_PSEUDO timm:$md, timm:$ms1, imm:$uimm3)>; + def : Pat<(int_riscv_th_mcmovb_mv_i timm:$md, timm:$ms1, imm:$uimm3), + (TH_MCMOVB_MV_I_PSEUDO timm:$md, timm:$ms1, imm:$uimm3)>; + def : Pat<(int_riscv_th_mcmovh_mv_i timm:$md, timm:$ms1, imm:$uimm3), + (TH_MCMOVH_MV_I_PSEUDO timm:$md, timm:$ms1, imm:$uimm3)>; + def : Pat<(int_riscv_th_mcmovw_mv_i timm:$md, timm:$ms1, imm:$uimm3), + (TH_MCMOVW_MV_I_PSEUDO timm:$md, timm:$ms1, imm:$uimm3)>; + def : Pat<(int_riscv_th_mcmovd_mv_i timm:$md, timm:$ms1, imm:$uimm3), + (TH_MCMOVD_MV_I_PSEUDO timm:$md, timm:$ms1, imm:$uimm3)>; + + // 6. Matrix Pack Instructions + def : Pat<(int_riscv_th_mpack_mm timm:$md, timm:$ms2, timm:$ms1), + (TH_MPACK_MM_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mpackhl_mm timm:$md, timm:$ms2, timm:$ms1), + (TH_MPACKHL_MM_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mpackhh_mm timm:$md, timm:$ms2, timm:$ms1), + (TH_MPACKHH_MM_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + + // Load instructions + def : Pat<(int_riscv_th_mlde8 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MLDE8_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mlde16 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MLDE16_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mlde32 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MLDE32_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mlde64 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MLDE64_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mldte8 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MLDTE8_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mldte16 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MLDTE16_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mldte32 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MLDTE32_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mldte64 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MLDTE64_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mslde8 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MSLDE8_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mslde16 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MSLDE16_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mslde32 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MSLDE32_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mslde64 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MSLDE64_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msldte8 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MSLDTE8_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msldte16 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MSLDTE16_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msldte32 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MSLDTE32_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msldte64 timm:$md, i64:$rs2, iPTR:$rs1), + (TH_MSLDTE64_PSEUDO timm:$md, GPR:$rs2, GPR:$rs1)>; + + // Prefetch instructions + def : Pat<(int_riscv_th_mplde8 i64:$rs2, iPTR:$rs1), + (TH_MPLDE8_PSEUDO GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mplde16 i64:$rs2, iPTR:$rs1), + (TH_MPLDE16_PSEUDO GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mplde32 i64:$rs2, iPTR:$rs1), + (TH_MPLDE32_PSEUDO GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mplde64 i64:$rs2, iPTR:$rs1), + (TH_MPLDE64_PSEUDO GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mpldte8 i64:$rs2, iPTR:$rs1), + (TH_MPLDTE8_PSEUDO GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mpldte16 i64:$rs2, iPTR:$rs1), + (TH_MPLDTE16_PSEUDO GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mpldte32 i64:$rs2, iPTR:$rs1), + (TH_MPLDTE32_PSEUDO GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mpldte64 i64:$rs2, iPTR:$rs1), + (TH_MPLDTE64_PSEUDO GPR:$rs2, GPR:$rs1)>; + + // Store instruction + def : Pat<(int_riscv_th_mste8 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSTE8_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mste16 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSTE16_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mste32 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSTE32_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mste64 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSTE64_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mstte8 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSTTE8_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mstte16 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSTTE16_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mstte32 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSTTE32_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_mstte64 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSTTE64_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msste8 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSSTE8_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msste16 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSSTE16_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msste32 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSSTE32_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msste64 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSSTE64_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msstte8 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSSTTE8_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msstte16 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSSTTE16_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msstte32 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSSTTE32_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + def : Pat<(int_riscv_th_msstte64 timm:$ms3, i64:$rs2, iPTR:$rs1), + (TH_MSSTTE64_PSEUDO timm:$ms3, GPR:$rs2, GPR:$rs1)>; + + // Matrix multiply instruction + def : Pat<(int_riscv_th_mmacc_w_b timm:$md, timm:$ms2, timm:$ms1), + (TH_MMACC_W_B_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mmaccu_w_b timm:$md, timm:$ms2, timm:$ms1), + (TH_MMACCU_W_B_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mmaccus_w_b timm:$md, timm:$ms2, timm:$ms1), + (TH_MMACCUS_W_B_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mmaccsu_w_b timm:$md, timm:$ms2, timm:$ms1), + (TH_MMACCSU_W_B_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_h timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_H_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_bf16 timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_BF16_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_s timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_S_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_d timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_D_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_h_e4m3 timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_H_E4M3_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_h_e5m2 timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_H_E5M2_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_bf16_e4m3 timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_BF16_E4M3_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_bf16_e5m2 timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_BF16_E5M2_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_s_h timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_S_H_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_s_bf16 timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_S_BF16_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_d_s timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_D_S_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_s_e4m3 timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_S_E4M3_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; + def : Pat<(int_riscv_th_mfmacc_s_e5m2 timm:$md, timm:$ms2, timm:$ms1), + (TH_MFMACC_S_E5M2_PSEUDO timm:$md, timm:$ms2, timm:$ms1)>; +} diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZtt.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZtt.td new file mode 100644 index 0000000000000..925122ef73669 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZtt.td @@ -0,0 +1,585 @@ +//===- RISCVInstrInfoZtt.td - 'Ztt' ----------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file describes the RISC-V instructions from the standard 'Ztt' +// extension, providing Attached Matrix Extension instructions. +// This version is still experimental as the 'Ztt' extension hasn't been +// ratified yet. +// +//===----------------------------------------------------------------------===// + +// These operand types allow intrinsics to pass immediate indices +// for matrix (0-16/0-32) and accumulator (0/0-1/0-3) registers. +// hardcodes "acc" and "m" prefixes so that indices 0-7 are printed as +// acc0-acc7 and tr0-tr7 respectively, without modifying LLVM submodule. + +// AsmOperandClass for matrix register index (m0-m16 or m0-m32 depending on implementation) +def ZttMatrixRegIndexAsmOperand : AsmOperandClass { + let Name = "ZttMatrixRegIndex"; + let ParserMethod = "parseZttMatrixRegIndex"; + let RenderMethod = "addImmOperands"; + let PredicateMethod = "isUImm5"; + let DiagnosticType = "InvalidZttMatrixRegIndex"; +} + +// AsmOperandClass for matrix register index (acc0, acc0-acc1, acc0-acc3 depending on implementation) +def ZttAccRegIndexAsmOperand : AsmOperandClass { + let Name = "ZttAccRegIndex"; + let ParserMethod = "parseZttAccRegIndex"; + let RenderMethod = "addImmOperands"; + let PredicateMethod = "isUImm2"; + let DiagnosticType = "InvalidZttAccRegIndex"; +} + +// Operand type for ZttMatrixRegIndex (0-16 or 0-32) +def ZttMatrixRegIndex : RISCVOp { + let ParserMatchClass = ZttMatrixRegIndexAsmOperand; + // TODO: implement register index numeric range checks + // against enabled mregs/accregs subfeature in decoder method. + let DecoderMethod = "decodeUImmOperand<5>"; + let OperandType = "OPERAND_ZTT_MATRIX_REG"; + let PrintMethod = "printZttMatrixRegIndex"; +} + +// Operand type for ZttAccRegIndex (0, 0-1 or 0-3) +def ZttAccRegIndex : RISCVOp { + let ParserMatchClass = ZttAccRegIndexAsmOperand; + // TODO: implement register index numeric range checks + // against enabled mregs/accregs subfeature in decoder method. + let DecoderMethod = "decodeUImmOperand<2>"; + let OperandType = "OPERAND_ZTT_ACC_REG"; + let PrintMethod = "printZttAccRegIndex"; +} + +//===----------------------------------------------------------------------===// +// AME (Ztt) 32-bit real/concrete instructions +//===----------------------------------------------------------------------===// + +// AME (Ztt) 32-bit instruction template +class RVInstAME32 funct7, dag outs, dag ins, string opcodestr, + string argstr> + : RVInst { + let Inst{31 - 25} = funct7; + let Inst{6 - 0} = 0b0101011; // RISC-V custom-1 opcode space +} + +// Base class for single matrix input, matrix output instructions (op md, ms1). +// Input and output are matrix registers +class ZTT_UNARYELEMENTWISE funct7, bits<3> funct3, string opcodestr> + : RVInstAME32 { + bits<5> ms1; + bits<5> md; + + // unused here; other instructions use this slot to encode ms2/xs1 + let Inst{24 - 20} = 0b00000; + let Inst{19 - 15} = ms1; + let Inst{14 - 12} = funct3; + let Inst{11 - 7} = md; +} + +let Predicates = [HasStdExtZtt], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def ZTT_MABS_EW : ZTT_UNARYELEMENTWISE<0b0000000, 0b001, "mabs.ew">; + def ZTT_MCOLUNZIP_EW : ZTT_UNARYELEMENTWISE<0b0000100, 0b001, "mcolunzip.ew">; + def ZTT_MCOLZIP_EW : ZTT_UNARYELEMENTWISE<0b0000011, 0b001, "mcolzip.ew">; + def ZTT_MCONV_EW : ZTT_UNARYELEMENTWISE<0b0000000, 0b101, "mconv.ew">; + def ZTT_MEXP2_EW : ZTT_UNARYELEMENTWISE<0b0000001, 0b001, "mexp2.ew">; + def ZTT_MLOG2_EW : ZTT_UNARYELEMENTWISE<0b0000010, 0b001, "mlog2.ew">; + def ZTT_MPREFIXADD_COL : ZTT_UNARYELEMENTWISE<0b0001100, 0b001, "mprefixadd.col">; + def ZTT_MPREFIXADD_ROW : ZTT_UNARYELEMENTWISE<0b0001000, 0b001, "mprefixadd.row">; + def ZTT_MPREFIXMAX_COL : ZTT_UNARYELEMENTWISE<0b0001101, 0b001, "mprefixmax.col">; + def ZTT_MPREFIXMAX_ROW : ZTT_UNARYELEMENTWISE<0b0001001, 0b001, "mprefixmax.row">; + def ZTT_MREDUCEADD_COL : ZTT_UNARYELEMENTWISE<0b0001010, 0b001, "mreduceadd.col">; + def ZTT_MREDUCEADD_ROW : ZTT_UNARYELEMENTWISE<0b0000110, 0b001, "mreduceadd.row">; + def ZTT_MREDUCEMAX_COL : ZTT_UNARYELEMENTWISE<0b0001011, 0b001, "mreducemax.col">; + def ZTT_MREDUCEMAX_ROW : ZTT_UNARYELEMENTWISE<0b0000111, 0b001, "mreducemax.row">; + def ZTT_MROWUNZIP_EW : ZTT_UNARYELEMENTWISE<0b0000101, 0b001, "mrowunzip.ew">; +} + +// Base class for binary matrix input, matrix output instructions +// (op md, ms1, ms2). +class ZTT_BINARYELEMENTWISE funct7, bits<3> funct3, string opcodestr> + : RVInstAME32 { + bits<5> ms1; + bits<5> ms2; + bits<5> md; + + let Inst{24 - 20} = ms2; + let Inst{19 - 15} = ms1; + let Inst{14 - 12} = funct3; + let Inst{11 - 7} = md; +} + +let Predicates = [HasStdExtZtt], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def ZTT_MABSDIFF_EW : ZTT_BINARYELEMENTWISE<0b0000100, 0b000, "mabsdiff.ew">; + def ZTT_MADD_EW : ZTT_BINARYELEMENTWISE<0b0000000, 0b000, "madd.ew">; + def ZTT_MAND_EW : ZTT_BINARYELEMENTWISE<0b0000111, 0b000, "mand.ew">; + def ZTT_MANDNOT_EW : ZTT_BINARYELEMENTWISE<0b0001000, 0b000, "mandnot.ew">; + def ZTT_MCMOVGE_EW : ZTT_BINARYELEMENTWISE<0b0010000, 0b000, "mcmovge.ew">; + def ZTT_MCMOVLT_EW : ZTT_BINARYELEMENTWISE<0b0010001, 0b000, "mcmovlt.ew">; + def ZTT_MCMPGE_EW : ZTT_BINARYELEMENTWISE<0b0001100, 0b000, "mcmpge.ew">; + def ZTT_MCMPLT_EW : ZTT_BINARYELEMENTWISE<0b0001101, 0b000, "mcmplt.ew">; + def ZTT_MGATHER_EW : ZTT_BINARYELEMENTWISE<0b0011000, 0b000, "mgather.ew">; + def ZTT_MHDIFF_EW : ZTT_BINARYELEMENTWISE<0b0000011, 0b000, "mhdiff.ew">; + def ZTT_MLDEXP_EW : ZTT_BINARYELEMENTWISE<0b0010010, 0b000, "mldexp.ew">; + def ZTT_MLDEXPACC_EW : ZTT_BINARYELEMENTWISE<0b0010011, 0b000, "mldexpacc.ew">; + def ZTT_MLOG2SUB_EW : ZTT_BINARYELEMENTWISE<0b0010110, 0b000, "mlog2sub.ew">; + def ZTT_MMAX_EW : ZTT_BINARYELEMENTWISE<0b0000101, 0b000, "mmax.ew">; + def ZTT_MMEAN_EW : ZTT_BINARYELEMENTWISE<0b0000010, 0b000, "mmean.ew">; + def ZTT_MMIN_EW : ZTT_BINARYELEMENTWISE<0b0000110, 0b000, "mmin.ew">; + def ZTT_MMUL_EW : ZTT_BINARYELEMENTWISE<0b0011100, 0b000, "mmul.ew">; + def ZTT_MMULACC_EW : ZTT_BINARYELEMENTWISE<0b0011101, 0b000, "mmulacc.ew">; + def ZTT_MMULACCNEG_EW : ZTT_BINARYELEMENTWISE<0b0100000, 0b000, "mmulaccneg.ew">; + def ZTT_MMULADD_EW : ZTT_BINARYELEMENTWISE<0b0011110, 0b000, "mmuladd.ew">; + def ZTT_MMULNEG_EW : ZTT_BINARYELEMENTWISE<0b0100001, 0b000, "mmulneg.ew">; + def ZTT_MMULSUB_EW : ZTT_BINARYELEMENTWISE<0b0011111, 0b000, "mmulsub.ew">; + def ZTT_MOR_EW : ZTT_BINARYELEMENTWISE<0b0001001, 0b000, "mor.ew">; + def ZTT_MORNOT_EW : ZTT_BINARYELEMENTWISE<0b0001010, 0b000, "mornot.ew">; + def ZTT_MRDEXP_EW : ZTT_BINARYELEMENTWISE<0b0010100, 0b000, "mrdexp.ew">; + def ZTT_MRDEXPACC_EW : ZTT_BINARYELEMENTWISE<0b0010101, 0b000, "mrdexpacc.ew">; + def ZTT_MROWZIP_EW : ZTT_BINARYELEMENTWISE<0b0011011, 0b000, "mrowzip.ew">; + def ZTT_MSCATADD_COL : ZTT_BINARYELEMENTWISE<0b0100010, 0b000, "mscatadd.col">; + def ZTT_MSCATADD_ROW : ZTT_BINARYELEMENTWISE<0b0011001, 0b000, "mscatadd.row">; + def ZTT_MSCATMAX_COL : ZTT_BINARYELEMENTWISE<0b0100011, 0b000, "mscatmax.col">; + def ZTT_MSCATMAX_ROW : ZTT_BINARYELEMENTWISE<0b0011010, 0b000, "mscatmax.row">; + def ZTT_MSELGE_EW : ZTT_BINARYELEMENTWISE<0b0001110, 0b000, "mselge.ew">; + def ZTT_MSELLT_EW : ZTT_BINARYELEMENTWISE<0b0001111, 0b000, "msellt.ew">; + def ZTT_MSUB_EW : ZTT_BINARYELEMENTWISE<0b0000001, 0b000, "msub.ew">; + def ZTT_MSUBLOG2_EW : ZTT_BINARYELEMENTWISE<0b0010111, 0b000, "msublog2.ew">; + def ZTT_MXOR_EW : ZTT_BINARYELEMENTWISE<0b0001011, 0b000, "mxor.ew">; +} + +// Base class for binary matrix input, matrix output instructions where the +// second source is a scalar GPR instead of a matrix register +// (op md, xs1, ms2). Scalar type is inferred from Md[ms2]. +class ZTT_BINARYELEMENTWISE_X funct7, bits<3> funct3, string opcodestr> + : RVInstAME32 { + bits<5> xs1; + bits<5> ms2; + bits<5> md; + + let Inst{24 - 20} = ms2; + let Inst{19 - 15} = xs1; + let Inst{14 - 12} = funct3; + let Inst{11 - 7} = md; +} + +let Predicates = [HasStdExtZtt], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def ZTT_MABSDIFF_EW_X : ZTT_BINARYELEMENTWISE_X<0b0000101, 0b010, "mabsdiff.ew.x">; + def ZTT_MADD_EW_X : ZTT_BINARYELEMENTWISE_X<0b0000000, 0b010, "madd.ew.x">; + def ZTT_MAND_EW_X : ZTT_BINARYELEMENTWISE_X<0b0000111, 0b010, "mand.ew.x">; + def ZTT_MANDNOT_EW_X : ZTT_BINARYELEMENTWISE_X<0b0001000, 0b010, "mandnot.ew.x">; + def ZTT_MCMPGE_EW_X : ZTT_BINARYELEMENTWISE_X<0b0001100, 0b010, "mcmpge.ew.x">; + def ZTT_MCMPLT_EW_X : ZTT_BINARYELEMENTWISE_X<0b0001101, 0b010, "mcmplt.ew.x">; + def ZTT_MHDIFF_EW_X : ZTT_BINARYELEMENTWISE_X<0b0000110, 0b010, "mhdiff.ew.x">; + def ZTT_MLDEXP_EW_X : ZTT_BINARYELEMENTWISE_X<0b0010000, 0b010, "mldexp.ew.x">; + def ZTT_MLDEXPACC_EW_X : ZTT_BINARYELEMENTWISE_X<0b0010001, 0b010, "mldexpacc.ew.x">; + def ZTT_MLOG2SUB_EW_X : ZTT_BINARYELEMENTWISE_X<0b0001110, 0b010, "mlog2sub.ew.x">; + def ZTT_MMAX_EW_X : ZTT_BINARYELEMENTWISE_X<0b0000011, 0b010, "mmax.ew.x">; + def ZTT_MMEAN_EW_X : ZTT_BINARYELEMENTWISE_X<0b0000100, 0b010, "mmean.ew.x">; + def ZTT_MMIN_EW_X : ZTT_BINARYELEMENTWISE_X<0b0000010, 0b010, "mmin.ew.x">; + def ZTT_MMUL_EW_X : ZTT_BINARYELEMENTWISE_X<0b0010010, 0b010, "mmul.ew.x">; + def ZTT_MMULACC_EW_X : ZTT_BINARYELEMENTWISE_X<0b0010011, 0b010, "mmulacc.ew.x">; + def ZTT_MMULACCNEG_EW_X : ZTT_BINARYELEMENTWISE_X<0b0010110, 0b010, "mmulaccneg.ew.x">; + def ZTT_MMULADD_EW_X : ZTT_BINARYELEMENTWISE_X<0b0010100, 0b010, "mmuladd.ew.x">; + def ZTT_MMULNEG_EW_X : ZTT_BINARYELEMENTWISE_X<0b0010111, 0b010, "mmulneg.ew.x">; + def ZTT_MMULSUB_EW_X : ZTT_BINARYELEMENTWISE_X<0b0010101, 0b010, "mmulsub.ew.x">; + def ZTT_MOR_EW_X : ZTT_BINARYELEMENTWISE_X<0b0001001, 0b010, "mor.ew.x">; + def ZTT_MORNOT_EW_X : ZTT_BINARYELEMENTWISE_X<0b0001010, 0b010, "mornot.ew.x">; + def ZTT_MSUB_EW_X : ZTT_BINARYELEMENTWISE_X<0b0000001, 0b010, "msub.ew.x">; + def ZTT_MSUBLOG2_EW_X : ZTT_BINARYELEMENTWISE_X<0b0001111, 0b010, "msublog2.ew.x">; + def ZTT_MXOR_EW_X : ZTT_BINARYELEMENTWISE_X<0b0001011, 0b010, "mxor.ew.x">; +} + +// Base class for 2D matrix multiply instructions (op acc, ms1, ms2). +// acc is only 2 bits, freeing bits 14-10. +class ZTT_MATMUL2D funct7, string opcodestr> + : RVInstAME32 { + bits<5> ms1; + bits<5> ms2; + bits<2> acc; + + let Inst{24 - 20} = ms2; + let Inst{19 - 15} = ms1; + let Inst{14 - 10} = 0b01000; + let Inst{9 - 8} = acc; + let Inst{7} = 0b0; +} + +let Predicates = [HasStdExtZtt], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + def ZTT_MMUL_2D : ZTT_MATMUL2D<0b0000000, "mmul.2d">; + def ZTT_MMULACC_2D : ZTT_MATMUL2D<0b0000001, "mmulacc.2d">; + def ZTT_MMULACCNEG_2D : ZTT_MATMUL2D<0b0000011, "mmulaccneg.2d">; + def ZTT_MMULAT_2D : ZTT_MATMUL2D<0b0000100, "mmulat.2d">; + def ZTT_MMULATACC_2D : ZTT_MATMUL2D<0b0000101, "mmulatacc.2d">; + def ZTT_MMULBT_2D : ZTT_MATMUL2D<0b0000110, "mmulbt.2d">; + def ZTT_MMULBTACC_2D : ZTT_MATMUL2D<0b0000111, "mmulbtacc.2d">; + def ZTT_MMULNEG_2D : ZTT_MATMUL2D<0b0000010, "mmulneg.2d">; +} + +// Base class for the interoperability load instructions (mls, mls.cm, +// mls.rm). +class ZTT_LOAD funct7, string opcodestr> + : RVInstAME32 { + bits<5> xs1; + bits<5> md; + + let Inst{24 - 20} = 0b00000; + let Inst{19 - 15} = xs1; + let Inst{14 - 12} = 0b011; + let Inst{11 - 7} = md; +} + +let Predicates = [HasStdExtZtt], hasSideEffects = 1, mayLoad = 1, + mayStore = 0 in { + def ZTT_MLS : ZTT_LOAD<0b0000010, "mls">; + def ZTT_MLS_CM : ZTT_LOAD<0b0000001, "mls.cm">; + def ZTT_MLS_RM : ZTT_LOAD<0b0000000, "mls.rm">; +} + +// Base class for the interoperability store instructions (mss, mss.cm, +// mss.rm). +class ZTT_STORE funct7, string opcodestr> + : RVInstAME32 { + bits<5> xs1; + bits<5> ms1; + + let Inst{24 - 20} = 0b00000; + let Inst{19 - 15} = xs1; + let Inst{14 - 12} = 0b011; + let Inst{11 - 7} = ms1; +} + +let Predicates = [HasStdExtZtt], hasSideEffects = 1, mayLoad = 0, + mayStore = 1 in { + def ZTT_MSS : ZTT_STORE<0b0000101, "mss">; + def ZTT_MSS_CM : ZTT_STORE<0b0000100, "mss.cm">; + def ZTT_MSS_RM : ZTT_STORE<0b0000011, "mss.rm">; +} + +def Simm7AsmOperand : SImmAsmOperand<7>; + +// 7-bit signed intermediate instr operand class (for ZTT_MSHIFT_EW) +def simm7 : Operand { + let ParserMatchClass = Simm7AsmOperand; + let DecoderMethod = "decodeSImmOperand<7>"; + let OperandType = "OPERAND_IMMEDIATE"; +} + +let Predicates = [HasStdExtZtt], hasSideEffects = 1, mayLoad = 0, + mayStore = 0 in { + // mzero.2d - clears an accumulator. + def ZTT_MZERO_2D : RVInstAME32<0b0001000, (outs), + (ins ZttAccRegIndex: $acc), "mzero.2d", "$acc"> { + bits<2> acc; + + let Inst{24 - 10} = 0b000000000010001; + let Inst{9 - 8} = acc; + let Inst{7} = 0b0; + } + + // mmov.a.m - copy an Acc register into an M register. + def ZTT_MMOV_A_M : RVInstAME32<0b0000011, (outs ), + (ins ZttMatrixRegIndex: $md, + ZttAccRegIndex: $acc), + "mmov.a.m", "$md, $acc"> { + bits<2> acc; + bits<5> md; + + let Inst{24 - 22} = 0b000; + let Inst{21 - 20} = acc; + let Inst{19 - 12} = 0b00000100; + let Inst{11 - 7} = md; + } + + // mmov.m.m - copy one M register into another. + def ZTT_MMOV_M_M : RVInstAME32<0b0000010, (outs ), + (ins ZttMatrixRegIndex: $md, + ZttMatrixRegIndex: $ms), + "mmov.m.m", "$md, $ms"> { + bits<5> ms; + bits<5> md; + + let Inst{24 - 20} = ms; + let Inst{19 - 12} = 0b00000100; + let Inst{11 - 7} = md; + } + + // mgettyp - read an M register's dtype tag into a GPR. + def ZTT_MGETTYP : RVInstAME32<0b0000010, (outs GPR: $xd), + (ins ZttMatrixRegIndex: $ms1), + "mgettyp", "$xd, $ms1"> { + bits<5> ms1; + bits<5> xd; + + let Inst{24 - 20} = 0b00000; + let Inst{19 - 15} = ms1; + let Inst{14 - 12} = 0b101; + let Inst{11 - 7} = xd; + } + + // agettyp - read an Acc register's dtype tag into a GPR. + def ZTT_AGETTYP : RVInstAME32<0b0000100, (outs GPR: $xd), + (ins ZttAccRegIndex: $ad), + "agettyp", "$xd, $ad"> { + bits<2> ad; + bits<5> xd; + + let Inst{24 - 17} = 0b00000000; + let Inst{16 - 15} = ad; + let Inst{14 - 12} = 0b101; + let Inst{11 - 7} = xd; + } + + // msettyp - write a GPR value as an M register's dtype tag. + def ZTT_MSETTYP : RVInstAME32<0b0000001, (outs ), + (ins ZttMatrixRegIndex: $md, + GPR: $xs1), + "msettyp", "$md, $xs1"> { + bits<5> xs1; + bits<5> md; + + let Inst{24 - 20} = 0b00000; + let Inst{19 - 15} = xs1; + let Inst{14 - 12} = 0b101; + let Inst{11 - 7} = md; + } + + // asettyp - write a GPR value as an Acc register's dtype tag. + def ZTT_ASETTYP : RVInstAME32<0b0000011, (outs ), + (ins ZttAccRegIndex: $ad, + GPR: $xs1), + "asettyp", "$ad, $xs1"> { + bits<5> xs1; + bits<2> ad; + + let Inst{19 - 15} = xs1; + let Inst{14 - 10} = 0b10100; + let Inst{9 - 8} = ad; + let Inst{7} = 0b0; + } + + // mbcast.x - broadcast a GPR scalar to every element of an M register. + def ZTT_MBCAST_X : RVInstAME32<0b0000000, (outs ), + (ins ZttMatrixRegIndex: $md, + GPR: $xs1), + "mbcast.x", "$md, $xs1"> { + bits<5> xs1; + bits<5> md; + + let Inst{24 - 20} = 0b00000; + let Inst{19 - 15} = xs1; + let Inst{14 - 12} = 0b100; + let Inst{11 - 7} = md; + } + + + // mshift.ew - md[i,j] = ms1[i, j+imm], or 0 if j+imm is out of bounds + def ZTT_MSHIFT_EW : RVInst<(outs ), + (ins ZttMatrixRegIndex:$md, ZttMatrixRegIndex:$ms1, simm7:$imm), + "mshift.ew", "$md, $ms1, $imm", [], InstFormatOther> { + bits<5> ms1; + bits<7> imm; + bits<5> md; + + let Inst{31 - 27} = 0b00000; + let Inst{26 - 20} = imm; + let Inst{19 - 15} = ms1; + let Inst{14 - 12} = 0b110; + let Inst{11 - 7} = md; + let Inst{6 - 0} = 0b0101011; + } +} + +//===-----------------------------------------------------------------------------------===// +// Pattern Matching: LLVM IR Intrinsics -> AME (Ztt) 32-bit real/concrete instructions +//===-----------------------------------------------------------------------------------===// + +// Base pattern helper for AME unary operations +class AMEUnaryPat + : Pat<(intr timm:$md, timm:$ms1), + (inst timm:$md, timm:$ms1)>; + +let Predicates = [HasStdExtZtt] in { + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; + def : AMEUnaryPat; +} + +// Base pattern helper for matrix-matrix binary operations +class AMEBinMatPat + : Pat<(intr timm:$md, timm:$ms1, timm:$ms2), + (inst timm:$md, timm:$ms1, timm:$ms2)>; + +let Predicates = [HasStdExtZtt] in { + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; + def : AMEBinMatPat; +} + +// Base pattern helper for matrix-scalar operations. +// The second argument in the intrinsic is the scalar. +class AMEBinScalarPat + : Pat<(intr timm:$md, i64:$xs1, timm:$ms2), + (inst timm:$md, GPR:$xs1, timm:$ms2)>; + +let Predicates = [HasStdExtZtt, IsRV64] in { + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; + def : AMEBinScalarPat; +} + +// Base class helper for matrix multiplication operations. +// These instructions write to accumulator registers. +class AMEMatMulPat + : Pat<(intr timm:$acc, timm:$ms1, timm:$ms2), + (inst timm:$acc, timm:$ms1, timm:$ms2)>; + +let Predicates = [HasStdExtZtt] in { + def : AMEMatMulPat; + def : AMEMatMulPat; + def : AMEMatMulPat; + def : AMEMatMulPat; + def : AMEMatMulPat; + def : AMEMatMulPat; + def : AMEMatMulPat; + def : AMEMatMulPat; +} + +// Miscellaneous instructions +let Predicates = [HasStdExtZtt] in { + // Load/Store Patterns (Memory addresses are handled as ptr_ty -> GPR) + def : Pat<(int_riscv_ztt_mls timm:$md, iPTR:$xs1), (ZTT_MLS timm:$md, GPR:$xs1)>; + def : Pat<(int_riscv_ztt_mls_cm timm:$md, iPTR:$xs1), (ZTT_MLS_CM timm:$md, GPR:$xs1)>; + def : Pat<(int_riscv_ztt_mls_rm timm:$md, iPTR:$xs1), (ZTT_MLS_RM timm:$md, GPR:$xs1)>; + def : Pat<(int_riscv_ztt_mss timm:$ms1, iPTR:$xs1), (ZTT_MSS timm:$ms1, GPR:$xs1)>; + def : Pat<(int_riscv_ztt_mss_cm timm:$ms1, iPTR:$xs1), (ZTT_MSS_CM timm:$ms1, GPR:$xs1)>; + def : Pat<(int_riscv_ztt_mss_rm timm:$ms1, iPTR:$xs1), (ZTT_MSS_RM timm:$ms1, GPR:$xs1)>; + + // Broadcast & Zero + def : Pat<(int_riscv_ztt_mzero_2d timm:$acc), (ZTT_MZERO_2D timm:$acc)>; + + // Type operations + def : Pat<(int_riscv_ztt_mgettyp timm:$ms1), (ZTT_MGETTYP timm:$ms1)>; + def : Pat<(int_riscv_ztt_agettyp timm:$ad), (ZTT_AGETTYP timm:$ad)>; + + // Moves + def : Pat<(int_riscv_ztt_mmov_a_m timm:$md, timm:$acc), (ZTT_MMOV_A_M timm:$md, timm:$acc)>; + def : Pat<(int_riscv_ztt_mmov_m_m timm:$md, timm:$ms), (ZTT_MMOV_M_M timm:$md, timm:$ms)>; + def : Pat<(int_riscv_ztt_mshift_ew timm:$md, timm:$ms1, timm:$imm), + (ZTT_MSHIFT_EW timm:$md, timm:$ms1, timm:$imm)>; +} + +// Miscellaneous instructions with scalar operands +let Predicates = [HasStdExtZtt, IsRV64] in { + // Broadcast & Zero + def : Pat<(int_riscv_ztt_mbcast_x timm:$md, i64:$xs1), (ZTT_MBCAST_X timm:$md, GPR:$xs1)>; + + // Type operations + def : Pat<(int_riscv_ztt_msettyp timm:$md, i64:$xs1), (ZTT_MSETTYP timm:$md, GPR:$xs1)>; + def : Pat<(int_riscv_ztt_asettyp timm:$ad, i64:$xs1), (ZTT_ASETTYP timm:$ad, GPR:$xs1)>; +} + +// TODO: +// Define mpack.ew.x / munpack.ew.x (instruction defintiion/encoding and LLVM IR intrinsics -> MachineInstruction/MCInst isel patterns). +// +// As currently drafted in the spec, both encode to funct7=0000001, funct3=101, identical field +// positions. The spec gives no way to distinguish them. diff --git a/llvm/lib/Target/RISCV/RISCVMatrixExt.td b/llvm/lib/Target/RISCV/RISCVMatrixExt.td new file mode 100644 index 0000000000000..6218c337befd3 --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVMatrixExt.td @@ -0,0 +1,108 @@ +//===- RISCVMatrixExt.td --------------------------------------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// This file contains shared matrix register definitions and instruction +// includes for vendor matrix extensions. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Vendor-Specific AME (RISC-V Matrix Extension) Register Definitions +//===----------------------------------------------------------------------===// +// Reference: Vendor-Specific RISC-V Matrix Extension Specifications +// +// Data Flow: +// Memory → tr (via mlae/mlbe) → acc (via mma/mwma/mqma) → Memory (via msce) +//===----------------------------------------------------------------------===// + +let Namespace = "RISCV" in { + // Base class for BOSCAME Tile Registers (tr0-7) + // Used for input matrices A and B + // Size: MLEN bits per register (hardware-defined) + class BOSCAMETileReg Enc, string n> : Register { + let HWEncoding{2 - 0} = Enc; + let HWEncoding{4 - 3} = 0b00; // Distinguish from accumulation registers + } + + // Define 8 BOSCAME Tile Registers: tr0-tr7 + foreach i = 0 - 7 in { def BOSCAMETR #i : BOSCAMETileReg; } + + // Base class for BOSCAME Accumulation Registers (acc0-acc7) + // Used for output/accumulation matrix C + // Size: MLEN × AMUL bits per register (hardware-defined) + // Note: AMUL can be: + // - Fractional (1/8, 1/4, 1/2): For C = A × Báµ€ mode with large K + // - Integer (1, 2, 4, 8): For widening operations + // * AMUL=4: Required for mmi8i32 (int8→int32 quad-widen) + // * AMUL=2: Required for mmi16i32 (int16→int32 double-widen) + // * AMUL=8: Required for mmi4i32 (int4→int32 oct-widen) + class BOSCAMEAccReg Enc, string n> : Register { + let HWEncoding{2 - 0} = Enc; + let HWEncoding{4 - 3} = 0b01; // Distinguish from tile registers + } + + // Define 8 BOSCAME Accumulation Registers: acc0-acc7 + foreach i = 0 - 7 in { def BOSCAMEACC #i : BOSCAMEAccReg; } + + // Base class for XuanTie AME Matrix Registers (m0-m7) + class THeadAMEMatrixReg Enc, string n> : Register { + let HWEncoding{2 - 0} = Enc; + let HWEncoding{4 - 3} = 0b10; + } + + // Define 8 XuanTie AME Matrix Registers: m0-m7 + // Upstream RISCVRegisterInfo.td already defines M0-M7 as mask registers. Keep + // the TableGen def names distinct while preserving the XTAME assembly names. + foreach i = 0 - 7 in{def THeadAMEM #i: THeadAMEMatrixReg; +} +} // End Namespace = "RISCV" + +//===----------------------------------------------------------------------===// +// AME Register Classes +//===----------------------------------------------------------------------===// +// These register classes define the operand types for AME instructions +//===----------------------------------------------------------------------===// + +// Tile Register class (tr0-tr7) +// Used for input operands in matrix multiplication and load/store of input +// matrices +// Note: Size is set to 256 as a placeholder; actual size depends on +// MLEN +def BOSCAMETileReg : RegisterClass<"RISCV", [untyped], 256, + (add(sequence "BOSCAMETR%u", 0, 7))> { + let Size = 256; // Placeholder: actual MLEN is hardware-defined +} + +// Accumulation Register class (acc0-acc7) +// Used for output/accumulator in matrix multiplication and load/store of +// output/accumulator +// Note: Size can be 256×AMUL where AMUL ∈ {1/8, 1/4, 1/2, 1, +// 2, 4, 8}. +// We use 1024 as a reasonable upper bound (256 × 4 for int8→int32) +def BOSCAMEAccReg : RegisterClass<"RISCV", [untyped], 1024, + (add(sequence "BOSCAMEACC%u", 0, 7))> { + let Size = 1024; // Placeholder: actual MLEN×AMUL is hardware-defined +} + +// XuanTie AME Matrix Register class. +def THeadAMEMatrixReg : RegisterClass<"RISCV", [untyped], 1024, + (add(sequence "THeadAMEM%u", 0, 7))> { + let Size = 1024; // Placeholder: actual MLEN is hardware-defined +} + +include "RISCVInstrInfoXBOSCAME.td" +include "RISCVInstrInfoXTHeadAME.td" +include "RISCVInstrInfoZtt.td" diff --git a/llvm/test/CodeGen/RISCV/attributes-thead.ll b/llvm/test/CodeGen/RISCV/attributes-thead.ll index 77b0d03437fc3..d303217657fb6 100644 --- a/llvm/test/CodeGen/RISCV/attributes-thead.ll +++ b/llvm/test/CodeGen/RISCV/attributes-thead.ll @@ -23,6 +23,7 @@ ; RUN: llc -mtriple=riscv64 -mattr=+xtheadmempair %s -o - | FileCheck --check-prefix=RV64XTHEADMEMPAIR %s ; RUN: llc -mtriple=riscv64 -mattr=+xtheadsync %s -o - | FileCheck --check-prefix=RV64XTHEADSYNC %s ; RUN: llc -mtriple=riscv64 -mattr=+xtheadvdot %s -o - | FileCheck --check-prefixes=CHECK,RV64XTHEADVDOT %s +; RUN: llc -mtriple=riscv64 -mattr=+m,+a,+f,+d,+c,+xtheadame %s -o - | FileCheck --check-prefixes=CHECK,RV64XTHEADAME %s ; CHECK: .attribute 4, 16 @@ -49,6 +50,7 @@ ; RV64XTHEADMEMPAIR: .attribute 5, "rv64i2p1_xtheadmempair1p0" ; RV64XTHEADSYNC: .attribute 5, "rv64i2p1_xtheadsync1p0" ; RV64XTHEADVDOT: .attribute 5, "rv64i2p1_f2p2_d2p2_v1p0_zicsr2p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_xtheadvdot1p0" +; RV64XTHEADAME: .attribute 5, "rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zmmul1p0_zaamo1p0_zalrsc1p0_zca1p0_zcd1p0_xtheadmatrix3p0_xtheadmatrixmin0p5_xtheadmdma0p5_xtheadmew16b0p5_xtheadmew32b0p5_xtheadmew4b0p5_xtheadmew64b0p5_xtheadmew8b0p5_xtheadmfew0p5_xtheadmfic0p5_xtheadmhp0p5_xtheadmiew0p5_xtheadmmbf16bbf16b0p5_xtheadmmbf16bf32b0p5_xtheadmmbf20bf32b0p5_xtheadmmf16bf16b0p5_xtheadmmf16bf32b0p5_xtheadmmf32bf32b0p5_xtheadmmf32bf64b0p5_xtheadmmf4bbf16b0p5_xtheadmmf4bf16b0p5_xtheadmmf4bf32b0p5_xtheadmmf64bf64b0p5_xtheadmmf8bbf16b0p5_xtheadmmf8bf16b0p5_xtheadmmf8bf32b0p5_xtheadmmi4bi32b0p5_xtheadmmi8bi32b0p5_xtheadmmmxf4b0p5_xtheadmmmxf8b0p5_xtheadmmmxf8bmxf4b0p5_xtheadmred0p5" define i32 @addi(i32 %a) { %1 = add i32 %a, 1 diff --git a/llvm/test/CodeGen/RISCV/attributes-vendor-extensions.ll b/llvm/test/CodeGen/RISCV/attributes-vendor-extensions.ll new file mode 100644 index 0000000000000..87e6740836060 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/attributes-vendor-extensions.ll @@ -0,0 +1,30 @@ +;; Generate ELF attributes for selected vendor extensions from llc. + +; RUN: llc -mtriple=riscv32 -mattr=+xboscame %s -o - | FileCheck --check-prefixes=CHECK,RV32XBOSCAME %s +; RUN: llc -mtriple=riscv64 -mattr=+xboscame %s -o - | FileCheck --check-prefixes=CHECK,RV64XBOSCAME %s +; RUN: llc -mtriple=riscv32 -mattr=+xbbpebble %s -o - | FileCheck --check-prefixes=CHECK,RV32XBBPEBBLE %s +; RUN: llc -mtriple=riscv64 -mattr=+xbbpebble %s -o - | FileCheck --check-prefixes=CHECK,RV64XBBPEBBLE %s +; RUN: llc -mtriple=riscv32 -mattr=+xbbmeteor %s -o - | FileCheck --check-prefixes=CHECK,RV32XBBMETEOR %s +; RUN: llc -mtriple=riscv64 -mattr=+xbbmeteor %s -o - | FileCheck --check-prefixes=CHECK,RV64XBBMETEOR %s +; RUN: llc -mtriple=riscv32 -mattr=+xgemmini %s -o - | FileCheck --check-prefixes=CHECK,RV32XGEMMINI %s +; RUN: llc -mtriple=riscv64 -mattr=+xgemmini %s -o - | FileCheck --check-prefixes=CHECK,RV64XGEMMINI %s +; RUN: llc -mtriple=riscv32 -mattr=+v,+xsmtime %s -o - | FileCheck --check-prefixes=CHECK,RV32XSMTIME %s +; RUN: llc -mtriple=riscv64 -mattr=+v,+xsmtime %s -o - | FileCheck --check-prefixes=CHECK,RV64XSMTIME %s + +; CHECK: .attribute 4, 16 + +; RV32XBOSCAME: .attribute 5, "rv32i2p1_xxiangshaname1p0" +; RV64XBOSCAME: .attribute 5, "rv64i2p1_xxiangshaname1p0" +; RV32XBBPEBBLE: .attribute 5, "rv32i2p1_xbbpebble1p0" +; RV64XBBPEBBLE: .attribute 5, "rv64i2p1_xbbpebble1p0" +; RV32XBBMETEOR: .attribute 5, "rv32i2p1_xbbmeteor1p0" +; RV64XBBMETEOR: .attribute 5, "rv64i2p1_xbbmeteor1p0" +; RV32XGEMMINI: .attribute 5, "rv32i2p1_xgemmini1p0" +; RV64XGEMMINI: .attribute 5, "rv64i2p1_xgemmini1p0" +; RV32XSMTIME: .attribute 5, "rv32i2p1_f2p2_d2p2_v1p0_zicsr2p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_xsmtime1p0" +; RV64XSMTIME: .attribute 5, "rv64i2p1_f2p2_d2p2_v1p0_zicsr2p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_xsmtime1p0" + +define i32 @addi(i32 %a) { + %1 = add i32 %a, 1 + ret i32 %1 +} diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll index 26ffd8bb4ec89..39c76cb500381 100644 --- a/llvm/test/CodeGen/RISCV/attributes.ll +++ b/llvm/test/CodeGen/RISCV/attributes.ll @@ -113,6 +113,7 @@ ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvdot4a8i %s -o - | FileCheck --check-prefix=RV32ZVDOT4A8I %s ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvzip %s -o - | FileCheck --check-prefix=RV32ZVZIP %s ; RUN: llc -mtriple=riscv32 -mattr=+zvfh %s -o - | FileCheck --check-prefix=RV32ZVFH %s +; RUN: llc -mtriple=riscv32 -mattr=+experimental-ztt %s -o - | FileCheck --check-prefix=RV32ZTT %s ; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvabd %s -o - | FileCheck --check-prefix=RV32ZVABD %s ; RUN: llc -mtriple=riscv32 -mattr=+zicond %s -o - | FileCheck --check-prefix=RV32ZICOND %s ; RUN: llc -mtriple=riscv32 -mattr=+zilsd %s -o - | FileCheck --check-prefix=RV32ZILSD %s @@ -281,6 +282,7 @@ ; RUN: llc -mtriple=riscv64 -mattr=+zve32f -mattr=+experimental-zvfqwdota8f %s -o - | FileCheck --check-prefix=RV64ZVFQWDOTA8F %s ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvzip %s -o - | FileCheck --check-prefix=RV64ZVZIP %s ; RUN: llc -mtriple=riscv64 -mattr=+zvfh %s -o - | FileCheck --check-prefix=RV64ZVFH %s +; RUN: llc -mtriple=riscv64 -mattr=+experimental-ztt %s -o - | FileCheck --check-prefix=RV64ZTT %s ; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvabd %s -o - | FileCheck --check-prefix=RV64ZVABD %s ; RUN: llc -mtriple=riscv64 -mattr=+zicond %s -o - | FileCheck --check-prefix=RV64ZICOND %s ; RUN: llc -mtriple=riscv64 -mattr=+zimop %s -o - | FileCheck --check-prefix=RV64ZIMOP %s @@ -454,6 +456,7 @@ ; RV32ZVDOT4A8I: .attribute 5, "rv32i2p1_zicsr2p0_zvdot4a8i0p1_zve32x1p0_zvl32b1p0" ; RV32ZVZIP: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_zvzip0p1" ; RV32ZVFH: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfhmin1p0_zve32f1p0_zve32x1p0_zvfh1p0_zvfhmin1p0_zvl32b1p0" +; RV32ZTT: .attribute 5, "rv32i2p1_ztt0p1" ; RV32ZVABD: .attribute 5, "rv32i2p1_zicsr2p0_zvabd0p7_zve32x1p0_zvl32b1p0" ; RV32ZICOND: .attribute 5, "rv32i2p1_zicond1p0" ; RV32ZILSD: .attribute 5, "rv32i2p1_zilsd1p0" @@ -619,6 +622,7 @@ ; RV64ZVFQWDOTA8F: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvfqwdota8f0p2_zvl32b1p0" ; RV64ZVZIP: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_zvzip0p1" ; RV64ZVFH: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfhmin1p0_zve32f1p0_zve32x1p0_zvfh1p0_zvfhmin1p0_zvl32b1p0" +; RV64ZTT: .attribute 5, "rv64i2p1_ztt0p1" ; RV64ZVABD: .attribute 5, "rv64i2p1_zicsr2p0_zvabd0p7_zve32x1p0_zvl32b1p0" ; RV64ZICOND: .attribute 5, "rv64i2p1_zicond1p0" ; RV64ZIMOP: .attribute 5, "rv64i2p1_zimop1p0" diff --git a/llvm/test/CodeGen/RISCV/features-info.ll b/llvm/test/CodeGen/RISCV/features-info.ll index 142e6b2694937..097e55c1b0cba 100644 --- a/llvm/test/CodeGen/RISCV/features-info.ll +++ b/llvm/test/CodeGen/RISCV/features-info.ll @@ -33,6 +33,12 @@ ; CHECK-NEXT: experimental-zibi - 'Zibi' (Branch with Immediate). ; CHECK-NEXT: experimental-zicfilp - 'Zicfilp' (Landing pad). ; CHECK-NEXT: experimental-zicfiss - 'Zicfiss' (Shadow stack). +; CHECK-NEXT: experimental-ztt - 'Ztt' (Attached Matrix Extension). +; CHECK-NEXT: experimental-ztt-ame-accregs-1 - Enable support for AME accumulator register. +; CHECK-NEXT: experimental-ztt-ame-accregs-2 - Enable support for 2 AME accumulator registers. +; CHECK-NEXT: experimental-ztt-ame-accregs-4 - Enable support for 4 AME accumulator registers. +; CHECK-NEXT: experimental-ztt-ame-mregs-16 - Enable support for 16 AME matrix registers. +; CHECK-NEXT: experimental-ztt-ame-mregs-32 - Enable support for 32 AME matrix registers. ; CHECK-NEXT: experimental-zvabd - 'Zvabd' (Vector Absolute Difference). ; CHECK-NEXT: experimental-zvbc32e - 'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements). ; CHECK-NEXT: experimental-zvdot4a8i - 'Zvdot4a8i' (Vector 4-element Dot Product of packed 8-bit Integers). @@ -209,6 +215,9 @@ ; CHECK-NEXT: xandesvpackfph - 'XAndesVPackFPH' (Andes Vector Packed FP16 Extension). ; CHECK-NEXT: xandesvsinth - 'XAndesVSIntH' (Andes Vector Small INT Handling Extension). ; CHECK-NEXT: xandesvsintload - 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension). +; CHECK-NEXT: xbbmeteor - 'XBBMeteor' (Buckyball Large Language Model extension). +; CHECK-NEXT: xbbpebble - 'XBBPebble' (Buckyball CNN extension). +; CHECK-NEXT: xboscame - BOSC AME Alias (Enable xxiangshaname). ; CHECK-NEXT: xcheriot - 'XCheriot' (CHERIoT extension). ; CHECK-NEXT: xcvalu - 'XCValu' (CORE-V ALU Operations). ; CHECK-NEXT: xcvbi - 'XCVbi' (CORE-V Immediate Branching). @@ -217,6 +226,7 @@ ; CHECK-NEXT: xcvmac - 'XCVmac' (CORE-V Multiply-Accumulate). ; CHECK-NEXT: xcvmem - 'XCVmem' (CORE-V Post-incrementing Load & Store). ; CHECK-NEXT: xcvsimd - 'XCVsimd' (CORE-V SIMD ALU). +; CHECK-NEXT: xgemmini - 'XGemmini' (Gemmini extension). ; CHECK-NEXT: xmipscbop - 'XMIPSCBOP' (MIPS Software Prefetch). ; CHECK-NEXT: xmipscmov - 'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov)). ; CHECK-NEXT: xmipsexectl - 'XMIPSEXECTL' (MIPS execution control). @@ -245,13 +255,13 @@ ; CHECK-NEXT: xsfmm128t - 'XSfmm128t' (TE=128 configuration). ; CHECK-NEXT: xsfmm16t - 'XSfmm16t' (TE=16 configuration). ; CHECK-NEXT: xsfmm32a - 'XSfmm32a' (TEW=32-bit accumulation operands - int: 8b; float: fp16, bf16, fp32) -; CHECK-NEXT: xsfmm32a16f - 'XSfmm32a16f' (TEW=32-bit accumulation, operands - float: 16b, widen=2 (IEEE, BF)). -; CHECK-NEXT: xsfmm32a32f - 'XSfmm32a32f' (TEW=32-bit accumulation, operands - float: 32b). -; CHECK-NEXT: xsfmm32a8f - 'XSfmm32a8f' (TEW=32-bit accumulation, operands - float: fp8). -; CHECK-NEXT: xsfmm32a8i - 'XSfmm32a8i' (TEW=32-bit accumulation, operands - int: 8b). -; CHECK-NEXT: xsfmm32t - 'XSfmm32t' (TE=32 configuration). -; CHECK-NEXT: xsfmm64a64f - 'XSfmm64a64f' (TEW=64-bit accumulation, operands - float: fp64). -; CHECK-NEXT: xsfmm64t - 'XSfmm64t' (TE=64 configuration). +; CHECK-NEXT: xsfmm32a16f - 'XSfmm32a16f' (TEW=32-bit accumulation, operands - float: 16b, widen=2 (IEEE, BF)). +; CHECK-NEXT: xsfmm32a32f - 'XSfmm32a32f' (TEW=32-bit accumulation, operands - float: 32b). +; CHECK-NEXT: xsfmm32a8f - 'XSfmm32a8f' (TEW=32-bit accumulation, operands - float: fp8). +; CHECK-NEXT: xsfmm32a8i - 'XSfmm32a8i' (TEW=32-bit accumulation, operands - int: 8b). +; CHECK-NEXT: xsfmm32t - 'XSfmm32t' (TE=32 configuration). +; CHECK-NEXT: xsfmm64a64f - 'XSfmm64a64f' (TEW=64-bit accumulation, operands - float: fp64). +; CHECK-NEXT: xsfmm64t - 'XSfmm64t' (TE=64 configuration). ; CHECK-NEXT: xsfmmbase - 'XSfmmbase' (All non arithmetic instructions for all TEWs and sf.vtzero). ; CHECK-NEXT: xsfvcp - 'XSfvcp' (SiFive Custom Vector Coprocessor Interface Instructions). ; CHECK-NEXT: xsfvfbfexp16e - 'XSfvfbfexp16e' (SiFive Vector Floating-Point Exponential Function Instruction, BFloat16). @@ -265,8 +275,10 @@ ; CHECK-NEXT: xsfvqmaccqoq - 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4)). ; CHECK-NEXT: xsifivecdiscarddlone - 'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction). ; CHECK-NEXT: xsifivecflushdlone - 'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction). +; CHECK-NEXT: xsmtime - 'XSMTIME' (SpacemiT IME extension). ; CHECK-NEXT: xsmtvdot - 'XSMTVDot' (SpacemiT Vector Dot Product Extension). ; CHECK-NEXT: xsmtvdotii - 'XSMTVDotII' (SpacemiT Vector Extension for Matrix 2.0). +; CHECK-NEXT: xtheadame - Enable all T-Head AME matrix extensions. ; CHECK-NEXT: xtheadba - 'XTHeadBa' (T-Head address calculation instructions). ; CHECK-NEXT: xtheadbb - 'XTHeadBb' (T-Head basic bit-manipulation instructions). ; CHECK-NEXT: xtheadbs - 'XTHeadBs' (T-Head single-bit instructions). @@ -274,12 +286,45 @@ ; CHECK-NEXT: xtheadcondmov - 'XTHeadCondMov' (T-Head conditional move instructions). ; CHECK-NEXT: xtheadfmemidx - 'XTHeadFMemIdx' (T-Head FP Indexed Memory Operations). ; CHECK-NEXT: xtheadmac - 'XTHeadMac' (T-Head Multiply-Accumulate Instructions). +; CHECK-NEXT: xtheadmatrix - 'XTHeadMatrix' (T-Head AME matrix extension). +; CHECK-NEXT: xtheadmatrixmin - 'XTHeadMatrixMin' (T-Head AME minimal matrix extension). +; CHECK-NEXT: xtheadmdma - 'XTHeadMDMA' (T-Head AME matrix DMA extension). ; CHECK-NEXT: xtheadmemidx - 'XTHeadMemIdx' (T-Head Indexed Memory Operations). ; CHECK-NEXT: xtheadmempair - 'XTHeadMemPair' (T-Head two-GPR Memory Operations). +; CHECK-NEXT: xtheadmew16b - 'XTHeadMEW16B' (T-Head AME 16-bit element width extension). +; CHECK-NEXT: xtheadmew32b - 'XTHeadMEW32B' (T-Head AME 32-bit element width extension). +; CHECK-NEXT: xtheadmew4b - 'XTHeadMEW4B' (T-Head AME 4-bit element width extension). +; CHECK-NEXT: xtheadmew64b - 'XTHeadMEW64B' (T-Head AME 64-bit element width extension). +; CHECK-NEXT: xtheadmew8b - 'XTHeadMEW8B' (T-Head AME 8-bit element width extension). +; CHECK-NEXT: xtheadmfew - 'XTHeadMFEW' (T-Head AME matrix fractional element width extension). +; CHECK-NEXT: xtheadmfic - 'XTHeadMFIC' (T-Head AME matrix fixed-point conversion extension). +; CHECK-NEXT: xtheadmhp - 'XTHeadMHP' (T-Head AME matrix half precision extension). +; CHECK-NEXT: xtheadmiew - 'XTHeadMIEW' (T-Head AME matrix integer element width extension). +; CHECK-NEXT: xtheadmmbf16bbf16b - 'XTHeadMMBF16BBF16B' (T-Head AME BF16-by-BF16 matrix multiply extension). +; CHECK-NEXT: xtheadmmbf16bf32b - 'XTHeadMMBF16BF32B' (T-Head AME BF16-by-F32 matrix multiply extension). +; CHECK-NEXT: xtheadmmbf20bf32b - 'XTHeadMMBF20BF32B' (T-Head AME BF20-by-F32 matrix multiply extension). +; CHECK-NEXT: xtheadmmf16bf16b - 'XTHeadMMF16BF16B' (T-Head AME F16-by-BF16 matrix multiply extension). +; CHECK-NEXT: xtheadmmf16bf32b - 'XTHeadMMF16BF32B' (T-Head AME F16-by-F32 matrix multiply extension). +; CHECK-NEXT: xtheadmmf32bf32b - 'XTHeadMMF32BF32B' (T-Head AME F32-by-F32 matrix multiply extension). +; CHECK-NEXT: xtheadmmf32bf64b - 'XTHeadMMF32BF64B' (T-Head AME F32-by-F64 matrix multiply extension). +; CHECK-NEXT: xtheadmmf4bbf16b - 'XTHeadMMF4BBF16B' (T-Head AME F4-by-BF16 matrix multiply extension). +; CHECK-NEXT: xtheadmmf4bf16b - 'XTHeadMMF4BF16B' (T-Head AME F4-by-F16 matrix multiply extension). +; CHECK-NEXT: xtheadmmf4bf32b - 'XTHeadMMF4BF32B' (T-Head AME F4-by-F32 matrix multiply extension). +; CHECK-NEXT: xtheadmmf64bf64b - 'XTHeadMMF64BF64B' (T-Head AME F64-by-F64 matrix multiply extension). +; CHECK-NEXT: xtheadmmf8bbf16b - 'XTHeadMMF8BBF16B' (T-Head AME F8-by-BF16 matrix multiply extension). +; CHECK-NEXT: xtheadmmf8bf16b - 'XTHeadMMF8BF16B' (T-Head AME F8-by-F16 matrix multiply extension). +; CHECK-NEXT: xtheadmmf8bf32b - 'XTHeadMMF8BF32B' (T-Head AME F8-by-F32 matrix multiply extension). +; CHECK-NEXT: xtheadmmi4bi32b - 'XTHeadMMI4BI32B' (T-Head AME I4-by-I32 matrix multiply extension). +; CHECK-NEXT: xtheadmmi8bi32b - 'XTHeadMMI8BI32B' (T-Head AME I8-by-I32 matrix multiply extension). +; CHECK-NEXT: xtheadmmmxf4b - 'XTHeadMMMXF4B' (T-Head AME MXF4 matrix multiply extension). +; CHECK-NEXT: xtheadmmmxf8b - 'XTHeadMMMXF8B' (T-Head AME MXF8 matrix multiply extension). +; CHECK-NEXT: xtheadmmmxf8bmxf4b - 'XTHeadMMMXF8BMXF4B' (T-Head AME MXF8-by-MXF4 matrix multiply extension). +; CHECK-NEXT: xtheadmred - 'XTHeadMRED' (T-Head AME matrix reduction extension). ; CHECK-NEXT: xtheadsync - 'XTHeadSync' (T-Head multicore synchronization instructions). ; CHECK-NEXT: xtheadvdot - 'XTHeadVdot' (T-Head Vector Extensions for Dot). ; CHECK-NEXT: xventanacondops - 'XVentanaCondOps' (Ventana Conditional Ops). ; CHECK-NEXT: xwchc - 'Xwchc' (WCH/QingKe additional compressed opcodes). +; CHECK-NEXT: xxiangshaname - 'XXiangShanAME' (OpenXiangShan Advanced Matrix Extension). ; CHECK-NEXT: za128rs - 'Za128rs' (Reservation Set Size of at Most 128 Bytes). ; CHECK-NEXT: za64rs - 'Za64rs' (Reservation Set Size of at Most 64 Bytes). ; CHECK-NEXT: zaamo - 'Zaamo' (Atomic Memory Operations). diff --git a/llvm/test/CodeGen/RISCV/ztt-intrinsics.ll b/llvm/test/CodeGen/RISCV/ztt-intrinsics.ll new file mode 100644 index 0000000000000..37b10a6ed15bc --- /dev/null +++ b/llvm/test/CodeGen/RISCV/ztt-intrinsics.ll @@ -0,0 +1,992 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py + +; RUN: llc -mtriple=riscv64 -mattr=+experimental-ztt -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV64 + +define void @test_mabs_ew() { +; RV64-LABEL: test_mabs_ew: +; RV64: # %bb.0: +; RV64-NEXT: mabs.ew m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mabs.ew(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mabs.ew(i64, i64) + +define void @test_mcolunzip_ew() { +; RV64-LABEL: test_mcolunzip_ew: +; RV64: # %bb.0: +; RV64-NEXT: mcolunzip.ew m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mcolunzip.ew(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mcolunzip.ew(i64, i64) + +define void @test_mcolzip_ew() { +; RV64-LABEL: test_mcolzip_ew: +; RV64: # %bb.0: +; RV64-NEXT: mcolzip.ew m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mcolzip.ew(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mcolzip.ew(i64, i64) + +define void @test_mconv_ew() { +; RV64-LABEL: test_mconv_ew: +; RV64: # %bb.0: +; RV64-NEXT: mconv.ew m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mconv.ew(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mconv.ew(i64, i64) + +define void @test_mexp2_ew() { +; RV64-LABEL: test_mexp2_ew: +; RV64: # %bb.0: +; RV64-NEXT: mexp2.ew m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mexp2.ew(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mexp2.ew(i64, i64) + +define void @test_mlog2_ew() { +; RV64-LABEL: test_mlog2_ew: +; RV64: # %bb.0: +; RV64-NEXT: mlog2.ew m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mlog2.ew(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mlog2.ew(i64, i64) + +define void @test_mprefixadd_col() { +; RV64-LABEL: test_mprefixadd_col: +; RV64: # %bb.0: +; RV64-NEXT: mprefixadd.col m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mprefixadd.col(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mprefixadd.col(i64, i64) + +define void @test_mprefixadd_row() { +; RV64-LABEL: test_mprefixadd_row: +; RV64: # %bb.0: +; RV64-NEXT: mprefixadd.row m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mprefixadd.row(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mprefixadd.row(i64, i64) + +define void @test_mprefixmax_col() { +; RV64-LABEL: test_mprefixmax_col: +; RV64: # %bb.0: +; RV64-NEXT: mprefixmax.col m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mprefixmax.col(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mprefixmax.col(i64, i64) + +define void @test_mprefixmax_row() { +; RV64-LABEL: test_mprefixmax_row: +; RV64: # %bb.0: +; RV64-NEXT: mprefixmax.row m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mprefixmax.row(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mprefixmax.row(i64, i64) + +define void @test_mreduceadd_col() { +; RV64-LABEL: test_mreduceadd_col: +; RV64: # %bb.0: +; RV64-NEXT: mreduceadd.col m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mreduceadd.col(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mreduceadd.col(i64, i64) + +define void @test_mreduceadd_row() { +; RV64-LABEL: test_mreduceadd_row: +; RV64: # %bb.0: +; RV64-NEXT: mreduceadd.row m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mreduceadd.row(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mreduceadd.row(i64, i64) + +define void @test_mreducemax_col() { +; RV64-LABEL: test_mreducemax_col: +; RV64: # %bb.0: +; RV64-NEXT: mreducemax.col m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mreducemax.col(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mreducemax.col(i64, i64) + +define void @test_mreducemax_row() { +; RV64-LABEL: test_mreducemax_row: +; RV64: # %bb.0: +; RV64-NEXT: mreducemax.row m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mreducemax.row(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mreducemax.row(i64, i64) + +define void @test_mrowunzip_ew() { +; RV64-LABEL: test_mrowunzip_ew: +; RV64: # %bb.0: +; RV64-NEXT: mrowunzip.ew m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mrowunzip.ew(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mrowunzip.ew(i64, i64) + +define void @test_mabsdiff_ew() { +; RV64-LABEL: test_mabsdiff_ew: +; RV64: # %bb.0: +; RV64-NEXT: mabsdiff.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mabsdiff.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mabsdiff.ew(i64, i64, i64) + +define void @test_madd_ew() { +; RV64-LABEL: test_madd_ew: +; RV64: # %bb.0: +; RV64-NEXT: madd.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.madd.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.madd.ew(i64, i64, i64) + +define void @test_mand_ew() { +; RV64-LABEL: test_mand_ew: +; RV64: # %bb.0: +; RV64-NEXT: mand.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mand.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mand.ew(i64, i64, i64) + +define void @test_mandnot_ew() { +; RV64-LABEL: test_mandnot_ew: +; RV64: # %bb.0: +; RV64-NEXT: mandnot.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mandnot.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mandnot.ew(i64, i64, i64) + +define void @test_mcmovge_ew() { +; RV64-LABEL: test_mcmovge_ew: +; RV64: # %bb.0: +; RV64-NEXT: mcmovge.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mcmovge.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mcmovge.ew(i64, i64, i64) + +define void @test_mcmovlt_ew() { +; RV64-LABEL: test_mcmovlt_ew: +; RV64: # %bb.0: +; RV64-NEXT: mcmovlt.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mcmovlt.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mcmovlt.ew(i64, i64, i64) + +define void @test_mcmpge_ew() { +; RV64-LABEL: test_mcmpge_ew: +; RV64: # %bb.0: +; RV64-NEXT: mcmpge.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mcmpge.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mcmpge.ew(i64, i64, i64) + +define void @test_mcmplt_ew() { +; RV64-LABEL: test_mcmplt_ew: +; RV64: # %bb.0: +; RV64-NEXT: mcmplt.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mcmplt.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mcmplt.ew(i64, i64, i64) + +define void @test_mgather_ew() { +; RV64-LABEL: test_mgather_ew: +; RV64: # %bb.0: +; RV64-NEXT: mgather.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mgather.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mgather.ew(i64, i64, i64) + +define void @test_mhdiff_ew() { +; RV64-LABEL: test_mhdiff_ew: +; RV64: # %bb.0: +; RV64-NEXT: mhdiff.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mhdiff.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mhdiff.ew(i64, i64, i64) + +define void @test_mldexp_ew() { +; RV64-LABEL: test_mldexp_ew: +; RV64: # %bb.0: +; RV64-NEXT: mldexp.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mldexp.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mldexp.ew(i64, i64, i64) + +define void @test_mldexpacc_ew() { +; RV64-LABEL: test_mldexpacc_ew: +; RV64: # %bb.0: +; RV64-NEXT: mldexpacc.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mldexpacc.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mldexpacc.ew(i64, i64, i64) + +define void @test_mlog2sub_ew() { +; RV64-LABEL: test_mlog2sub_ew: +; RV64: # %bb.0: +; RV64-NEXT: mlog2sub.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mlog2sub.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mlog2sub.ew(i64, i64, i64) + +define void @test_mmax_ew() { +; RV64-LABEL: test_mmax_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmax.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmax.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmax.ew(i64, i64, i64) + +define void @test_mmean_ew() { +; RV64-LABEL: test_mmean_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmean.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmean.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmean.ew(i64, i64, i64) + +define void @test_mmin_ew() { +; RV64-LABEL: test_mmin_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmin.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmin.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmin.ew(i64, i64, i64) + +define void @test_mmul_ew() { +; RV64-LABEL: test_mmul_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmul.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmul.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmul.ew(i64, i64, i64) + +define void @test_mmulacc_ew() { +; RV64-LABEL: test_mmulacc_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmulacc.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulacc.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulacc.ew(i64, i64, i64) + +define void @test_mmulaccneg_ew() { +; RV64-LABEL: test_mmulaccneg_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmulaccneg.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulaccneg.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulaccneg.ew(i64, i64, i64) + +define void @test_mmuladd_ew() { +; RV64-LABEL: test_mmuladd_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmuladd.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmuladd.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmuladd.ew(i64, i64, i64) + +define void @test_mmulneg_ew() { +; RV64-LABEL: test_mmulneg_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmulneg.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulneg.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulneg.ew(i64, i64, i64) + +define void @test_mmulsub_ew() { +; RV64-LABEL: test_mmulsub_ew: +; RV64: # %bb.0: +; RV64-NEXT: mmulsub.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulsub.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulsub.ew(i64, i64, i64) + +define void @test_mor_ew() { +; RV64-LABEL: test_mor_ew: +; RV64: # %bb.0: +; RV64-NEXT: mor.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mor.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mor.ew(i64, i64, i64) + +define void @test_mornot_ew() { +; RV64-LABEL: test_mornot_ew: +; RV64: # %bb.0: +; RV64-NEXT: mornot.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mornot.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mornot.ew(i64, i64, i64) + +define void @test_mrdexp_ew() { +; RV64-LABEL: test_mrdexp_ew: +; RV64: # %bb.0: +; RV64-NEXT: mrdexp.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mrdexp.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mrdexp.ew(i64, i64, i64) + +define void @test_mrdexpacc_ew() { +; RV64-LABEL: test_mrdexpacc_ew: +; RV64: # %bb.0: +; RV64-NEXT: mrdexpacc.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mrdexpacc.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mrdexpacc.ew(i64, i64, i64) + +define void @test_mrowzip_ew() { +; RV64-LABEL: test_mrowzip_ew: +; RV64: # %bb.0: +; RV64-NEXT: mrowzip.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mrowzip.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mrowzip.ew(i64, i64, i64) + +define void @test_mscatadd_col() { +; RV64-LABEL: test_mscatadd_col: +; RV64: # %bb.0: +; RV64-NEXT: mscatadd.col m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mscatadd.col(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mscatadd.col(i64, i64, i64) + +define void @test_mscatadd_row() { +; RV64-LABEL: test_mscatadd_row: +; RV64: # %bb.0: +; RV64-NEXT: mscatadd.row m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mscatadd.row(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mscatadd.row(i64, i64, i64) + +define void @test_mscatmax_col() { +; RV64-LABEL: test_mscatmax_col: +; RV64: # %bb.0: +; RV64-NEXT: mscatmax.col m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mscatmax.col(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mscatmax.col(i64, i64, i64) + +define void @test_mscatmax_row() { +; RV64-LABEL: test_mscatmax_row: +; RV64: # %bb.0: +; RV64-NEXT: mscatmax.row m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mscatmax.row(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mscatmax.row(i64, i64, i64) + +define void @test_mselge_ew() { +; RV64-LABEL: test_mselge_ew: +; RV64: # %bb.0: +; RV64-NEXT: mselge.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mselge.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mselge.ew(i64, i64, i64) + +define void @test_msellt_ew() { +; RV64-LABEL: test_msellt_ew: +; RV64: # %bb.0: +; RV64-NEXT: msellt.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.msellt.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.msellt.ew(i64, i64, i64) + +define void @test_msub_ew() { +; RV64-LABEL: test_msub_ew: +; RV64: # %bb.0: +; RV64-NEXT: msub.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.msub.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.msub.ew(i64, i64, i64) + +define void @test_msublog2_ew() { +; RV64-LABEL: test_msublog2_ew: +; RV64: # %bb.0: +; RV64-NEXT: msublog2.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.msublog2.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.msublog2.ew(i64, i64, i64) + +define void @test_mxor_ew() { +; RV64-LABEL: test_mxor_ew: +; RV64: # %bb.0: +; RV64-NEXT: mxor.ew m1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mxor.ew(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mxor.ew(i64, i64, i64) + +define void @test_mabsdiff_ew_x(i64 %a1) { +; RV64-LABEL: test_mabsdiff_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mabsdiff.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mabsdiff.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mabsdiff.ew.x(i64, i64, i64) + +define void @test_madd_ew_x(i64 %a1) { +; RV64-LABEL: test_madd_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: madd.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.madd.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.madd.ew.x(i64, i64, i64) + +define void @test_mand_ew_x(i64 %a1) { +; RV64-LABEL: test_mand_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mand.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mand.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mand.ew.x(i64, i64, i64) + +define void @test_mandnot_ew_x(i64 %a1) { +; RV64-LABEL: test_mandnot_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mandnot.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mandnot.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mandnot.ew.x(i64, i64, i64) + +define void @test_mcmpge_ew_x(i64 %a1) { +; RV64-LABEL: test_mcmpge_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mcmpge.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mcmpge.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mcmpge.ew.x(i64, i64, i64) + +define void @test_mcmplt_ew_x(i64 %a1) { +; RV64-LABEL: test_mcmplt_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mcmplt.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mcmplt.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mcmplt.ew.x(i64, i64, i64) + +define void @test_mhdiff_ew_x(i64 %a1) { +; RV64-LABEL: test_mhdiff_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mhdiff.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mhdiff.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mhdiff.ew.x(i64, i64, i64) + +define void @test_mldexp_ew_x(i64 %a1) { +; RV64-LABEL: test_mldexp_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mldexp.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mldexp.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mldexp.ew.x(i64, i64, i64) + +define void @test_mldexpacc_ew_x(i64 %a1) { +; RV64-LABEL: test_mldexpacc_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mldexpacc.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mldexpacc.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mldexpacc.ew.x(i64, i64, i64) + +define void @test_mlog2sub_ew_x(i64 %a1) { +; RV64-LABEL: test_mlog2sub_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mlog2sub.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mlog2sub.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mlog2sub.ew.x(i64, i64, i64) + +define void @test_mmax_ew_x(i64 %a1) { +; RV64-LABEL: test_mmax_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmax.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmax.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmax.ew.x(i64, i64, i64) + +define void @test_mmean_ew_x(i64 %a1) { +; RV64-LABEL: test_mmean_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmean.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmean.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmean.ew.x(i64, i64, i64) + +define void @test_mmin_ew_x(i64 %a1) { +; RV64-LABEL: test_mmin_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmin.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmin.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmin.ew.x(i64, i64, i64) + +define void @test_mmul_ew_x(i64 %a1) { +; RV64-LABEL: test_mmul_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmul.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmul.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmul.ew.x(i64, i64, i64) + +define void @test_mmulacc_ew_x(i64 %a1) { +; RV64-LABEL: test_mmulacc_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmulacc.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulacc.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmulacc.ew.x(i64, i64, i64) + +define void @test_mmulaccneg_ew_x(i64 %a1) { +; RV64-LABEL: test_mmulaccneg_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmulaccneg.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulaccneg.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmulaccneg.ew.x(i64, i64, i64) + +define void @test_mmuladd_ew_x(i64 %a1) { +; RV64-LABEL: test_mmuladd_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmuladd.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmuladd.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmuladd.ew.x(i64, i64, i64) + +define void @test_mmulneg_ew_x(i64 %a1) { +; RV64-LABEL: test_mmulneg_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmulneg.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulneg.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmulneg.ew.x(i64, i64, i64) + +define void @test_mmulsub_ew_x(i64 %a1) { +; RV64-LABEL: test_mmulsub_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mmulsub.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulsub.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmulsub.ew.x(i64, i64, i64) + +define void @test_mor_ew_x(i64 %a1) { +; RV64-LABEL: test_mor_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mor.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mor.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mor.ew.x(i64, i64, i64) + +define void @test_mornot_ew_x(i64 %a1) { +; RV64-LABEL: test_mornot_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mornot.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mornot.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mornot.ew.x(i64, i64, i64) + +define void @test_msub_ew_x(i64 %a1) { +; RV64-LABEL: test_msub_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: msub.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.msub.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.msub.ew.x(i64, i64, i64) + +define void @test_msublog2_ew_x(i64 %a1) { +; RV64-LABEL: test_msublog2_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: msublog2.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.msublog2.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.msublog2.ew.x(i64, i64, i64) + +define void @test_mxor_ew_x(i64 %a1) { +; RV64-LABEL: test_mxor_ew_x: +; RV64: # %bb.0: +; RV64-NEXT: mxor.ew.x m1, a0, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mxor.ew.x(i64 1, i64 %a1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mxor.ew.x(i64, i64, i64) + +define void @test_mmul_2d() { +; RV64-LABEL: test_mmul_2d: +; RV64: # %bb.0: +; RV64-NEXT: mmul.2d acc1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmul.2d(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmul.2d(i64, i64, i64) + +define void @test_mmulacc_2d() { +; RV64-LABEL: test_mmulacc_2d: +; RV64: # %bb.0: +; RV64-NEXT: mmulacc.2d acc1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulacc.2d(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulacc.2d(i64, i64, i64) + +define void @test_mmulaccneg_2d() { +; RV64-LABEL: test_mmulaccneg_2d: +; RV64: # %bb.0: +; RV64-NEXT: mmulaccneg.2d acc1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulaccneg.2d(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulaccneg.2d(i64, i64, i64) + +define void @test_mmulat_2d() { +; RV64-LABEL: test_mmulat_2d: +; RV64: # %bb.0: +; RV64-NEXT: mmulat.2d acc1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulat.2d(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulat.2d(i64, i64, i64) + +define void @test_mmulatacc_2d() { +; RV64-LABEL: test_mmulatacc_2d: +; RV64: # %bb.0: +; RV64-NEXT: mmulatacc.2d acc1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulatacc.2d(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulatacc.2d(i64, i64, i64) + +define void @test_mmulbt_2d() { +; RV64-LABEL: test_mmulbt_2d: +; RV64: # %bb.0: +; RV64-NEXT: mmulbt.2d acc1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulbt.2d(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulbt.2d(i64, i64, i64) + +define void @test_mmulbtacc_2d() { +; RV64-LABEL: test_mmulbtacc_2d: +; RV64: # %bb.0: +; RV64-NEXT: mmulbtacc.2d acc1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulbtacc.2d(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulbtacc.2d(i64, i64, i64) + +define void @test_mmulneg_2d() { +; RV64-LABEL: test_mmulneg_2d: +; RV64: # %bb.0: +; RV64-NEXT: mmulneg.2d acc1, m2, m3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmulneg.2d(i64 1, i64 2, i64 3) + ret void +} +declare void @llvm.riscv.ztt.mmulneg.2d(i64, i64, i64) + +define void @test_mls(ptr %addr) { +; RV64-LABEL: test_mls: +; RV64: # %bb.0: +; RV64-NEXT: mls m1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mls(i64 1, ptr %addr) + ret void +} +declare void @llvm.riscv.ztt.mls(i64, ptr) + +define void @test_mls_cm(ptr %addr) { +; RV64-LABEL: test_mls_cm: +; RV64: # %bb.0: +; RV64-NEXT: mls.cm m1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mls.cm(i64 1, ptr %addr) + ret void +} +declare void @llvm.riscv.ztt.mls.cm(i64, ptr) + +define void @test_mls_rm(ptr %addr) { +; RV64-LABEL: test_mls_rm: +; RV64: # %bb.0: +; RV64-NEXT: mls.rm m1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mls.rm(i64 1, ptr %addr) + ret void +} +declare void @llvm.riscv.ztt.mls.rm(i64, ptr) + +define void @test_mss(ptr %addr) { +; RV64-LABEL: test_mss: +; RV64: # %bb.0: +; RV64-NEXT: mss m1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mss(i64 1, ptr %addr) + ret void +} +declare void @llvm.riscv.ztt.mss(i64, ptr) + +define void @test_mss_cm(ptr %addr) { +; RV64-LABEL: test_mss_cm: +; RV64: # %bb.0: +; RV64-NEXT: mss.cm m1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mss.cm(i64 1, ptr %addr) + ret void +} +declare void @llvm.riscv.ztt.mss.cm(i64, ptr) + +define void @test_mss_rm(ptr %addr) { +; RV64-LABEL: test_mss_rm: +; RV64: # %bb.0: +; RV64-NEXT: mss.rm m1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mss.rm(i64 1, ptr %addr) + ret void +} +declare void @llvm.riscv.ztt.mss.rm(i64, ptr) + +define void @test_mzero_2d() { +; RV64-LABEL: test_mzero_2d: +; RV64: # %bb.0: +; RV64-NEXT: mzero.2d acc1 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mzero.2d(i64 1) + ret void +} +declare void @llvm.riscv.ztt.mzero.2d(i64) + +define void @test_mmov_a_m() { +; RV64-LABEL: test_mmov_a_m: +; RV64: # %bb.0: +; RV64-NEXT: mmov.a.m m1, acc2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmov.a.m(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmov.a.m(i64, i64) + +define void @test_mmov_m_m() { +; RV64-LABEL: test_mmov_m_m: +; RV64: # %bb.0: +; RV64-NEXT: mmov.m.m m1, m2 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mmov.m.m(i64 1, i64 2) + ret void +} +declare void @llvm.riscv.ztt.mmov.m.m(i64, i64) + +define i64 @test_mgettyp() { +; RV64-LABEL: test_mgettyp: +; RV64: # %bb.0: +; RV64-NEXT: mgettyp a0, m1 +; RV64-NEXT: ret + %type = call i64 @llvm.riscv.ztt.mgettyp(i64 1) + ret i64 %type +} +declare i64 @llvm.riscv.ztt.mgettyp(i64) + +define i64 @test_agettyp() { +; RV64-LABEL: test_agettyp: +; RV64: # %bb.0: +; RV64-NEXT: agettyp a0, acc1 +; RV64-NEXT: ret + %type = call i64 @llvm.riscv.ztt.agettyp(i64 1) + ret i64 %type +} +declare i64 @llvm.riscv.ztt.agettyp(i64) + +define void @test_msettyp(i64 %a1) { +; RV64-LABEL: test_msettyp: +; RV64: # %bb.0: +; RV64-NEXT: msettyp m1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.msettyp(i64 1, i64 %a1) + ret void +} +declare void @llvm.riscv.ztt.msettyp(i64, i64) + +define void @test_asettyp(i64 %a1) { +; RV64-LABEL: test_asettyp: +; RV64: # %bb.0: +; RV64-NEXT: asettyp acc1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.asettyp(i64 1, i64 %a1) + ret void +} +declare void @llvm.riscv.ztt.asettyp(i64, i64) + +define void @test_mbcast_x(i64 %a1) { +; RV64-LABEL: test_mbcast_x: +; RV64: # %bb.0: +; RV64-NEXT: mbcast.x m1, a0 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mbcast.x(i64 1, i64 %a1) + ret void +} +declare void @llvm.riscv.ztt.mbcast.x(i64, i64) + +define void @test_mshift_ew_pos() { +; RV64-LABEL: test_mshift_ew_pos: +; RV64: # %bb.0: +; RV64-NEXT: mshift.ew m1, m2, 5 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mshift.ew(i64 1, i64 2, i64 5) + ret void +} + +define void @test_mshift_ew_neg() { +; RV64-LABEL: test_mshift_ew_neg: +; RV64: # %bb.0: +; RV64-NEXT: mshift.ew m1, m2, -3 +; RV64-NEXT: ret + call void @llvm.riscv.ztt.mshift.ew(i64 1, i64 2, i64 -3) + ret void +} +declare void @llvm.riscv.ztt.mshift.ew(i64, i64, i64) diff --git a/llvm/test/MC/RISCV/xboscame-valid.s b/llvm/test/MC/RISCV/xboscame-valid.s new file mode 100644 index 0000000000000..c9d1de0091f3c --- /dev/null +++ b/llvm/test/MC/RISCV/xboscame-valid.s @@ -0,0 +1,145 @@ +# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+xboscame %s \ +# RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +# RUN: llvm-mc -triple=riscv32 -show-encoding --mattr=+xboscame %s \ +# RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+xboscame %s \ +# RUN: | llvm-objdump --no-print-imm-hex -d --mattr=+xboscame - \ +# RUN: | FileCheck %s --check-prefix=CHECK-INST +# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+xboscame %s \ +# RUN: | llvm-objdump --no-print-imm-hex -d --mattr=+xboscame - \ +# RUN: | FileCheck %s --check-prefix=CHECK-INST +# RUN: echo "mma.h.mm acc0, tr1, tr2" \ +# RUN: | not llvm-mc -triple=riscv64 --mattr=+xtheadame 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-NO-XBOSCAME + +mma.h.mm acc0, tr1, tr2 +# CHECK-INST: mma.h.mm acc0, tr1, tr2 +# CHECK-ENCODING: [0x77,0x9c,0x28,0x20] + +mma.w.mm acc0, tr1, tr2 +# CHECK-INST: mma.w.mm acc0, tr1, tr2 +# CHECK-ENCODING: [0x77,0xac,0x28,0x20] + +mma.dw.mm acc0, tr1, tr2 +# CHECK-INST: mma.dw.mm acc0, tr1, tr2 +# CHECK-ENCODING: [0x77,0xbc,0x28,0x20] + +msettilem a0, a1 +# CHECK-INST: msettilem a0, a1 +# CHECK-ENCODING: [0x77,0xd5,0x05,0x04] + +msettilemi a0, 7 +# CHECK-INST: msettilemi a0, 7 +# CHECK-ENCODING: [0x77,0xd5,0x03,0x06] + +msettilen a0, a1 +# CHECK-INST: msettilen a0, a1 +# CHECK-ENCODING: [0x77,0xc5,0x05,0x04] + +msettileni a0, 7 +# CHECK-INST: msettileni a0, 7 +# CHECK-ENCODING: [0x77,0xc5,0x03,0x06] + +msettilek a0, a1 +# CHECK-INST: msettilek a0, a1 +# CHECK-ENCODING: [0x77,0xe5,0x05,0x04] + +msettileki a0, 7 +# CHECK-INST: msettileki a0, 7 +# CHECK-ENCODING: [0x77,0xe5,0x03,0x06] + +mlae8.m tr0, (a0), a1 +# CHECK-INST: mlae8.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x00,0xb5,0x04] + +mlae16.m tr0, (a0), a1 +# CHECK-INST: mlae16.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x10,0xb5,0x04] + +mlae32.m tr0, (a0), a1 +# CHECK-INST: mlae32.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x20,0xb5,0x04] + +mlae64.m tr0, (a0), a1 +# CHECK-INST: mlae64.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x30,0xb5,0x04] + +mlbe8.m tr0, (a0), a1 +# CHECK-INST: mlbe8.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x00,0xb5,0x08] + +mlbe16.m tr0, (a0), a1 +# CHECK-INST: mlbe16.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x10,0xb5,0x08] + +mlbe32.m tr0, (a0), a1 +# CHECK-INST: mlbe32.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x20,0xb5,0x08] + +mlbe64.m tr0, (a0), a1 +# CHECK-INST: mlbe64.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x30,0xb5,0x08] + +mlce8.m acc0, (a0), a1 +# CHECK-INST: mlce8.m acc0, (a0), a1 +# CHECK-ENCODING: [0x77,0x04,0xb5,0x00] + +mlce16.m acc0, (a0), a1 +# CHECK-INST: mlce16.m acc0, (a0), a1 +# CHECK-ENCODING: [0x77,0x14,0xb5,0x00] + +mlce32.m acc0, (a0), a1 +# CHECK-INST: mlce32.m acc0, (a0), a1 +# CHECK-ENCODING: [0x77,0x24,0xb5,0x00] + +mlce64.m acc0, (a0), a1 +# CHECK-INST: mlce64.m acc0, (a0), a1 +# CHECK-ENCODING: [0x77,0x34,0xb5,0x00] + +msae8.m tr0, (a0), a1 +# CHECK-INST: msae8.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x00,0xb5,0x06] + +msae16.m tr0, (a0), a1 +# CHECK-INST: msae16.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x10,0xb5,0x06] + +msae32.m tr0, (a0), a1 +# CHECK-INST: msae32.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x20,0xb5,0x06] + +msae64.m tr0, (a0), a1 +# CHECK-INST: msae64.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x30,0xb5,0x06] + +msbe8.m tr0, (a0), a1 +# CHECK-INST: msbe8.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x00,0xb5,0x0a] + +msbe16.m tr0, (a0), a1 +# CHECK-INST: msbe16.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x10,0xb5,0x0a] + +msbe32.m tr0, (a0), a1 +# CHECK-INST: msbe32.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x20,0xb5,0x0a] + +msbe64.m tr0, (a0), a1 +# CHECK-INST: msbe64.m tr0, (a0), a1 +# CHECK-ENCODING: [0x77,0x30,0xb5,0x0a] + +msce8.m acc0, (a0), a1 +# CHECK-INST: msce8.m acc0, (a0), a1 +# CHECK-ENCODING: [0x77,0x04,0xb5,0x02] + +msce16.m acc0, (a0), a1 +# CHECK-INST: msce16.m acc0, (a0), a1 +# CHECK-ENCODING: [0x77,0x14,0xb5,0x02] + +msce32.m acc0, (a0), a1 +# CHECK-INST: msce32.m acc0, (a0), a1 +# CHECK-ENCODING: [0x77,0x24,0xb5,0x02] + +msce64.m acc0, (a0), a1 +# CHECK-INST: msce64.m acc0, (a0), a1 +# CHECK-ENCODING: [0x77,0x34,0xb5,0x02] diff --git a/llvm/test/MC/RISCV/xgemmini-valid.s b/llvm/test/MC/RISCV/xgemmini-valid.s new file mode 100644 index 0000000000000..0d8f63f8815cf --- /dev/null +++ b/llvm/test/MC/RISCV/xgemmini-valid.s @@ -0,0 +1,96 @@ +# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+xgemmini %s \ +# RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+xgemmini %s \ +# RUN: | llvm-objdump -d --mattr=+xgemmini - \ +# RUN: | FileCheck %s --check-prefix=CHECK-INST +# RUN: not llvm-mc -triple=riscv64 --mattr=+xsmtime < %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-NO-XGEMMINI + +mvin ra, sp +# CHECK-INST: mvin ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x04] +# CHECK-NO-XGEMMINI: instruction requires the following: 'XGemmini' (Gemmini extension){{$}} + +mvin2 ra, sp +# CHECK-INST: mvin2 ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x02] + +mvin3 ra, sp +# CHECK-INST: mvin3 ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x1c] + +mvout ra, sp +# CHECK-INST: mvout ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x06] + +flush ra +# CHECK-INST: flush ra +# CHECK-ENCODING: [0x7b,0xb0,0x00,0x0e] + +config ra, sp +# CHECK-INST: config ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x00] + +preload ra, sp +# CHECK-INST: preload ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x0c] + +compute_preloaded ra, sp +# CHECK-INST: compute_preloaded ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x08] + +compute_accumulated ra, sp +# CHECK-INST: compute_accumulated ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x0a] + +loop_ws_config_bounds ra, sp +# CHECK-INST: loop_ws_config_bounds ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x12] + +loop_ws_config_addrs_ab ra, sp +# CHECK-INST: loop_ws_config_addrs_ab ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x14] + +loop_ws_config_addrs_dc ra, sp +# CHECK-INST: loop_ws_config_addrs_dc ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x16] + +loop_ws_config_strides_ab ra, sp +# CHECK-INST: loop_ws_config_strides_ab ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x18] + +loop_ws_config_strides_dc ra, sp +# CHECK-INST: loop_ws_config_strides_dc ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x1a] + +loop_ws ra, sp +# CHECK-INST: loop_ws ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x10] + +loop_conv_ws ra, sp +# CHECK-INST: loop_conv_ws ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x1e] + +loop_conv_ws_config1 ra, sp +# CHECK-INST: loop_conv_ws_config1 ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x20] + +loop_conv_ws_config2 ra, sp +# CHECK-INST: loop_conv_ws_config2 ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x22] + +loop_conv_ws_config3 ra, sp +# CHECK-INST: loop_conv_ws_config3 ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x24] + +loop_conv_ws_config4 ra, sp +# CHECK-INST: loop_conv_ws_config4 ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x26] + +loop_conv_ws_config5 ra, sp +# CHECK-INST: loop_conv_ws_config5 ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x28] + +loop_conv_ws_config6 ra, sp +# CHECK-INST: loop_conv_ws_config6 ra, sp +# CHECK-ENCODING: [0x7b,0xb0,0x20,0x2a] diff --git a/llvm/test/MC/RISCV/xsmtime-valid.s b/llvm/test/MC/RISCV/xsmtime-valid.s new file mode 100644 index 0000000000000..e9b8065ccddf5 --- /dev/null +++ b/llvm/test/MC/RISCV/xsmtime-valid.s @@ -0,0 +1,88 @@ +# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+v,+xsmtime %s \ +# RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+v,+xsmtime %s \ +# RUN: | llvm-objdump -d --mattr=+v,+xsmtime - \ +# RUN: | FileCheck %s --check-prefix=CHECK-INST +# RUN: not llvm-mc -triple=riscv64 --mattr=+v,+xgemmini < %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-NO-XSMTIME + +vmadot v0, v4, v8 +# CHECK-INST: vmadot v0, v4, v8 +# CHECK-ENCODING: [0x2b,0x00,0x82,0xe2] +# CHECK-NO-XSMTIME: instruction requires the following: 'XSMTIME' (SpacemiT IME extension){{$}} + +vmadotu v0, v4, v8 +# CHECK-INST: vmadotu v0, v4, v8 +# CHECK-ENCODING: [0x2b,0x30,0x82,0xe2] + +vmadotsu v0, v4, v8 +# CHECK-INST: vmadotsu v0, v4, v8 +# CHECK-ENCODING: [0x2b,0x10,0x82,0xe2] + +vmadotus v0, v4, v8 +# CHECK-INST: vmadotus v0, v4, v8 +# CHECK-ENCODING: [0x2b,0x20,0x82,0xe2] + +vfmadot v0, v4, v8 +# CHECK-INST: vfmadot v0, v4, v8 +# CHECK-ENCODING: [0x2b,0x00,0x82,0xea] + +vmadot1 v0, v8, v16 +# CHECK-INST: vmadot1 v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x00,0x04,0xe5] + +vmadot1u v0, v8, v16 +# CHECK-INST: vmadot1u v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x30,0x04,0xe5] + +vmadot1su v0, v8, v16 +# CHECK-INST: vmadot1su v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x10,0x04,0xe5] + +vmadot1us v0, v8, v16 +# CHECK-INST: vmadot1us v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x20,0x04,0xe5] + +vmadot2 v0, v8, v16 +# CHECK-INST: vmadot2 v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x00,0x04,0xe7] + +vmadot2u v0, v8, v16 +# CHECK-INST: vmadot2u v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x30,0x04,0xe7] + +vmadot2su v0, v8, v16 +# CHECK-INST: vmadot2su v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x10,0x04,0xe7] + +vmadot2us v0, v8, v16 +# CHECK-INST: vmadot2us v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x20,0x04,0xe7] + +vmadot3 v0, v8, v16 +# CHECK-INST: vmadot3 v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x00,0x04,0xe9] + +vmadot3u v0, v8, v16 +# CHECK-INST: vmadot3u v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x30,0x04,0xe9] + +vmadot3su v0, v8, v16 +# CHECK-INST: vmadot3su v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x10,0x04,0xe9] + +vmadot3us v0, v8, v16 +# CHECK-INST: vmadot3us v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x20,0x04,0xe9] + +vfmadot1 v0, v8, v16 +# CHECK-INST: vfmadot1 v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x00,0x04,0xed] + +vfmadot2 v0, v8, v16 +# CHECK-INST: vfmadot2 v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x00,0x04,0xef] + +vfmadot3 v0, v8, v16 +# CHECK-INST: vfmadot3 v0, v8, v16 +# CHECK-ENCODING: [0x2b,0x00,0x04,0xf1] diff --git a/llvm/test/MC/RISCV/xtheadame-valid.s b/llvm/test/MC/RISCV/xtheadame-valid.s new file mode 100644 index 0000000000000..31bfbda37e554 --- /dev/null +++ b/llvm/test/MC/RISCV/xtheadame-valid.s @@ -0,0 +1,113 @@ +# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+xtheadame %s \ +# RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+xtheadame %s \ +# RUN: | llvm-objdump --no-print-imm-hex -d --mattr=+xtheadame - \ +# RUN: | FileCheck %s --check-prefix=CHECK-INST + +th.mmacc.w.b m0, m1, m2 +# CHECK-INST: th.mmacc.w.b m0, m1, m2 +# CHECK-ENCODING: [0x2b,0x00,0x28,0x20] + +th.mmaccu.w.b m0, m1, m2 +# CHECK-INST: th.mmaccu.w.b m0, m1, m2 +# CHECK-ENCODING: [0xab,0x00,0x28,0x20] + +th.mfmacc.s m0, m1, m2 +# CHECK-INST: th.mfmacc.s m0, m1, m2 +# CHECK-ENCODING: [0x2b,0x08,0x28,0x10] + +th.mfmacc.d.s m0, m1, m2 +# CHECK-INST: th.mfmacc.d.s m0, m1, m2 +# CHECK-ENCODING: [0x2b,0x08,0x28,0x11] + +th.mcfgk a0 +# CHECK-INST: th.mcfgk a0 +# CHECK-ENCODING: [0x2b,0x00,0x05,0x8e] + +th.mcfgm a0 +# CHECK-INST: th.mcfgm a0 +# CHECK-ENCODING: [0x2b,0x00,0x05,0x9e] + +th.mcfgn a0 +# CHECK-INST: th.mcfgn a0 +# CHECK-ENCODING: [0x2b,0x00,0x05,0xae] + +th.mcfg a0 +# CHECK-INST: th.mcfg a0 +# CHECK-ENCODING: [0x2b,0x00,0x05,0xfe] + +th.mcfgki 7 +# CHECK-INST: th.mcfgki 7 +# CHECK-ENCODING: [0x2b,0x00,0x1c,0x0e] + +th.mcfgmi 7 +# CHECK-INST: th.mcfgmi 7 +# CHECK-ENCODING: [0x2b,0x00,0x1c,0x1e] + +th.mcfgni 7 +# CHECK-INST: th.mcfgni 7 +# CHECK-ENCODING: [0x2b,0x00,0x1c,0x2e] + +th.mlde8 m0, a1, a0 +# CHECK-INST: th.mlde8 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x00,0xb5,0x08] + +th.mlde16 m0, a1, a0 +# CHECK-INST: th.mlde16 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x04,0xb5,0x08] + +th.mlde32 m0, a1, a0 +# CHECK-INST: th.mlde32 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x08,0xb5,0x08] + +th.mlde64 m0, a1, a0 +# CHECK-INST: th.mlde64 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x0c,0xb5,0x08] + +th.mste8 m0, a1, a0 +# CHECK-INST: th.mste8 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x00,0xb5,0x0a] + +th.mste16 m0, a1, a0 +# CHECK-INST: th.mste16 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x04,0xb5,0x0a] + +th.mste32 m0, a1, a0 +# CHECK-INST: th.mste32 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x08,0xb5,0x0a] + +th.mste64 m0, a1, a0 +# CHECK-INST: th.mste64 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x0c,0xb5,0x0a] + +th.mldte8 m0, a1, a0 +# CHECK-INST: th.mldte8 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x00,0xb5,0x38] + +th.mldte16 m0, a1, a0 +# CHECK-INST: th.mldte16 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x04,0xb5,0x38] + +th.mldte32 m0, a1, a0 +# CHECK-INST: th.mldte32 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x08,0xb5,0x38] + +th.mldte64 m0, a1, a0 +# CHECK-INST: th.mldte64 m0, a1, a0 +# CHECK-ENCODING: [0x2b,0x0c,0xb5,0x38] + +th.mzero m0 +# CHECK-INST: th.mzero m0 +# CHECK-ENCODING: [0x2b,0x00,0x00,0xa0] + +th.mzero2r m0 +# CHECK-INST: th.mzero2r m0 +# CHECK-ENCODING: [0xab,0x00,0x00,0xa0] + +th.mzero4r m0 +# CHECK-INST: th.mzero4r m0 +# CHECK-ENCODING: [0xab,0x01,0x00,0xa0] + +th.mzero8r m0 +# CHECK-INST: th.mzero8r m0 +# CHECK-ENCODING: [0xab,0x03,0x00,0xa0] diff --git a/llvm/test/MC/RISCV/ztt-invalid.s b/llvm/test/MC/RISCV/ztt-invalid.s new file mode 100644 index 0000000000000..43f4298ef5630 --- /dev/null +++ b/llvm/test/MC/RISCV/ztt-invalid.s @@ -0,0 +1,68 @@ + # RUN: not llvm-mc -triple=riscv64 --mattr=-experimental-ztt-ame-mregs-16,-experimental-ztt-ame-mregs-32 %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-NO-M-CONFIG +# RUN: not llvm-mc -triple=riscv64 --mattr=+experimental-ztt-ame-mregs-16,+experimental-ztt-ame-mregs-32 %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-CONFLICT-M + +# CHECK-NO-M-CONFIG: error: no ztt (AME) matrix register count feature enabled +# CHECK-CONFLICT-M: error: conflicting ztt (AME) matrix register bounds chosen; cannot enable both 16 and 32 matrix registers simultaneously + +# RUN: not llvm-mc -triple=riscv64 --mattr=-experimental-ztt-ame-accregs-1,-experimental-ztt-ame-accregs-2,-experimental-ztt-ame-accregs-4 %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-NO-ACC-CONFIG +# RUN: not llvm-mc -triple=riscv64 --mattr=+experimental-ztt-ame-accregs-1,+experimental-ztt-ame-accregs-2 %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-CONFLICT-ACC + +# CHECK-NO-ACC-CONFIG: error: no ztt (AME) accumulator register count feature enabled +# CHECK-CONFLICT-ACC: error: conflicting ztt (AME) accumulator register bounds chosen; cannot enable multiple accumulator register configurations simultaneously + +# RUN: not llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-ztt %s 2>&1 \ +# RUN: | FileCheck %s --check-prefixes=CHECK +# RUN: not llvm-mc -triple=riscv32 -show-encoding --mattr=+experimental-ztt %s 2>&1 \ +# RUN: | FileCheck %s --check-prefixes=CHECK +# RUN: not llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-ztt,+experimental-ztt-ame-mregs-16,-experimental-ztt-ame-mregs-32 %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-M16 +# RUN: not llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-ztt,+experimental-ztt-ame-accregs-1,-experimental-ztt-ame-accregs-2,-experimental-ztt-ame-accregs-4 %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-ACC1 +# RUN: not llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-ztt,+experimental-ztt-ame-accregs-2,-experimental-ztt-ame-accregs-4 %s 2>&1 \ +# RUN: | FileCheck %s --check-prefix=CHECK-ACC2 + +# Immediate underflow: Value falls below the 7-bit signed boundary +# CHECK: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: {{invalid operand for instruction|invalid instruction}} +mshift.ew m1, m2, -65 + +# Immediate overflow: Value exceeds the 7-bit signed boundary +# CHECK: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: {{invalid operand for instruction|invalid instruction}} +mshift.ew m1, m2, 64 + +# Base register mismatch: Instruction expects matrix register class +# but gets scalar GPR. +# CHECK: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: {{invalid operand for instruction|invalid instruction}} +mshift.ew x10, m2, 5 + +# Operand type mismatch: Instruction expects scalar register class but +# gets accumulator register. +# CHECK: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: register must be a GPR +msettyp m1, acc1 + +# Destination target mismatch: Broadcast instruction targets accumulator +# instead of matrix register. +# CHECK: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: {{invalid operand for instruction|invalid instruction}} +mbcast.x acc1, a0 + +# Syntax error: Operand count is lower than instruction signature requirements +# CHECK: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: too few operands for instruction +mshift.ew m1, m2 + +# Matrix register out of bounds: Attempting to access m16 when constraint +# drops limit down to 16. +# CHECK-M16: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: {{invalid operand for instruction|invalid instruction}} +mabs.ew m16, m2 + +# Accumulator register out of bounds: Attempting to access acc1 when configuration +# limits context to 1 accumulator (acc0). +# CHECK-ACC1: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: {{invalid operand for instruction|invalid instruction}} +agettyp a0, acc1 + +# Accumulator register out of bounds: Attempting to access acc2 when configuration +# limits context to 2 accumulators (acc0-acc1). +# CHECK-ACC2: {{.*}}:{{[0-9]+}}:{{[0-9]+}}: error: {{invalid operand for instruction|invalid instruction}} +agettyp a0, acc2 diff --git a/llvm/test/MC/RISCV/ztt-valid.s b/llvm/test/MC/RISCV/ztt-valid.s new file mode 100644 index 0000000000000..458749b84d7c2 --- /dev/null +++ b/llvm/test/MC/RISCV/ztt-valid.s @@ -0,0 +1,408 @@ +# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+experimental-ztt %s \ +# RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +# RUN: llvm-mc -triple=riscv32 -show-encoding --mattr=+experimental-ztt %s \ +# RUN: | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST +# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+experimental-ztt %s \ +# RUN: | llvm-objdump --no-print-imm-hex -d --mattr=+experimental-ztt - \ +# RUN: | FileCheck %s --check-prefix=CHECK-INST +# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+experimental-ztt %s \ +# RUN: | llvm-objdump --no-print-imm-hex -d --mattr=+experimental-ztt - \ +# RUN: | FileCheck %s --check-prefix=CHECK-INST + +mabs.ew m1, m2 +# CHECK-INST: mabs.ew m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x00] + +mcolunzip.ew m1, m2 +# CHECK-INST: mcolunzip.ew m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x08] + +mcolzip.ew m1, m2 +# CHECK-INST: mcolzip.ew m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x06] + +mconv.ew m1, m2 +# CHECK-INST: mconv.ew m1, m2 +# CHECK-ENCODING: [0xab,0x50,0x01,0x00] + +mexp2.ew m1, m2 +# CHECK-INST: mexp2.ew m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x02] + +mlog2.ew m1, m2 +# CHECK-INST: mlog2.ew m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x04] + +mprefixadd.col m1, m2 +# CHECK-INST: mprefixadd.col m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x18] + +mprefixadd.row m1, m2 +# CHECK-INST: mprefixadd.row m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x10] + +mprefixmax.col m1, m2 +# CHECK-INST: mprefixmax.col m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x1a] + +mprefixmax.row m1, m2 +# CHECK-INST: mprefixmax.row m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x12] + +mreduceadd.col m1, m2 +# CHECK-INST: mreduceadd.col m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x14] + +mreduceadd.row m1, m2 +# CHECK-INST: mreduceadd.row m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x0c] + +mreducemax.col m1, m2 +# CHECK-INST: mreducemax.col m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x16] + +mreducemax.row m1, m2 +# CHECK-INST: mreducemax.row m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x0e] + +mrowunzip.ew m1, m2 +# CHECK-INST: mrowunzip.ew m1, m2 +# CHECK-ENCODING: [0xab,0x10,0x01,0x0a] + +mabsdiff.ew m1, m2, m3 +# CHECK-INST: mabsdiff.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x08] + +madd.ew m1, m2, m3 +# CHECK-INST: madd.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x00] + +mand.ew m1, m2, m3 +# CHECK-INST: mand.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x0e] + +mandnot.ew m1, m2, m3 +# CHECK-INST: mandnot.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x10] + +mcmovge.ew m1, m2, m3 +# CHECK-INST: mcmovge.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x20] + +mcmovlt.ew m1, m2, m3 +# CHECK-INST: mcmovlt.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x22] + +mcmpge.ew m1, m2, m3 +# CHECK-INST: mcmpge.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x18] + +mcmplt.ew m1, m2, m3 +# CHECK-INST: mcmplt.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x1a] + +mgather.ew m1, m2, m3 +# CHECK-INST: mgather.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x30] + +mhdiff.ew m1, m2, m3 +# CHECK-INST: mhdiff.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x06] + +mldexp.ew m1, m2, m3 +# CHECK-INST: mldexp.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x24] + +mldexpacc.ew m1, m2, m3 +# CHECK-INST: mldexpacc.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x26] + +mlog2sub.ew m1, m2, m3 +# CHECK-INST: mlog2sub.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x2c] + +mmax.ew m1, m2, m3 +# CHECK-INST: mmax.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x0a] + +mmean.ew m1, m2, m3 +# CHECK-INST: mmean.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x04] + +mmin.ew m1, m2, m3 +# CHECK-INST: mmin.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x0c] + +mmul.ew m1, m2, m3 +# CHECK-INST: mmul.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x38] + +mmulacc.ew m1, m2, m3 +# CHECK-INST: mmulacc.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x3a] + +mmulaccneg.ew m1, m2, m3 +# CHECK-INST: mmulaccneg.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x40] + +mmuladd.ew m1, m2, m3 +# CHECK-INST: mmuladd.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x3c] + +mmulneg.ew m1, m2, m3 +# CHECK-INST: mmulneg.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x42] + +mmulsub.ew m1, m2, m3 +# CHECK-INST: mmulsub.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x3e] + +mor.ew m1, m2, m3 +# CHECK-INST: mor.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x12] + +mornot.ew m1, m2, m3 +# CHECK-INST: mornot.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x14] + +mrdexp.ew m1, m2, m3 +# CHECK-INST: mrdexp.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x28] + +mrdexpacc.ew m1, m2, m3 +# CHECK-INST: mrdexpacc.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x2a] + +mrowzip.ew m1, m2, m3 +# CHECK-INST: mrowzip.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x36] + +mscatadd.col m1, m2, m3 +# CHECK-INST: mscatadd.col m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x44] + +mscatadd.row m1, m2, m3 +# CHECK-INST: mscatadd.row m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x32] + +mscatmax.col m1, m2, m3 +# CHECK-INST: mscatmax.col m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x46] + +mscatmax.row m1, m2, m3 +# CHECK-INST: mscatmax.row m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x34] + +mselge.ew m1, m2, m3 +# CHECK-INST: mselge.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x1c] + +msellt.ew m1, m2, m3 +# CHECK-INST: msellt.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x1e] + +msub.ew m1, m2, m3 +# CHECK-INST: msub.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x02] + +msublog2.ew m1, m2, m3 +# CHECK-INST: msublog2.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x2e] + +mxor.ew m1, m2, m3 +# CHECK-INST: mxor.ew m1, m2, m3 +# CHECK-ENCODING: [0xab,0x00,0x31,0x16] + +mabsdiff.ew.x m1, a0, m3 +# CHECK-INST: mabsdiff.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x0a] + +madd.ew.x m1, a0, m3 +# CHECK-INST: madd.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x00] + +mand.ew.x m1, a0, m3 +# CHECK-INST: mand.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x0e] + +mandnot.ew.x m1, a0, m3 +# CHECK-INST: mandnot.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x10] + +mcmpge.ew.x m1, a0, m3 +# CHECK-INST: mcmpge.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x18] + +mcmplt.ew.x m1, a0, m3 +# CHECK-INST: mcmplt.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x1a] + +mhdiff.ew.x m1, a0, m3 +# CHECK-INST: mhdiff.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x0c] + +mldexp.ew.x m1, a0, m3 +# CHECK-INST: mldexp.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x20] + +mldexpacc.ew.x m1, a0, m3 +# CHECK-INST: mldexpacc.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x22] + +mlog2sub.ew.x m1, a0, m3 +# CHECK-INST: mlog2sub.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x1c] + +mmax.ew.x m1, a0, m3 +# CHECK-INST: mmax.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x06] + +mmean.ew.x m1, a0, m3 +# CHECK-INST: mmean.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x08] + +mmin.ew.x m1, a0, m3 +# CHECK-INST: mmin.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x04] + +mmul.ew.x m1, a0, m3 +# CHECK-INST: mmul.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x24] + +mmulacc.ew.x m1, a0, m3 +# CHECK-INST: mmulacc.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x26] + +mmulaccneg.ew.x m1, a0, m3 +# CHECK-INST: mmulaccneg.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x2c] + +mmuladd.ew.x m1, a0, m3 +# CHECK-INST: mmuladd.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x28] + +mmulneg.ew.x m1, a0, m3 +# CHECK-INST: mmulneg.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x2e] + +mmulsub.ew.x m1, a0, m3 +# CHECK-INST: mmulsub.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x2a] + +mor.ew.x m1, a0, m3 +# CHECK-INST: mor.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x12] + +mornot.ew.x m1, a0, m3 +# CHECK-INST: mornot.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x14] + +msub.ew.x m1, a0, m3 +# CHECK-INST: msub.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x02] + +msublog2.ew.x m1, a0, m3 +# CHECK-INST: msublog2.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x1e] + +mxor.ew.x m1, a0, m3 +# CHECK-INST: mxor.ew.x m1, a0, m3 +# CHECK-ENCODING: [0xab,0x20,0x35,0x16] + +mmul.2d acc1, m2, m3 +# CHECK-INST: mmul.2d acc1, m2, m3 +# CHECK-ENCODING: [0x2b,0x21,0x31,0x00] + +mmulacc.2d acc1, m2, m3 +# CHECK-INST: mmulacc.2d acc1, m2, m3 +# CHECK-ENCODING: [0x2b,0x21,0x31,0x02] + +mmulaccneg.2d acc1, m2, m3 +# CHECK-INST: mmulaccneg.2d acc1, m2, m3 +# CHECK-ENCODING: [0x2b,0x21,0x31,0x06] + +mmulat.2d acc1, m2, m3 +# CHECK-INST: mmulat.2d acc1, m2, m3 +# CHECK-ENCODING: [0x2b,0x21,0x31,0x08] + +mmulatacc.2d acc1, m2, m3 +# CHECK-INST: mmulatacc.2d acc1, m2, m3 +# CHECK-ENCODING: [0x2b,0x21,0x31,0x0a] + +mmulbt.2d acc1, m2, m3 +# CHECK-INST: mmulbt.2d acc1, m2, m3 +# CHECK-ENCODING: [0x2b,0x21,0x31,0x0c] + +mmulbtacc.2d acc1, m2, m3 +# CHECK-INST: mmulbtacc.2d acc1, m2, m3 +# CHECK-ENCODING: [0x2b,0x21,0x31,0x0e] + +mmulneg.2d acc1, m2, m3 +# CHECK-INST: mmulneg.2d acc1, m2, m3 +# CHECK-ENCODING: [0x2b,0x21,0x31,0x04] + +# Interoperability loads (md, xs1:address) +mls m1, a0 +# CHECK-INST: mls m1, a0 +# CHECK-ENCODING: [0xab,0x30,0x05,0x04] + +mls.cm m1, a0 +# CHECK-INST: mls.cm m1, a0 +# CHECK-ENCODING: [0xab,0x30,0x05,0x02] + +mls.rm m1, a0 +# CHECK-INST: mls.rm m1, a0 +# CHECK-ENCODING: [0xab,0x30,0x05,0x00] + +# Interoperability stores (ms1, xs1:address) +mss m1, a0 +# CHECK-INST: mss m1, a0 +# CHECK-ENCODING: [0xab,0x30,0x05,0x0a] + +mss.cm m1, a0 +# CHECK-INST: mss.cm m1, a0 +# CHECK-ENCODING: [0xab,0x30,0x05,0x08] + +mss.rm m1, a0 +# CHECK-INST: mss.rm m1, a0 +# CHECK-ENCODING: [0xab,0x30,0x05,0x06] + +mzero.2d acc1 +# CHECK-INST: mzero.2d acc1 +# CHECK-ENCODING: [0x2b,0x45,0x00,0x10] + +mmov.a.m m1, acc2 +# CHECK-INST: mmov.a.m m1, acc2 +# CHECK-ENCODING: [0xab,0x40,0x20,0x06] + +mmov.m.m m1, m2 +# CHECK-INST: mmov.m.m m1, m2 +# CHECK-ENCODING: [0xab,0x40,0x20,0x04] + +mgettyp a0, m1 +# CHECK-INST: mgettyp a0, m1 +# CHECK-ENCODING: [0x2b,0xd5,0x00,0x04] + +agettyp a0, acc1 +# CHECK-INST: agettyp a0, acc1 +# CHECK-ENCODING: [0x2b,0xd5,0x00,0x08] + +msettyp m1, a0 +# CHECK-INST: msettyp m1, a0 +# CHECK-ENCODING: [0xab,0x50,0x05,0x02] + +asettyp acc1, a0 +# CHECK-INST: asettyp acc1, a0 +# CHECK-ENCODING: [0x2b,0x51,0x05,0x06] + +mbcast.x m1, a0 +# CHECK-INST: mbcast.x m1, a0 +# CHECK-ENCODING: [0xab,0x40,0x05,0x00] + +mshift.ew m1, m2, 5 +# CHECK-INST: mshift.ew m1, m2, 5 +# CHECK-ENCODING: [0xab,0x60,0x51,0x00] + +mshift.ew m1, m2, -3 +# CHECK-INST: mshift.ew m1, m2, -3 +# CHECK-ENCODING: [0xab,0x60,0xd1,0x07] diff --git a/mlir/include/mlir/IR/Visitors.h b/mlir/include/mlir/IR/Visitors.h index 907f470c0d248..d0c123c3bc600 100644 --- a/mlir/include/mlir/IR/Visitors.h +++ b/mlir/include/mlir/IR/Visitors.h @@ -120,9 +120,10 @@ void walk(Operation *op, function_ref callback, WalkOrder order) { for (auto ®ion : Iterator::makeIterable(*op)) { // Early increment here in the case where the block is erased. - // PostOrderTraversal keeps state outside of iterators, so store it here. - auto &&It = Iterator::makeIterable(region); - for (auto &block : llvm::make_early_inc_range(It)) { + // Store the block range to ensure the iteratable (e.g., + // PostOrderTraversal) outlives the iterators of make_early_inc_range. + auto &&blockRange = Iterator::makeIterable(region); + for (auto &block : llvm::make_early_inc_range(blockRange)) { if (order == WalkOrder::PreOrder) callback(&block); for (auto &nestedOp : Iterator::makeIterable(block)) @@ -196,9 +197,10 @@ WalkResult walk(Operation *op, function_ref callback, WalkOrder order) { for (auto ®ion : Iterator::makeIterable(*op)) { // Early increment here in the case where the block is erased. - // PostOrderTraversal keeps state outside of iterators, so store it here. - auto &&It = Iterator::makeIterable(region); - for (auto &block : llvm::make_early_inc_range(It)) { + // Store the block range to ensure the iteratable (e.g., + // PostOrderTraversal) outlives the iterators of make_early_inc_range. + auto &&blockRange = Iterator::makeIterable(region); + for (auto &block : llvm::make_early_inc_range(blockRange)) { if (order == WalkOrder::PreOrder) { WalkResult result = callback(&block); if (result.wasSkipped()) diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp index 0bb130eac1bff..1621ad1bc4349 100644 --- a/mlir/lib/Dialect/Shape/IR/Shape.cpp +++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp @@ -48,6 +48,9 @@ bool shape::isExtentTensorType(Type type) { LogicalResult shape::getShapeVec(Value input, SmallVectorImpl &shapeValues) { + // Look through tensor.cast operations to find the underlying shape. + if (auto castOp = input.getDefiningOp()) + return getShapeVec(castOp.getSource(), shapeValues); if (auto inputOp = input.getDefiningOp()) { auto type = llvm::cast(inputOp.getArg().getType()); if (!type.hasRank()) @@ -799,27 +802,27 @@ struct CanonicalizeCastExtentTensorOperandsPattern LogicalResult matchAndRewrite(OpTy op, PatternRewriter &rewriter) const override { - // Canonicalize operands. + // Canonicalize operands by stripping information-losing tensor.cast ops. + SmallVector newOperands; bool anyChange = false; - auto canonicalizeOperand = [&](Value operand) -> Value { + for (Value operand : op.getShapes()) { if (auto castOp = operand.getDefiningOp()) { // Only eliminate the cast if it holds no shape information. - bool isInformationLoosingCast = - llvm::cast(castOp.getType()).isDynamicDim(0); - if (isInformationLoosingCast) { + if (llvm::cast(castOp.getType()).isDynamicDim(0)) { anyChange = true; - return castOp.getSource(); + newOperands.push_back(castOp.getSource()); + continue; } } - return operand; - }; - auto newOperands = - llvm::map_to_vector<8>(op.getOperands(), canonicalizeOperand); + newOperands.push_back(operand); + } // Rewrite op if any change required. if (!anyChange) return failure(); - rewriter.replaceOpWithNewOp(op, op->getResultTypes(), newOperands); + rewriter.modifyOpInPlace(op, [&]() { + op.getShapesMutable().assign(newOperands); + }); return success(); } }; @@ -1017,6 +1020,22 @@ OpFoldResult CstrBroadcastableOp::fold(FoldAdaptor adaptor) { }()) return BoolAttr::get(getContext(), true); + // No broadcasting is needed if all operands but one are scalar, using + // getShapeVec to look through tensor.cast and shape_of ops. + if ([&] { + bool nonScalarSeen = false; + for (auto shapeValue : getShapes()) { + SmallVector extents; + if (failed(getShapeVec(shapeValue, extents)) || !extents.empty()) { + if (nonScalarSeen) + return false; + nonScalarSeen = true; + } + } + return true; + }()) + return BoolAttr::get(getContext(), true); + // Because a failing witness result here represents an eventual assertion // failure, we do not replace it with a constant witness. return nullptr; diff --git a/mlir/lib/ExecutionEngine/CMakeLists.txt b/mlir/lib/ExecutionEngine/CMakeLists.txt index 3199c88313087..35facca9689c8 100644 --- a/mlir/lib/ExecutionEngine/CMakeLists.txt +++ b/mlir/lib/ExecutionEngine/CMakeLists.txt @@ -260,6 +260,68 @@ if(LLVM_ENABLE_PIC) target_link_options(mlir_async_runtime PRIVATE "-Wl,-exclude-libs,ALL") endif() + add_mlir_library(StaticMLIRFloat16Utils + STATIC + Float16bits.cpp + + EXCLUDE_FROM_LIBMLIR + ) + set_property(TARGET StaticMLIRFloat16Utils PROPERTY CXX_STANDARD 17) + + add_mlir_library(StaticMLIRSparseTensorRuntime + STATIC + SparseTensor/File.cpp + SparseTensor/MapRef.cpp + SparseTensor/Storage.cpp + + EXCLUDE_FROM_LIBMLIR + + LINK_LIBS PUBLIC + MLIRSparseTensorEnums + StaticMLIRFloat16Utils + ) + set_property(TARGET StaticMLIRSparseTensorRuntime PROPERTY CXX_STANDARD 17) + + add_mlir_library(StaticMLIRCRunnerUtils + STATIC + CRunnerUtils.cpp + SparseTensorRuntime.cpp + + EXCLUDE_FROM_LIBMLIR + + LINK_LIBS PUBLIC + StaticMLIRFloat16Utils + MLIRSparseTensorEnums + StaticMLIRSparseTensorRuntime + ) + set_property(TARGET StaticMLIRCRunnerUtils PROPERTY CXX_STANDARD 17) + target_compile_definitions(StaticMLIRCRunnerUtils PRIVATE StaticMLIRCRunnerUtils_EXPORTS) + + add_mlir_library(StaticMLIRRunnerUtils + STATIC + RunnerUtils.cpp + + EXCLUDE_FROM_LIBMLIR + + LINK_LIBS PUBLIC + StaticMLIRFloat16Utils + ) + target_compile_definitions(StaticMLIRRunnerUtils PRIVATE StaticMLIRRunnerUtils_EXPORTS) + + add_mlir_library(StaticMLIRAsyncRuntime + STATIC + AsyncRuntime.cpp + + EXCLUDE_FROM_LIBMLIR + + LINK_LIBS PUBLIC + ${LLVM_PTHREAD_LIB} + + DISABLE_PCH_REUSE + ) + set_property(TARGET StaticMLIRAsyncRuntime PROPERTY CXX_VISIBILITY_PRESET hidden) + target_compile_definitions(StaticMLIRAsyncRuntime PRIVATE StaticMLIRAsyncRuntime_EXPORTS) + add_mlir_library(mlir_arm_sme_abi_stubs SHARED ArmSMEStubs.cpp diff --git a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp index 49a88e0000511..ba6a600b88cdd 100644 --- a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp @@ -10,19 +10,21 @@ // JIT engine. // //===----------------------------------------------------------------------===// -#include "mlir/ExecutionEngine/ExecutionEngine.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/ExecutionEngine/ExecutionEngine.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/Support/FileUtilities.h" #include "mlir/Target/LLVMIR/Export.h" #include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/ObjectCache.h" #include "llvm/ExecutionEngine/Orc/CompileUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/IR/IRBuilder.h" #include "llvm/MC/TargetRegistry.h" @@ -316,27 +318,57 @@ ExecutionEngine::create(Operation *m, const ExecutionEngineOptions &options, &IgnoredMemMgr) { // Needed to respect AArch64 ABI requirements on the distance between // TEXT and GOT sections. - bool reserveAlloc = llvmModule->getTargetTriple().isAArch64(); - auto objectLayer = std::make_unique( - session, [sectionMemoryMapper = options.sectionMemoryMapper, - reserveAlloc](const MemoryBuffer &) { - return std::make_unique(sectionMemoryMapper, - reserveAlloc); - }); - - // Register JIT event listeners if they are enabled. - if (engine->gdbListener) - objectLayer->registerJITEventListener(*engine->gdbListener); - if (engine->perfListener) - objectLayer->registerJITEventListener(*engine->perfListener); + + // Check if we should use ObjectLinkingLayer (JITLink) + // JITLink supports modern architectures like RISC-V, AArch64 + // RuntimeDyld is older and provides better compatibility with legacy + // platforms + + // Decide which layer to use + bool useJITLink = llvmModule->getTargetTriple().isAArch64() || + llvmModule->getTargetTriple().isRISCV(); + + std::unique_ptr objectLayer; + + if (useJITLink) { + // JITLink path + objectLayer = std::make_unique(session, IgnoredMemMgr); + + LLVM_DEBUG(llvm::dbgs() << "Using ObjectLinkingLayer (JITLink)\n"); + + } else { + // RuntimeDyld path + auto rtDyldLayer = std::make_unique( + session, + [sectionMemoryMapper = + options.sectionMemoryMapper](const llvm::MemoryBuffer &) + -> std::unique_ptr { + return std::make_unique(sectionMemoryMapper); + }); + + // Only RTDyld supports listener + if (engine->gdbListener) + rtDyldLayer->registerJITEventListener(*engine->gdbListener); + + if (engine->perfListener) + rtDyldLayer->registerJITEventListener(*engine->perfListener); + + LLVM_DEBUG(llvm::dbgs() << "Using RTDyldObjectLinkingLayer\n"); + + // Upcast + objectLayer = std::move(rtDyldLayer); + } // COFF format binaries (Windows) need special handling to deal with // exported symbol visibility. // cf llvm/lib/ExecutionEngine/Orc/LLJIT.cpp LLJIT::createObjectLinkingLayer const llvm::Triple &targetTriple = llvmModule->getTargetTriple(); - if (targetTriple.isOSBinFormatCOFF()) { - objectLayer->setOverrideObjectFlagsWithResponsibilityFlags(true); - objectLayer->setAutoClaimResponsibilityForObjectSymbols(true); + if (!useJITLink && targetTriple.isOSBinFormatCOFF()) { + if (auto *rtDyldLayer = dyn_cast( + objectLayer.get())) { + rtDyldLayer->setOverrideObjectFlagsWithResponsibilityFlags(true); + rtDyldLayer->setAutoClaimResponsibilityForObjectSymbols(true); + } } // Resolve symbols from shared libraries. diff --git a/mlir/lib/ExecutionEngine/JitRunner.cpp b/mlir/lib/ExecutionEngine/JitRunner.cpp index db0516533afcb..4b52669df9a6e 100644 --- a/mlir/lib/ExecutionEngine/JitRunner.cpp +++ b/mlir/lib/ExecutionEngine/JitRunner.cpp @@ -31,6 +31,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/LegacyPassNameParser.h" +#include "llvm/Support/CodeGen.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/FileUtilities.h" @@ -361,6 +362,11 @@ int mlir::JitRunnerMain(int argc, char **argv, const DialectRegistry ®istry, tmBuilderOrError->getTargetTriple().setArchName(options.mArch); } + if (tmBuilderOrError->getTargetTriple().isRISCV()){ + tmBuilderOrError->setRelocationModel(llvm::Reloc::PIC_); + tmBuilderOrError->setCodeModel(llvm::CodeModel::Medium); + } + // Build TargetMachine auto tmOrError = tmBuilderOrError->createTargetMachine(); diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt index e0c32cd4bd9a0..e249093358879 100644 --- a/mlir/test/CMakeLists.txt +++ b/mlir/test/CMakeLists.txt @@ -158,6 +158,11 @@ if(LLVM_ENABLE_PIC AND TARGET ${LLVM_NATIVE_ARCH}) mlir_c_runner_utils mlir_runner_utils mlir_float16_utils + StaticMLIRAsyncRuntime + StaticMLIRCRunnerUtils + StaticMLIRFloat16Utils + StaticMLIRRunnerUtils + StaticMLIRSparseTensorRuntime ) endif() diff --git a/mlir/test/mlir-runner/test-expand-math-approx.mlir b/mlir/test/mlir-runner/test-expand-math-approx.mlir index 06b3171a2349e..5ba0f7106e101 100644 --- a/mlir/test/mlir-runner/test-expand-math-approx.mlir +++ b/mlir/test/mlir-runner/test-expand-math-approx.mlir @@ -233,12 +233,12 @@ func.func @powf() { %g_p = arith.constant 23598.0 : f64 call @func_powff64(%g, %g_p) : (f64, f64) -> () - // CHECK-NEXT: -nan + // CHECK-NEXT: {{-?}}nan %h = arith.constant 1.0 : f64 %h_p = arith.constant 0xfff0000001000000 : f64 call @func_powff64(%h, %h_p) : (f64, f64) -> () - // CHECK-NEXT: -nan + // CHECK-NEXT: {{-?}}nan %i = arith.constant 1.0 : f32 %i_p = arith.constant 0xffffffff : f32 call @func_powff32(%i, %i_p) : (f32, f32) -> () diff --git a/mlir/test/python/integration/dialects/linalg/opsrun.py b/mlir/test/python/integration/dialects/linalg/opsrun.py index 8eff573f98ad3..665879acb5cc9 100644 --- a/mlir/test/python/integration/dialects/linalg/opsrun.py +++ b/mlir/test/python/integration/dialects/linalg/opsrun.py @@ -19,6 +19,11 @@ def log(*args): sys.stderr.flush() +def run(f): + f() + return f + + fill_boiler = """ func.func @main() -> i32 attributes {llvm.emit_c_interface} { %O0 = memref.alloc() : memref @@ -177,7 +182,7 @@ def fill_2d_on_buffers(value, out): # CHECK: RESULT: 6 -test_fill_builtin() +run(test_fill_builtin) def test_fill_generic(): @@ -211,7 +216,7 @@ def fill_2d_on_buffers(value, out): # CHECK: RESULT: 6 -test_fill_generic() +run(test_fill_generic) def test_fill_rng_builtin(): @@ -238,7 +243,7 @@ def fill_rng_on_buffers(min, max, seed, out): # CHECK: RESULT: -480 -test_fill_rng_builtin() +run(test_fill_rng_builtin) def test_fill_rng_generic(): @@ -265,7 +270,7 @@ def fill_rng_on_buffers(min, max, seed, out): # CHECK: RESULT: -480 -test_fill_rng_generic() +run(test_fill_rng_generic) def test_max_pooling_builtin(): @@ -299,7 +304,7 @@ def pooling_on_buffers(input, shape, output): # CHECK: RESULT: 42 -test_max_pooling_builtin() +run(test_max_pooling_builtin) def test_max_pooling_generic(): @@ -338,7 +343,7 @@ def pooling_on_buffers(input, shape, output): # CHECK: RESULT: 42 -test_max_pooling_generic() +run(test_max_pooling_generic) def test_min_pooling_builtin(): @@ -370,7 +375,7 @@ def pooling_on_buffers(input, shape, output): # CHECK: RESULT: -13 -test_min_pooling_builtin() +run(test_min_pooling_builtin) def test_min_pooling_generic(): @@ -404,4 +409,4 @@ def pooling_on_buffers(input, shape, output): # CHECK: RESULT: -13 -test_min_pooling_generic() +run(test_min_pooling_generic) diff --git a/openmp/runtime/test/affinity/libomp_test_topology.h b/openmp/runtime/test/affinity/libomp_test_topology.h index 410103d809249..244143f3ebb42 100644 --- a/openmp/runtime/test/affinity/libomp_test_topology.h +++ b/openmp/runtime/test/affinity/libomp_test_topology.h @@ -145,13 +145,144 @@ static int topology_using_full_mask() { return has_all; } +// Read a single integer from a sysfs file. Returns 0 on success, -1 on error. +static int topology_read_int_from_file(const char *path, int *value) { + FILE *f = fopen(path, "r"); + if (!f) + return -1; + int n = fscanf(f, "%d", value); + fclose(f); + return (n == 1) ? 0 : -1; +} + +// On non-x86 Linux, the OpenMP runtime reads core_id and physical_package_id +// from sysfs to determine topology. Use the same method in the test so that +// the test's view of the topology matches the runtime's view. +// On x86/x86_64 (or when sysfs id files are unavailable), fall back to the +// siblings-list approach. +static int topology_use_sysfs_ids() { +#if defined(__i386__) || defined(__x86_64__) + return 0; +#else + // Check whether the sysfs id files exist for cpu0 + int dummy; + if (topology_read_int_from_file( + "/sys/devices/system/cpu/cpu0/topology/core_id", &dummy) == 0 && + topology_read_int_from_file( + "/sys/devices/system/cpu/cpu0/topology/physical_package_id", + &dummy) == 0) + return 1; + return 0; +#endif +} + +// Build place list using core_id / physical_package_id from sysfs, +// matching what the OpenMP runtime does on non-x86 Linux. +static place_list_t *topology_alloc_type_places_sysfs(topology_obj_type_t type, + int num_cpus) { + char buf[1024]; + int i, cpu, num_unique; + int *place_nums; + place_list_t *places = (place_list_t *)malloc(sizeof(place_list_t)); + affinity_mask_t **masks = + (affinity_mask_t **)malloc(sizeof(affinity_mask_t *) * num_cpus); + + // Read per-cpu core_id and physical_package_id + int *core_ids = (int *)malloc(sizeof(int) * num_cpus); + int *pkg_ids = (int *)malloc(sizeof(int) * num_cpus); + for (cpu = 0; cpu < num_cpus; ++cpu) { + snprintf(buf, sizeof(buf), + "/sys/devices/system/cpu/cpu%d/topology/core_id", cpu); + if (topology_read_int_from_file(buf, &core_ids[cpu]) != 0) + core_ids[cpu] = cpu; + snprintf(buf, sizeof(buf), + "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu); + if (topology_read_int_from_file(buf, &pkg_ids[cpu]) != 0) + pkg_ids[cpu] = 0; + } + + num_unique = 0; + if (type == TOPOLOGY_OBJ_THREAD) { + for (cpu = 0; cpu < num_cpus; ++cpu) { + affinity_mask_t *mask = affinity_mask_alloc(); + affinity_mask_set(mask, cpu); + masks[num_unique++] = mask; + } + } else if (type == TOPOLOGY_OBJ_CORE) { + // Group CPUs with same (physical_package_id, core_id) into one place + for (cpu = 0; cpu < num_cpus; ++cpu) { + int found = 0; + for (i = 0; i < num_unique; ++i) { + // Find a cpu already in this group to compare ids + int rep; + for (rep = 0; rep < num_cpus; ++rep) { + if (affinity_mask_isset(masks[i], rep)) + break; + } + if (rep < num_cpus && pkg_ids[rep] == pkg_ids[cpu] && + core_ids[rep] == core_ids[cpu]) { + affinity_mask_set(masks[i], cpu); + found = 1; + break; + } + } + if (!found) { + affinity_mask_t *mask = affinity_mask_alloc(); + affinity_mask_set(mask, cpu); + masks[num_unique++] = mask; + } + } + } else if (type == TOPOLOGY_OBJ_SOCKET) { + // Group CPUs with same physical_package_id into one place + for (cpu = 0; cpu < num_cpus; ++cpu) { + int found = 0; + for (i = 0; i < num_unique; ++i) { + int rep; + for (rep = 0; rep < num_cpus; ++rep) { + if (affinity_mask_isset(masks[i], rep)) + break; + } + if (rep < num_cpus && pkg_ids[rep] == pkg_ids[cpu]) { + affinity_mask_set(masks[i], cpu); + found = 1; + break; + } + } + if (!found) { + affinity_mask_t *mask = affinity_mask_alloc(); + affinity_mask_set(mask, cpu); + masks[num_unique++] = mask; + } + } + } else { + fprintf(stderr, "Unknown topology type (%d)\n", (int)type); + exit(EXIT_FAILURE); + } + + free(core_ids); + free(pkg_ids); + place_nums = (int *)malloc(sizeof(int) * num_unique); + for (i = 0; i < num_unique; ++i) + place_nums[i] = i; + places->num_places = num_unique; + places->masks = masks; + places->place_nums = place_nums; + places->current_place = -1; + return places; +} + // Return array of masks representing OMP_PLACES keyword (e.g., sockets, cores, // threads) static place_list_t *topology_alloc_type_places(topology_obj_type_t type) { char buf[1024]; - int i, cpu, num_places, num_unique; + int i, cpu, num_unique; int *place_nums; int num_cpus = topology_get_num_cpus(); + + // On non-x86 Linux, use core_id/physical_package_id to match the runtime + if (topology_use_sysfs_ids()) + return topology_alloc_type_places_sysfs(type, num_cpus); + place_list_t *places = (place_list_t *)malloc(sizeof(place_list_t)); affinity_mask_t **masks = (affinity_mask_t **)malloc(sizeof(affinity_mask_t *) * num_cpus); diff --git a/openmp/runtime/test/ompt/callback.h b/openmp/runtime/test/ompt/callback.h index c0305c0daf606..36391444c6bd3 100644 --- a/openmp/runtime/test/ompt/callback.h +++ b/openmp/runtime/test/ompt/callback.h @@ -290,14 +290,14 @@ static void print_ids(int level) { #elif KMP_ARCH_RISCV64 #if __riscv_compressed // On RV64GC the C.NOP instruction is 2 byte long. In addition, the compiler -// inserts a J instruction (targeting the successor basic block), which -// accounts for another 4 bytes. Finally, an additional J instruction may -// appear (adding 4 more bytes) when the C.NOP is referenced elsewhere (ie. -// another branch). +// may insert a J or C.J instruction (targeting the successor basic block), +// which accounts for 4 or 2 bytes respectively. Finally, an additional jump +// instruction may appear (adding 2 or 4 more bytes) when the C.NOP is +// referenced elsewhere (ie. another branch). #define print_possible_return_addresses(addr) \ - printf("%" PRIu64 ": current_address=%p or %p\n", \ - ompt_get_thread_data()->value, ((char *)addr) - 6, \ - ((char *)addr) - 10) + printf("%" PRIu64 ": current_address=%p or %p or %p\n", \ + ompt_get_thread_data()->value, ((char *)addr) - 4, \ + ((char *)addr) - 6, ((char *)addr) - 10) #else // On RV64G the NOP instruction is 4 byte long. In addition, the compiler // inserts a J instruction (targeting the successor basic block), which