//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation  -------===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// This file defines the interfaces that RISC-V uses to lower LLVM code into a

// selection DAG.

//

//===----------------------------------------------------------------------===//

#include "RISCVISelLowering.h"

#include "MCTargetDesc/RISCVMatInt.h"

#include "RISCV.h"

#include "RISCVMachineFunctionInfo.h"

#include "RISCVRegisterInfo.h"

#include "RISCVSubtarget.h"

#include "RISCVTargetMachine.h"

#include "llvm/ADT/SmallSet.h"

#include "llvm/ADT/Statistic.h"

#include "llvm/Analysis/MemoryLocation.h"

#include "llvm/Analysis/VectorUtils.h"

#include "llvm/CodeGen/MachineFrameInfo.h"

#include "llvm/CodeGen/MachineFunction.h"

#include "llvm/CodeGen/MachineInstrBuilder.h"

#include "llvm/CodeGen/MachineJumpTableInfo.h"

#include "llvm/CodeGen/MachineRegisterInfo.h"

#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"

#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"

#include "llvm/CodeGen/ValueTypes.h"

#include "llvm/IR/DiagnosticInfo.h"

#include "llvm/IR/DiagnosticPrinter.h"

#include "llvm/IR/IRBuilder.h"

#include "llvm/IR/Instructions.h"

#include "llvm/IR/IntrinsicsRISCV.h"

#include "llvm/IR/PatternMatch.h"

#include "llvm/Support/CommandLine.h"

#include "llvm/Support/Debug.h"

#include "llvm/Support/ErrorHandling.h"

#include "llvm/Support/InstructionCost.h"

#include "llvm/Support/KnownBits.h"

#include "llvm/Support/MathExtras.h"

#include "llvm/Support/raw_ostream.h"

#include <optional>

using namespace llvm;

#define DEBUG_TYPE "riscv-lower"

STATISTIC(NumTailCalls, "Number of tail calls");

static cl::opt<unsigned> ExtensionMaxWebSize(

    DEBUG_TYPE "-ext-max-web-size", cl::Hidden,

    cl::desc("Give the maximum size (in number of nodes) of the web of "

             "instructions that we will consider for VW expansion"),

    cl::init(18));

static cl::opt<bool>

    AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,

                     cl::desc("Allow the formation of VW_W operations (e.g., "

                              "VWADD_W) with splat constants"),

                     cl::init(false));

static cl::opt<unsigned> NumRepeatedDivisors(

    DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,

    cl::desc("Set the minimum number of repetitions of a divisor to allow "

             "transformation to multiplications by the reciprocal"),

    cl::init(2));

static cl::opt<int>

    FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,

              cl::desc("Give the maximum number of instructions that we will "

                       "use for creating a floating-point immediate value"),

              cl::init(2));

static cl::opt<bool>

    RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden,

                 cl::desc("Make i32 a legal type for SelectionDAG on RV64."));

RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,

                                         const RISCVSubtarget &STI)

    : TargetLowering(TM), Subtarget(STI) {

  RISCVABI::ABI ABI = Subtarget.getTargetABI();

  assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");

  if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&

      !Subtarget.hasStdExtF()) {

    errs() << "Hard-float 'f' ABI can't be used for a target that "

                "doesn't support the F instruction set extension (ignoring "

                          "target-abi)\n";

    ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;

  } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&

             !Subtarget.hasStdExtD()) {

    errs() << "Hard-float 'd' ABI can't be used for a target that "

              "doesn't support the D instruction set extension (ignoring "

              "target-abi)\n";

    ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;

  }

  switch (ABI) {

  default:

    report_fatal_error("Don't know how to lower this ABI");

  case RISCVABI::ABI_ILP32:

  case RISCVABI::ABI_ILP32E:

  case RISCVABI::ABI_LP64E:

  case RISCVABI::ABI_ILP32F:

  case RISCVABI::ABI_ILP32D:

  case RISCVABI::ABI_LP64:

  case RISCVABI::ABI_LP64F:

  case RISCVABI::ABI_LP64D:

    break;

  }

  MVT XLenVT = Subtarget.getXLenVT();

  // Set up the register classes.

  addRegisterClass(XLenVT, &RISCV::GPRRegClass);

  if (Subtarget.is64Bit() && RV64LegalI32)

    addRegisterClass(MVT::i32, &RISCV::GPRRegClass);

  if (Subtarget.hasStdExtZfhmin())

    addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);

  if (Subtarget.hasStdExtZfbfmin())

    addRegisterClass(MVT::bf16, &RISCV::FPR16RegClass);

  if (Subtarget.hasStdExtF())

    addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);

  if (Subtarget.hasStdExtD())

    addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);

  if (Subtarget.hasStdExtZhinxmin())

    addRegisterClass(MVT::f16, &RISCV::GPRF16RegClass);

  if (Subtarget.hasStdExtZfinx())

    addRegisterClass(MVT::f32, &RISCV::GPRF32RegClass);

  if (Subtarget.hasStdExtZdinx()) {

    if (Subtarget.is64Bit())

      addRegisterClass(MVT::f64, &RISCV::GPRRegClass);

    else

      addRegisterClass(MVT::f64, &RISCV::GPRPairRegClass);

  }

  static const MVT::SimpleValueType BoolVecVTs[] = {

      MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,

      MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};

  static const MVT::SimpleValueType IntVecVTs[] = {

      MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,

      MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,

      MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,

      MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,

      MVT::nxv4i64, MVT::nxv8i64};

  static const MVT::SimpleValueType F16VecVTs[] = {

      MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,

      MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};

  static const MVT::SimpleValueType BF16VecVTs[] = {

      MVT::nxv1bf16, MVT::nxv2bf16,  MVT::nxv4bf16,

      MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};

  static const MVT::SimpleValueType F32VecVTs[] = {

      MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};

  static const MVT::SimpleValueType F64VecVTs[] = {

      MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};

  if (Subtarget.hasVInstructions()) {

    auto addRegClassForRVV = [this](MVT VT) {

      // Disable the smallest fractional LMUL types if ELEN is less than

      // RVVBitsPerBlock.

      unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();

      if (VT.getVectorMinNumElements() < MinElts)

        return;

      unsigned Size = VT.getSizeInBits().getKnownMinValue();

      const TargetRegisterClass *RC;

      if (Size <= RISCV::RVVBitsPerBlock)

        RC = &RISCV::VRRegClass;

      else if (Size == 2 * RISCV::RVVBitsPerBlock)

        RC = &RISCV::VRM2RegClass;

      else if (Size == 4 * RISCV::RVVBitsPerBlock)

        RC = &RISCV::VRM4RegClass;

      else if (Size == 8 * RISCV::RVVBitsPerBlock)

        RC = &RISCV::VRM8RegClass;

      else

        llvm_unreachable("Unexpected size");

      addRegisterClass(VT, RC);

    };

    for (MVT VT : BoolVecVTs)

      addRegClassForRVV(VT);

    for (MVT VT : IntVecVTs) {

      if (VT.getVectorElementType() == MVT::i64 &&

          !Subtarget.hasVInstructionsI64())

        continue;

      addRegClassForRVV(VT);

    }

    if (Subtarget.hasVInstructionsF16Minimal())

      for (MVT VT : F16VecVTs)

        addRegClassForRVV(VT);

    if (Subtarget.hasVInstructionsBF16())

      for (MVT VT : BF16VecVTs)

        addRegClassForRVV(VT);

    if (Subtarget.hasVInstructionsF32())

      for (MVT VT : F32VecVTs)

        addRegClassForRVV(VT);

    if (Subtarget.hasVInstructionsF64())

      for (MVT VT : F64VecVTs)

        addRegClassForRVV(VT);

    if (Subtarget.useRVVForFixedLengthVectors()) {

      auto addRegClassForFixedVectors = [this](MVT VT) {

        MVT ContainerVT = getContainerForFixedLengthVector(VT);

        unsigned RCID = getRegClassIDForVecVT(ContainerVT);

        const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();

        addRegisterClass(VT, TRI.getRegClass(RCID));

      };

      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())

        if (useRVVForFixedLengthVectorVT(VT))

          addRegClassForFixedVectors(VT);

      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())

        if (useRVVForFixedLengthVectorVT(VT))

          addRegClassForFixedVectors(VT);

    }

  }

  // Compute derived properties from the register classes.

  computeRegisterProperties(STI.getRegisterInfo());

  setStackPointerRegisterToSaveRestore(RISCV::X2);

  setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT,

                   MVT::i1, Promote);

  // DAGCombiner can call isLoadExtLegal for types that aren't legal.

  setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,

                   MVT::i1, Promote);

  // TODO: add all necessary setOperationAction calls.

  setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);

  setOperationAction(ISD::BR_JT, MVT::Other, Expand);

  setOperationAction(ISD::BR_CC, XLenVT, Expand);

  if (RV64LegalI32 && Subtarget.is64Bit())

    setOperationAction(ISD::BR_CC, MVT::i32, Expand);

  setOperationAction(ISD::BRCOND, MVT::Other, Custom);

  setOperationAction(ISD::SELECT_CC, XLenVT, Expand);

  if (RV64LegalI32 && Subtarget.is64Bit())

    setOperationAction(ISD::SELECT_CC, MVT::i32, Expand);

  setCondCodeAction(ISD::SETLE, XLenVT, Expand);

  setCondCodeAction(ISD::SETGT, XLenVT, Custom);

  setCondCodeAction(ISD::SETGE, XLenVT, Expand);

  setCondCodeAction(ISD::SETULE, XLenVT, Expand);

  setCondCodeAction(ISD::SETUGT, XLenVT, Custom);

  setCondCodeAction(ISD::SETUGE, XLenVT, Expand);

  if (RV64LegalI32 && Subtarget.is64Bit())

    setOperationAction(ISD::SETCC, MVT::i32, Promote);

  setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);

  setOperationAction(ISD::VASTART, MVT::Other, Custom);

  setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);

  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);

  setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);

  if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())

    setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);

  if (Subtarget.is64Bit()) {

    setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);

    if (!RV64LegalI32) {

      setOperationAction(ISD::LOAD, MVT::i32, Custom);

      setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},

                         MVT::i32, Custom);

      setOperationAction(ISD::SADDO, MVT::i32, Custom);

      setOperationAction({ISD::UADDO, ISD::USUBO, ISD::UADDSAT, ISD::USUBSAT},

                         MVT::i32, Custom);

    }

  } else {

    setLibcallName(

        {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},

        nullptr);

    setLibcallName(RTLIB::MULO_I64, nullptr);

  }

  if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) {

    setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand);

    if (RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction(ISD::MUL, MVT::i32, Promote);

  } else if (Subtarget.is64Bit()) {

    setOperationAction(ISD::MUL, MVT::i128, Custom);

    if (!RV64LegalI32)

      setOperationAction(ISD::MUL, MVT::i32, Custom);

  } else {

    setOperationAction(ISD::MUL, MVT::i64, Custom);

  }

  if (!Subtarget.hasStdExtM()) {

    setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM},

                       XLenVT, Expand);

    if (RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, MVT::i32,

                         Promote);

  } else if (Subtarget.is64Bit()) {

    if (!RV64LegalI32)

      setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM},

                         {MVT::i8, MVT::i16, MVT::i32}, Custom);

  }

  if (RV64LegalI32 && Subtarget.is64Bit()) {

    setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::i32, Expand);

    setOperationAction(

        {ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32,

        Expand);

  }

  setOperationAction(

      {ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT,

      Expand);

  setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT,

                     Custom);

  if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {

    if (!RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);

  } else if (Subtarget.hasVendorXTHeadBb()) {

    if (Subtarget.is64Bit())

      setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);

    setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom);

  } else if (Subtarget.hasVendorXCVbitmanip()) {

    setOperationAction(ISD::ROTL, XLenVT, Expand);

  } else {

    setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand);

    if (RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Expand);

  }

  // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll

  // pattern match it directly in isel.

  setOperationAction(ISD::BSWAP, XLenVT,

                     (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||

                      Subtarget.hasVendorXTHeadBb())

                         ? Legal

                         : Expand);

  if (RV64LegalI32 && Subtarget.is64Bit())

    setOperationAction(ISD::BSWAP, MVT::i32,

                       (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||

                        Subtarget.hasVendorXTHeadBb())

                           ? Promote

                           : Expand);

  if (Subtarget.hasVendorXCVbitmanip()) {

    setOperationAction(ISD::BITREVERSE, XLenVT, Legal);

  } else {

    // Zbkb can use rev8+brev8 to implement bitreverse.

    setOperationAction(ISD::BITREVERSE, XLenVT,

                       Subtarget.hasStdExtZbkb() ? Custom : Expand);

  }

  if (Subtarget.hasStdExtZbb()) {

    setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT,

                       Legal);

    if (RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, MVT::i32,

                         Promote);

    if (Subtarget.is64Bit()) {

      if (RV64LegalI32)

        setOperationAction(ISD::CTTZ, MVT::i32, Legal);

      else

        setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom);

    }

  } else if (!Subtarget.hasVendorXCVbitmanip()) {

    setOperationAction({ISD::CTTZ, ISD::CTPOP}, XLenVT, Expand);

    if (RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction({ISD::CTTZ, ISD::CTPOP}, MVT::i32, Expand);

  }

  if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||

      Subtarget.hasVendorXCVbitmanip()) {

    // We need the custom lowering to make sure that the resulting sequence

    // for the 32bit case is efficient on 64bit targets.

    if (Subtarget.is64Bit()) {

      if (RV64LegalI32) {

        setOperationAction(ISD::CTLZ, MVT::i32,

                           Subtarget.hasStdExtZbb() ? Legal : Promote);

        if (!Subtarget.hasStdExtZbb())

          setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Promote);

      } else

        setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom);

    }

  } else {

    setOperationAction(ISD::CTLZ, XLenVT, Expand);

    if (RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction(ISD::CTLZ, MVT::i32, Expand);

  }

  if (!RV64LegalI32 && Subtarget.is64Bit() &&

      !Subtarget.hasShortForwardBranchOpt())

    setOperationAction(ISD::ABS, MVT::i32, Custom);

  // We can use PseudoCCSUB to implement ABS.

  if (Subtarget.hasShortForwardBranchOpt())

    setOperationAction(ISD::ABS, XLenVT, Legal);

  if (!Subtarget.hasVendorXTHeadCondMov())

    setOperationAction(ISD::SELECT, XLenVT, Custom);

  if (RV64LegalI32 && Subtarget.is64Bit())

    setOperationAction(ISD::SELECT, MVT::i32, Promote);

  static const unsigned FPLegalNodeTypes[] = {

      ISD::FMINNUM,        ISD::FMAXNUM,       ISD::LRINT,

      ISD::LLRINT,         ISD::LROUND,        ISD::LLROUND,

      ISD::STRICT_LRINT,   ISD::STRICT_LLRINT, ISD::STRICT_LROUND,

      ISD::STRICT_LLROUND, ISD::STRICT_FMA,    ISD::STRICT_FADD,

      ISD::STRICT_FSUB,    ISD::STRICT_FMUL,   ISD::STRICT_FDIV,

      ISD::STRICT_FSQRT,   ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS};

  static const ISD::CondCode FPCCToExpand[] = {

      ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,

      ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,

      ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};

  static const unsigned FPOpToExpand[] = {

      ISD::FSIN, ISD::FCOS,       ISD::FSINCOS,   ISD::FPOW,

      ISD::FREM};

  static const unsigned FPRndMode[] = {

      ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,

      ISD::FROUNDEVEN};

  if (Subtarget.hasStdExtZfhminOrZhinxmin())

    setOperationAction(ISD::BITCAST, MVT::i16, Custom);

  static const unsigned ZfhminZfbfminPromoteOps[] = {

      ISD::FMINNUM,      ISD::FMAXNUM,       ISD::FADD,

      ISD::FSUB,         ISD::FMUL,          ISD::FMA,

      ISD::FDIV,         ISD::FSQRT,         ISD::FABS,

      ISD::FNEG,         ISD::STRICT_FMA,    ISD::STRICT_FADD,

      ISD::STRICT_FSUB,  ISD::STRICT_FMUL,   ISD::STRICT_FDIV,

      ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,

      ISD::SETCC,        ISD::FCEIL,         ISD::FFLOOR,

      ISD::FTRUNC,       ISD::FRINT,         ISD::FROUND,

      ISD::FROUNDEVEN,   ISD::SELECT};

  if (Subtarget.hasStdExtZfbfmin()) {

    setOperationAction(ISD::BITCAST, MVT::i16, Custom);

    setOperationAction(ISD::BITCAST, MVT::bf16, Custom);

    setOperationAction(ISD::FP_ROUND, MVT::bf16, Custom);

    setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);

    setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);

    setOperationAction(ISD::ConstantFP, MVT::bf16, Expand);

    setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand);

    setOperationAction(ISD::BR_CC, MVT::bf16, Expand);

    setOperationAction(ZfhminZfbfminPromoteOps, MVT::bf16, Promote);

    setOperationAction(ISD::FREM, MVT::bf16, Promote);

    // FIXME: Need to promote bf16 FCOPYSIGN to f32, but the

    // DAGCombiner::visitFP_ROUND probably needs improvements first.

    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Expand);

  }

  if (Subtarget.hasStdExtZfhminOrZhinxmin()) {

    if (Subtarget.hasStdExtZfhOrZhinx()) {

      setOperationAction(FPLegalNodeTypes, MVT::f16, Legal);

      setOperationAction(FPRndMode, MVT::f16,

                         Subtarget.hasStdExtZfa() ? Legal : Custom);

      setOperationAction(ISD::SELECT, MVT::f16, Custom);

      setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);

    } else {

      setOperationAction(ZfhminZfbfminPromoteOps, MVT::f16, Promote);

      setOperationAction({ISD::STRICT_LRINT, ISD::STRICT_LLRINT,

                          ISD::STRICT_LROUND, ISD::STRICT_LLROUND},

                         MVT::f16, Legal);

      // FIXME: Need to promote f16 FCOPYSIGN to f32, but the

      // DAGCombiner::visitFP_ROUND probably needs improvements first.

      setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);

    }

    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);

    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);

    setCondCodeAction(FPCCToExpand, MVT::f16, Expand);

    setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);

    setOperationAction(ISD::BR_CC, MVT::f16, Expand);

    setOperationAction(ISD::FNEARBYINT, MVT::f16,

                       Subtarget.hasStdExtZfa() ? Legal : Promote);

    setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI,

                        ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,

                        ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,

                        ISD::FLOG10},

                       MVT::f16, Promote);

    // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have

    // complete support for all operations in LegalizeDAG.

    setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,

                        ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,

                        ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,

                        ISD::STRICT_FTRUNC},

                       MVT::f16, Promote);

    // We need to custom promote this.

    if (Subtarget.is64Bit())

      setOperationAction(ISD::FPOWI, MVT::i32, Custom);

    if (!Subtarget.hasStdExtZfa())

      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16, Custom);

  }

  if (Subtarget.hasStdExtFOrZfinx()) {

    setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);

    setOperationAction(FPRndMode, MVT::f32,

                       Subtarget.hasStdExtZfa() ? Legal : Custom);

    setCondCodeAction(FPCCToExpand, MVT::f32, Expand);

    setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);

    setOperationAction(ISD::SELECT, MVT::f32, Custom);

    setOperationAction(ISD::BR_CC, MVT::f32, Expand);

    setOperationAction(FPOpToExpand, MVT::f32, Expand);

    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);

    setTruncStoreAction(MVT::f32, MVT::f16, Expand);

    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);

    setTruncStoreAction(MVT::f32, MVT::bf16, Expand);

    setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);

    setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);

    setOperationAction(ISD::FP_TO_BF16, MVT::f32,

                       Subtarget.isSoftFPABI() ? LibCall : Custom);

    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);

    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);

    if (Subtarget.hasStdExtZfa())

      setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);

    else

      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Custom);

  }

  if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())

    setOperationAction(ISD::BITCAST, MVT::i32, Custom);

  if (Subtarget.hasStdExtDOrZdinx()) {

    setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);

    if (Subtarget.hasStdExtZfa()) {

      setOperationAction(FPRndMode, MVT::f64, Legal);

      setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);

      setOperationAction(ISD::BITCAST, MVT::i64, Custom);

      setOperationAction(ISD::BITCAST, MVT::f64, Custom);

    } else {

      if (Subtarget.is64Bit())

        setOperationAction(FPRndMode, MVT::f64, Custom);

      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Custom);

    }

    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);

    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);

    setCondCodeAction(FPCCToExpand, MVT::f64, Expand);

    setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);

    setOperationAction(ISD::SELECT, MVT::f64, Custom);

    setOperationAction(ISD::BR_CC, MVT::f64, Expand);

    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);

    setTruncStoreAction(MVT::f64, MVT::f32, Expand);

    setOperationAction(FPOpToExpand, MVT::f64, Expand);

    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);

    setTruncStoreAction(MVT::f64, MVT::f16, Expand);

    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);

    setTruncStoreAction(MVT::f64, MVT::bf16, Expand);

    setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);

    setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);

    setOperationAction(ISD::FP_TO_BF16, MVT::f64,

                       Subtarget.isSoftFPABI() ? LibCall : Custom);

    setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);

    setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);

  }

  if (Subtarget.is64Bit()) {

    setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,

                        ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},

                       MVT::i32, Custom);

    setOperationAction(ISD::LROUND, MVT::i32, Custom);

  }

  if (Subtarget.hasStdExtFOrZfinx()) {

    setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,

                       Custom);

    setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT,

                        ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP},

                       XLenVT, Legal);

    if (RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT,

                          ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP},

                         MVT::i32, Legal);

    setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);

    setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);

  }

  setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,

                      ISD::JumpTable},

                     XLenVT, Custom);

  setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);

  if (Subtarget.is64Bit())

    setOperationAction(ISD::Constant, MVT::i64, Custom);

  // TODO: On M-mode only targets, the cycle[h] CSR may not be present.

  // Unfortunately this can't be determined just from the ISA naming string.

  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,

                     Subtarget.is64Bit() ? Legal : Custom);

  setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);

  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);

  if (Subtarget.is64Bit())

    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);

  if (Subtarget.hasStdExtZicbop()) {

    setOperationAction(ISD::PREFETCH, MVT::Other, Legal);

  }

  if (Subtarget.hasStdExtA()) {

    setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());

    setMinCmpXchgSizeInBits(32);

  } else if (Subtarget.hasForcedAtomics()) {

    setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());

  } else {

    setMaxAtomicSizeInBitsSupported(0);

  }

  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);

  setBooleanContents(ZeroOrOneBooleanContent);

  if (Subtarget.hasVInstructions()) {

    setBooleanVectorContents(ZeroOrOneBooleanContent);

    setOperationAction(ISD::VSCALE, XLenVT, Custom);

    if (RV64LegalI32 && Subtarget.is64Bit())

      setOperationAction(ISD::VSCALE, MVT::i32, Custom);

    // RVV intrinsics may have illegal operands.

    // We also need to custom legalize vmv.x.s.

    setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,

                        ISD::INTRINSIC_VOID},

                       {MVT::i8, MVT::i16}, Custom);

    if (Subtarget.is64Bit())

      setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},

                         MVT::i32, Custom);

    else

      setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},

                         MVT::i64, Custom);

    setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},

                       MVT::Other, Custom);

    static const unsigned IntegerVPOps[] = {

        ISD::VP_ADD,         ISD::VP_SUB,         ISD::VP_MUL,

        ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,

        ISD::VP_UREM,        ISD::VP_AND,         ISD::VP_OR,

        ISD::VP_XOR,         ISD::VP_ASHR,        ISD::VP_LSHR,

        ISD::VP_SHL,         ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,

        ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,

        ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,

        ISD::VP_MERGE,       ISD::VP_SELECT,      ISD::VP_FP_TO_SINT,

        ISD::VP_FP_TO_UINT,  ISD::VP_SETCC,       ISD::VP_SIGN_EXTEND,

        ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE,    ISD::VP_SMIN,

        ISD::VP_SMAX,        ISD::VP_UMIN,        ISD::VP_UMAX,

        ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE};

    static const unsigned FloatingPointVPOps[] = {

        ISD::VP_FADD,        ISD::VP_FSUB,        ISD::VP_FMUL,

        ISD::VP_FDIV,        ISD::VP_FNEG,        ISD::VP_FABS,

        ISD::VP_FMA,         ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,

        ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,

        ISD::VP_SELECT,      ISD::VP_SINT_TO_FP,  ISD::VP_UINT_TO_FP,

        ISD::VP_SETCC,       ISD::VP_FP_ROUND,    ISD::VP_FP_EXTEND,

        ISD::VP_SQRT,        ISD::VP_FMINNUM,     ISD::VP_FMAXNUM,

        ISD::VP_FCEIL,       ISD::VP_FFLOOR,      ISD::VP_FROUND,

        ISD::VP_FROUNDEVEN,  ISD::VP_FCOPYSIGN,   ISD::VP_FROUNDTOZERO,

        ISD::VP_FRINT,       ISD::VP_FNEARBYINT,  ISD::VP_IS_FPCLASS,

        ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE};

    static const unsigned IntegerVecReduceOps[] = {

        ISD::VECREDUCE_ADD,  ISD::VECREDUCE_AND,  ISD::VECREDUCE_OR,

        ISD::VECREDUCE_XOR,  ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,

        ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};

    static const unsigned FloatingPointVecReduceOps[] = {

        ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,

        ISD::VECREDUCE_FMAX};

    if (!Subtarget.is64Bit()) {

      // We must custom-lower certain vXi64 operations on RV32 due to the vector

      // element type being illegal.

      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},

                         MVT::i64, Custom);

      setOperationAction(IntegerVecReduceOps, MVT::i64, Custom);

      setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,

                          ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,

                          ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,

                          ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},

                         MVT::i64, Custom);

    }

    for (MVT VT : BoolVecVTs) {

      if (!isTypeLegal(VT))

        continue;

      setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);

      // Mask VTs are custom-expanded into a series of standard nodes

      setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS,

                          ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,

                          ISD::SCALAR_TO_VECTOR},

                         VT, Custom);

      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,

                         Custom);

      setOperationAction(ISD::SELECT, VT, Custom);

      setOperationAction(

          {ISD::SELECT_CC, ISD::VSELECT, ISD::VP_MERGE, ISD::VP_SELECT}, VT,

          Expand);

      setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom);

      setOperationAction(

          {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,

          Custom);

      setOperationAction(

          {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,

          Custom);

      // RVV has native int->float & float->int conversions where the

      // element type sizes are within one power-of-two of each other. Any

      // wider distances between type sizes have to be lowered as sequences

      // which progressively narrow the gap in stages.

      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,

                          ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,

                          ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,

                          ISD::STRICT_FP_TO_UINT},

                         VT, Custom);

      setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,

                         Custom);

      // Expand all extending loads to types larger than this, and truncating

      // stores from types larger than this.

      for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {

        setTruncStoreAction(VT, OtherVT, Expand);

        setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,

                         OtherVT, Expand);

      }

      setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,

                          ISD::VP_TRUNCATE, ISD::VP_SETCC},

                         VT, Custom);

      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);

      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);

      setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);

      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);

      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);

      setOperationPromotedToType(

          ISD::VECTOR_SPLICE, VT,

          MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));

    }

    for (MVT VT : IntVecVTs) {

      if (!isTypeLegal(VT))

        continue;

      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);

      setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);

      // Vectors implement MULHS/MULHU.

      setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);

      // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.

      if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())

        setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);

      setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,

                         Legal);

      // Custom-lower extensions and truncations from/to mask types.

      setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},

                         VT, Custom);

      // RVV has native int->float & float->int conversions where the

      // element type sizes are within one power-of-two of each other. Any

      // wider distances between type sizes have to be lowered as sequences

      // which progressively narrow the gap in stages.

      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,

                          ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,

                          ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,

                          ISD::STRICT_FP_TO_UINT},

                         VT, Custom);

      setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,

                         Custom);

      setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);

      setOperationAction({ISD::AVGFLOORU, ISD::AVGCEILU, ISD::SADDSAT,

                          ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT},

                         VT, Legal);

      // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"

      // nodes which truncate by one power of two at a time.

      setOperationAction(ISD::TRUNCATE, VT, Custom);

      // Custom-lower insert/extract operations to simplify patterns.

      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,

                         Custom);

      // Custom-lower reduction operations to set up the corresponding custom

      // nodes' operands.

      setOperationAction(IntegerVecReduceOps, VT, Custom);

      setOperationAction(IntegerVPOps, VT, Custom);

      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);

      setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},

                         VT, Custom);

      setOperationAction(

          {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,

           ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},

          VT, Custom);

      setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,

                          ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},

                         VT, Custom);

      setOperationAction(ISD::SELECT, VT, Custom);

      setOperationAction(ISD::SELECT_CC, VT, Expand);

      setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);

      for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {

        setTruncStoreAction(VT, OtherVT, Expand);

        setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,

                         OtherVT, Expand);

      }

      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);

      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);

      // Splice

      setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);

      if (Subtarget.hasStdExtZvkb()) {

        setOperationAction(ISD::BSWAP, VT, Legal);

        setOperationAction(ISD::VP_BSWAP, VT, Custom);

      } else {

        setOperationAction({ISD::BSWAP, ISD::VP_BSWAP}, VT, Expand);

        setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);

      }

      if (Subtarget.hasStdExtZvbb()) {

        setOperationAction(ISD::BITREVERSE, VT, Legal);

        setOperationAction(ISD::VP_BITREVERSE, VT, Custom);

        setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,

                            ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},

                           VT, Custom);

      } else {

        setOperationAction({ISD::BITREVERSE, ISD::VP_BITREVERSE}, VT, Expand);

        setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand);

        setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,

                            ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},

                           VT, Expand);

        // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the

        // range of f32.

        EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());

        if (isTypeLegal(FloatVT)) {

          setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,

                              ISD::CTTZ_ZERO_UNDEF, ISD::VP_CTLZ,

                              ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},

                             VT, Custom);

        }

      }

    }

    // Expand various CCs to best match the RVV ISA, which natively supports UNE

    // but no other unordered comparisons, and supports all ordered comparisons

    // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization

    // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),

    // and we pattern-match those back to the "original", swapping operands once

    // more. This way we catch both operations and both "vf" and "fv" forms with

    // fewer patterns.

    static const ISD::CondCode VFPCCToExpand[] = {

        ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,

        ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,

        ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,

    };

    // TODO: support more ops.

    static const unsigned ZvfhminPromoteOps[] = {

        ISD::FMINNUM,     ISD::FMAXNUM,      ISD::FADD,        ISD::FSUB,

        ISD::FMUL,        ISD::FMA,          ISD::FDIV,        ISD::FSQRT,

        ISD::FABS,        ISD::FNEG,         ISD::FCOPYSIGN,   ISD::FCEIL,

        ISD::FFLOOR,      ISD::FROUND,       ISD::FROUNDEVEN,  ISD::FRINT,

        ISD::FNEARBYINT,  ISD::IS_FPCLASS,   ISD::SETCC,       ISD::FMAXIMUM,

        ISD::FMINIMUM,    ISD::STRICT_FADD,  ISD::STRICT_FSUB, ISD::STRICT_FMUL,

        ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA};

    // TODO: support more vp ops.

    static const unsigned ZvfhminPromoteVPOps[] = {

        ISD::VP_FADD,        ISD::VP_FSUB,         ISD::VP_FMUL,

        ISD::VP_FDIV,        ISD::VP_FNEG,         ISD::VP_FABS,