//===- AArch64.cpp --------------------------------------------------------===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

#include "ABIInfoImpl.h"

#include "TargetInfo.h"

using namespace clang;

using namespace clang::CodeGen;

//===----------------------------------------------------------------------===//

// AArch64 ABI Implementation

//===----------------------------------------------------------------------===//

namespace {

class AArch64ABIInfo : public ABIInfo {

  AArch64ABIKind Kind;

public:

  AArch64ABIInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)

      : ABIInfo(CGT), Kind(Kind) {}

private:

  AArch64ABIKind getABIKind() const { return Kind; }

  bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }

  ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadic) const;

  ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadic,

                                  unsigned CallingConvention) const;

  ABIArgInfo coerceIllegalVector(QualType Ty) const;

  bool isHomogeneousAggregateBaseType(QualType Ty) const override;

  bool isHomogeneousAggregateSmallEnough(const Type *Ty,

                                         uint64_t Members) const override;

  bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;

  bool isIllegalVectorType(QualType Ty) const;

  void computeInfo(CGFunctionInfo &FI) const override {

    if (!::classifyReturnType(getCXXABI(), FI, *this))

      FI.getReturnInfo() =

          classifyReturnType(FI.getReturnType(), FI.isVariadic());

    for (auto &it : FI.arguments())

      it.info = classifyArgumentType(it.type, FI.isVariadic(),

                                     FI.getCallingConvention());

  }

  Address EmitDarwinVAArg(Address VAListAddr, QualType Ty,

                          CodeGenFunction &CGF) const;

  Address EmitAAPCSVAArg(Address VAListAddr, QualType Ty,

                         CodeGenFunction &CGF) const;

  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,

                    QualType Ty) const override {

    llvm::Type *BaseTy = CGF.ConvertType(Ty);

    if (isa<llvm::ScalableVectorType>(BaseTy))

      llvm::report_fatal_error("Passing SVE types to variadic functions is "

                               "currently not supported");

    return Kind == AArch64ABIKind::Win64 ? EmitMSVAArg(CGF, VAListAddr, Ty)

           : isDarwinPCS()               ? EmitDarwinVAArg(VAListAddr, Ty, CGF)

                                         : EmitAAPCSVAArg(VAListAddr, Ty, CGF);

  }

  Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,

                      QualType Ty) const override;

  bool allowBFloatArgsAndRet() const override {

    return getTarget().hasBFloat16Type();

  }

};

class AArch64SwiftABIInfo : public SwiftABIInfo {

public:

  explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)

      : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}

  bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,

                         unsigned NumElts) const override;

};

class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {

public:

  AArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind Kind)

      : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGT, Kind)) {

    SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(CGT);

  }

  StringRef getARCRetainAutoreleasedReturnValueMarker() const override {

    return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";

  }

  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {

    return 31;

  }

  bool doesReturnSlotInterfereWithArgs() const override { return false; }

  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,

                           CodeGen::CodeGenModule &CGM) const override {

    const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);

    if (!FD)

      return;

    const auto *TA = FD->getAttr<TargetAttr>();

    if (TA == nullptr)

      return;

    ParsedTargetAttr Attr =

        CGM.getTarget().parseTargetAttr(TA->getFeaturesStr());

    if (Attr.BranchProtection.empty())

      return;

    TargetInfo::BranchProtectionInfo BPI;

    StringRef Error;

    (void)CGM.getTarget().validateBranchProtection(Attr.BranchProtection,

                                                   Attr.CPU, BPI, Error);

    assert(Error.empty());

    auto *Fn = cast<llvm::Function>(GV);

    static const char *SignReturnAddrStr[] = {"none", "non-leaf", "all"};

    Fn->addFnAttr("sign-return-address", SignReturnAddrStr[static_cast<int>(BPI.SignReturnAddr)]);

    if (BPI.SignReturnAddr != LangOptions::SignReturnAddressScopeKind::None) {

      Fn->addFnAttr("sign-return-address-key",

                    BPI.SignKey == LangOptions::SignReturnAddressKeyKind::AKey

                        ? "a_key"

                        : "b_key");

    }

    Fn->addFnAttr("branch-target-enforcement",

                  BPI.BranchTargetEnforcement ? "true" : "false");

    Fn->addFnAttr("branch-protection-pauth-lr",

                  BPI.BranchProtectionPAuthLR ? "true" : "false");

    Fn->addFnAttr("guarded-control-stack",

                  BPI.GuardedControlStack ? "true" : "false");

  }

  bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,

                                llvm::Type *Ty) const override {

    if (CGF.getTarget().hasFeature("ls64")) {

      auto *ST = dyn_cast<llvm::StructType>(Ty);

      if (ST && ST->getNumElements() == 1) {

        auto *AT = dyn_cast<llvm::ArrayType>(ST->getElementType(0));

        if (AT && AT->getNumElements() == 8 &&

            AT->getElementType()->isIntegerTy(64))

          return true;

      }

    }

    return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);

  }

};

class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {

public:

  WindowsAArch64TargetCodeGenInfo(CodeGenTypes &CGT, AArch64ABIKind K)

      : AArch64TargetCodeGenInfo(CGT, K) {}

  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,

                           CodeGen::CodeGenModule &CGM) const override;

  void getDependentLibraryOption(llvm::StringRef Lib,

                                 llvm::SmallString<24> &Opt) const override {

    Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);

  }

  void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,

                               llvm::SmallString<32> &Opt) const override {

    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";

  }

};

void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(

    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {

  AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);

  if (GV->isDeclaration())

    return;

  addStackProbeTargetAttributes(D, GV, CGM);

}

}

ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty) const {

  assert(Ty->isVectorType() && "expected vector type!");

  const auto *VT = Ty->castAs<VectorType>();

  if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {

    assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");

    assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==

               BuiltinType::UChar &&

           "unexpected builtin type for SVE predicate!");

    return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(

        llvm::Type::getInt1Ty(getVMContext()), 16));

  }

  if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {

    assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");

    const auto *BT = VT->getElementType()->castAs<BuiltinType>();

    llvm::ScalableVectorType *ResType = nullptr;

    switch (BT->getKind()) {

    default:

      llvm_unreachable("unexpected builtin type for SVE vector!");

    case BuiltinType::SChar:

    case BuiltinType::UChar:

      ResType = llvm::ScalableVectorType::get(

          llvm::Type::getInt8Ty(getVMContext()), 16);

      break;

    case BuiltinType::Short:

    case BuiltinType::UShort:

      ResType = llvm::ScalableVectorType::get(

          llvm::Type::getInt16Ty(getVMContext()), 8);

      break;

    case BuiltinType::Int:

    case BuiltinType::UInt:

      ResType = llvm::ScalableVectorType::get(

          llvm::Type::getInt32Ty(getVMContext()), 4);

      break;

    case BuiltinType::Long:

    case BuiltinType::ULong:

      ResType = llvm::ScalableVectorType::get(

          llvm::Type::getInt64Ty(getVMContext()), 2);

      break;

    case BuiltinType::Half:

      ResType = llvm::ScalableVectorType::get(

          llvm::Type::getHalfTy(getVMContext()), 8);

      break;

    case BuiltinType::Float:

      ResType = llvm::ScalableVectorType::get(

          llvm::Type::getFloatTy(getVMContext()), 4);

      break;

    case BuiltinType::Double:

      ResType = llvm::ScalableVectorType::get(

          llvm::Type::getDoubleTy(getVMContext()), 2);

      break;

    case BuiltinType::BFloat16:

      ResType = llvm::ScalableVectorType::get(

          llvm::Type::getBFloatTy(getVMContext()), 8);

      break;

    }

    return ABIArgInfo::getDirect(ResType);

  }

  uint64_t Size = getContext().getTypeSize(Ty);

  // Android promotes <2 x i8> to i16, not i32

  if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {

    llvm::Type *ResType = llvm::Type::getInt16Ty(getVMContext());

    return ABIArgInfo::getDirect(ResType);

  }

  if (Size <= 32) {

    llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext());

    return ABIArgInfo::getDirect(ResType);

  }

  if (Size == 64) {

    auto *ResType =

        llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2);

    return ABIArgInfo::getDirect(ResType);

  }

  if (Size == 128) {

    auto *ResType =

        llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);

    return ABIArgInfo::getDirect(ResType);

  }

  return getNaturalAlignIndirect(Ty, /*ByVal=*/false);

}

ABIArgInfo

AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadic,

                                     unsigned CallingConvention) const {

  Ty = useFirstFieldIfTransparentUnion(Ty);

  // Handle illegal vector types here.

  if (isIllegalVectorType(Ty))

    return coerceIllegalVector(Ty);

  if (!isAggregateTypeForABI(Ty)) {

    // Treat an enum type as its underlying type.

    if (const EnumType *EnumTy = Ty->getAs<EnumType>())

      Ty = EnumTy->getDecl()->getIntegerType();

    if (const auto *EIT = Ty->getAs<BitIntType>())

      if (EIT->getNumBits() > 128)

        return getNaturalAlignIndirect(Ty);

    return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()

                ? ABIArgInfo::getExtend(Ty)

                : ABIArgInfo::getDirect());

  }

  // Structures with either a non-trivial destructor or a non-trivial

  // copy constructor are always indirect.

  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {

    return getNaturalAlignIndirect(Ty, /*ByVal=*/RAA ==

                                     CGCXXABI::RAA_DirectInMemory);

  }

  // Empty records are always ignored on Darwin, but actually passed in C++ mode

  // elsewhere for GNU compatibility.

  uint64_t Size = getContext().getTypeSize(Ty);

  bool IsEmpty = isEmptyRecord(getContext(), Ty, true);

  if (IsEmpty || Size == 0) {

    if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())

      return ABIArgInfo::getIgnore();

    // GNU C mode. The only argument that gets ignored is an empty one with size

    // 0.

    if (IsEmpty && Size == 0)

      return ABIArgInfo::getIgnore();

    return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));

  }

  // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.

  const Type *Base = nullptr;

  uint64_t Members = 0;

  bool IsWin64 = Kind == AArch64ABIKind::Win64 ||

                 CallingConvention == llvm::CallingConv::Win64;

  bool IsWinVariadic = IsWin64 && IsVariadic;

  // In variadic functions on Windows, all composite types are treated alike,

  // no special handling of HFAs/HVAs.

  if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {

    if (Kind != AArch64ABIKind::AAPCS)

      return ABIArgInfo::getDirect(

          llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members));

    // For HFAs/HVAs, cap the argument alignment to 16, otherwise

    // set it to 8 according to the AAPCS64 document.

    unsigned Align =

        getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();

    Align = (Align >= 16) ? 16 : 8;

    return ABIArgInfo::getDirect(

        llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members), 0,

        nullptr, true, Align);

  }

  // Aggregates <= 16 bytes are passed directly in registers or on the stack.

  if (Size <= 128) {

    // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of

    // same size and alignment.

    if (getTarget().isRenderScriptTarget()) {

      return coerceToIntArray(Ty, getContext(), getVMContext());

    }

    unsigned Alignment;

    if (Kind == AArch64ABIKind::AAPCS) {

      Alignment = getContext().getTypeUnadjustedAlign(Ty);

      Alignment = Alignment < 128 ? 64 : 128;

    } else {

      Alignment =

          std::max(getContext().getTypeAlign(Ty),

                   (unsigned)getTarget().getPointerWidth(LangAS::Default));

    }

    Size = llvm::alignTo(Size, Alignment);

    // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.

    // For aggregates with 16-byte alignment, we use i128.

    llvm::Type *BaseTy = llvm::Type::getIntNTy(getVMContext(), Alignment);

    return ABIArgInfo::getDirect(

        Size == Alignment ? BaseTy

                          : llvm::ArrayType::get(BaseTy, Size / Alignment));

  }

  return getNaturalAlignIndirect(Ty, /*ByVal=*/false);

}

ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,

                                              bool IsVariadic) const {

  if (RetTy->isVoidType())

    return ABIArgInfo::getIgnore();

  if (const auto *VT = RetTy->getAs<VectorType>()) {

    if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||

        VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)

      return coerceIllegalVector(RetTy);

  }

  // Large vector types should be returned via memory.

  if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128)

    return getNaturalAlignIndirect(RetTy);

  if (!isAggregateTypeForABI(RetTy)) {

    // Treat an enum type as its underlying type.

    if (const EnumType *EnumTy = RetTy->getAs<EnumType>())

      RetTy = EnumTy->getDecl()->getIntegerType();

    if (const auto *EIT = RetTy->getAs<BitIntType>())

      if (EIT->getNumBits() > 128)

        return getNaturalAlignIndirect(RetTy);

    return (isPromotableIntegerTypeForABI(RetTy) && isDarwinPCS()

                ? ABIArgInfo::getExtend(RetTy)

                : ABIArgInfo::getDirect());

  }

  uint64_t Size = getContext().getTypeSize(RetTy);

  if (isEmptyRecord(getContext(), RetTy, true) || Size == 0)

    return ABIArgInfo::getIgnore();

  const Type *Base = nullptr;

  uint64_t Members = 0;

  if (isHomogeneousAggregate(RetTy, Base, Members) &&

      !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&

        IsVariadic))

    // Homogeneous Floating-point Aggregates (HFAs) are returned directly.

    return ABIArgInfo::getDirect();

  // Aggregates <= 16 bytes are returned directly in registers or on the stack.

  if (Size <= 128) {

    // On RenderScript, coerce Aggregates <= 16 bytes to an integer array of

    // same size and alignment.

    if (getTarget().isRenderScriptTarget()) {

      return coerceToIntArray(RetTy, getContext(), getVMContext());

    }

    if (Size <= 64 && getDataLayout().isLittleEndian()) {

      // Composite types are returned in lower bits of a 64-bit register for LE,

      // and in higher bits for BE. However, integer types are always returned

      // in lower bits for both LE and BE, and they are not rounded up to

      // 64-bits. We can skip rounding up of composite types for LE, but not for

      // BE, otherwise composite types will be indistinguishable from integer

      // types.

      return ABIArgInfo::getDirect(

          llvm::IntegerType::get(getVMContext(), Size));

    }

    unsigned Alignment = getContext().getTypeAlign(RetTy);

    Size = llvm::alignTo(Size, 64); // round up to multiple of 8 bytes

    // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.

    // For aggregates with 16-byte alignment, we use i128.

    if (Alignment < 128 && Size == 128) {

      llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext());

      return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64));

    }

    return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size));

  }

  return getNaturalAlignIndirect(RetTy);

}

/// isIllegalVectorType - check whether the vector type is legal for AArch64.

bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {

  if (const VectorType *VT = Ty->getAs<VectorType>()) {

    // Check whether VT is a fixed-length SVE vector. These types are

    // represented as scalable vectors in function args/return and must be

    // coerced from fixed vectors.

    if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||

        VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)

      return true;

    // Check whether VT is legal.

    unsigned NumElements = VT->getNumElements();

    uint64_t Size = getContext().getTypeSize(VT);

    // NumElements should be power of 2.

    if (!llvm::isPowerOf2_32(NumElements))

      return true;

    // arm64_32 has to be compatible with the ARM logic here, which allows huge

    // vectors for some reason.

    llvm::Triple Triple = getTarget().getTriple();

    if (Triple.getArch() == llvm::Triple::aarch64_32 &&

        Triple.isOSBinFormatMachO())

      return Size <= 32;

    return Size != 64 && (Size != 128 || NumElements == 1);

  }

  return false;

}

bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,

                                            llvm::Type *EltTy,

                                            unsigned NumElts) const {

  if (!llvm::isPowerOf2_32(NumElts))

    return false;

  if (VectorSize.getQuantity() != 8 &&

      (VectorSize.getQuantity() != 16 || NumElts == 1))

    return false;

  return true;

}

bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {

  // Homogeneous aggregates for AAPCS64 must have base types of a floating

  // point type or a short-vector type. This is the same as the 32-bit ABI,

  // but with the difference that any floating-point type is allowed,

  // including __fp16.

  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {

    if (BT->isFloatingPoint())

      return true;

  } else if (const VectorType *VT = Ty->getAs<VectorType>()) {

    unsigned VecSize = getContext().getTypeSize(VT);

    if (VecSize == 64 || VecSize == 128)

      return true;

  }

  return false;

}

bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,

                                                       uint64_t Members) const {

  return Members <= 4;

}

bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()

    const {

  // AAPCS64 says that the rule for whether something is a homogeneous

  // aggregate is applied to the output of the data layout decision. So

  // anything that doesn't affect the data layout also does not affect

  // homogeneity. In particular, zero-length bitfields don't stop a struct

  // being homogeneous.

  return true;

}

Address AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,

                                       CodeGenFunction &CGF) const {

  ABIArgInfo AI = classifyArgumentType(Ty, /*IsVariadic=*/true,

                                       CGF.CurFnInfo->getCallingConvention());

  // Empty records are ignored for parameter passing purposes.

  if (AI.isIgnore()) {

    uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8;

    CharUnits SlotSize = CharUnits::fromQuantity(PointerSize);

    VAListAddr = VAListAddr.withElementType(CGF.Int8PtrTy);

    auto *Load = CGF.Builder.CreateLoad(VAListAddr);

    return Address(Load, CGF.ConvertTypeForMem(Ty), SlotSize);

  }

  bool IsIndirect = AI.isIndirect();

  llvm::Type *BaseTy = CGF.ConvertType(Ty);

  if (IsIndirect)

    BaseTy = llvm::PointerType::getUnqual(BaseTy);

  else if (AI.getCoerceToType())

    BaseTy = AI.getCoerceToType();

  unsigned NumRegs = 1;

  if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(BaseTy)) {

    BaseTy = ArrTy->getElementType();

    NumRegs = ArrTy->getNumElements();

  }

  bool IsFPR = BaseTy->isFloatingPointTy() || BaseTy->isVectorTy();

  // The AArch64 va_list type and handling is specified in the Procedure Call

  // Standard, section B.4:

  //

  // struct {

  //   void *__stack;

  //   void *__gr_top;

  //   void *__vr_top;

  //   int __gr_offs;

  //   int __vr_offs;

  // };

  llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");

  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");

  llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");

  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");

  CharUnits TySize = getContext().getTypeSizeInChars(Ty);

  CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty);

  Address reg_offs_p = Address::invalid();

  llvm::Value *reg_offs = nullptr;

  int reg_top_index;

  int RegSize = IsIndirect ? 8 : TySize.getQuantity();

  if (!IsFPR) {

    // 3 is the field number of __gr_offs

    reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p");

    reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs");

    reg_top_index = 1; // field number for __gr_top

    RegSize = llvm::alignTo(RegSize, 8);

  } else {

    // 4 is the field number of __vr_offs.

    reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p");

    reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs");

    reg_top_index = 2; // field number for __vr_top

    RegSize = 16 * NumRegs;

  }

  //=======================================

  // Find out where argument was passed

  //=======================================

  // If reg_offs >= 0 we're already using the stack for this type of

  // argument. We don't want to keep updating reg_offs (in case it overflows,

  // though anyone passing 2GB of arguments, each at most 16 bytes, deserves

  // whatever they get).

  llvm::Value *UsingStack = nullptr;

  UsingStack = CGF.Builder.CreateICmpSGE(

      reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, 0));

  CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, MaybeRegBlock);

  // Otherwise, at least some kind of argument could go in these registers, the

  // question is whether this particular type is too big.

  CGF.EmitBlock(MaybeRegBlock);

  // Integer arguments may need to correct register alignment (for example a

  // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we

  // align __gr_offs to calculate the potential address.

  if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {

    int Align = TyAlign.getQuantity();

    reg_offs = CGF.Builder.CreateAdd(

        reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1),

        "align_regoffs");

    reg_offs = CGF.Builder.CreateAnd(

        reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, -Align),

        "aligned_regoffs");

  }

  // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.

  // The fact that this is done unconditionally reflects the fact that

  // allocating an argument to the stack also uses up all the remaining

  // registers of the appropriate kind.

  llvm::Value *NewOffset = nullptr;

  NewOffset = CGF.Builder.CreateAdd(

      reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs");

  CGF.Builder.CreateStore(NewOffset, reg_offs_p);

  // Now we're in a position to decide whether this argument really was in

  // registers or not.

  llvm::Value *InRegs = nullptr;

  InRegs = CGF.Builder.CreateICmpSLE(

      NewOffset, llvm::ConstantInt::get(CGF.Int32Ty, 0), "inreg");

  CGF.Builder.CreateCondBr(InRegs, InRegBlock, OnStackBlock);

  //=======================================

  // Argument was in registers

  //=======================================

  // Now we emit the code for if the argument was originally passed in

  // registers. First start the appropriate block:

  CGF.EmitBlock(InRegBlock);

  llvm::Value *reg_top = nullptr;

  Address reg_top_p =

      CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p");

  reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top");

  Address BaseAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, reg_top, reg_offs),

                   CGF.Int8Ty, CharUnits::fromQuantity(IsFPR ? 16 : 8));

  Address RegAddr = Address::invalid();

  llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty), *ElementTy = MemTy;

  if (IsIndirect) {

    // If it's been passed indirectly (actually a struct), whatever we find from

    // stored registers or on the stack will actually be a struct **.

    MemTy = llvm::PointerType::getUnqual(MemTy);

  }

  const Type *Base = nullptr;

  uint64_t NumMembers = 0;

  bool IsHFA = isHomogeneousAggregate(Ty, Base, NumMembers);

  if (IsHFA && NumMembers > 1) {

    // Homogeneous aggregates passed in registers will have their elements split

    // and stored 16-bytes apart regardless of size (they're notionally in qN,

    // qN+1, ...). We reload and store into a temporary local variable

    // contiguously.

    assert(!IsIndirect && "Homogeneous aggregates should be passed directly");

    auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0));

    llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0));

    llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers);

    Address Tmp = CGF.CreateTempAlloca(HFATy,

                                       std::max(TyAlign, BaseTyInfo.Align));

    // On big-endian platforms, the value will be right-aligned in its slot.

    int Offset = 0;

    if (CGF.CGM.getDataLayout().isBigEndian() &&

        BaseTyInfo.Width.getQuantity() < 16)

      Offset = 16 - BaseTyInfo.Width.getQuantity();

    for (unsigned i = 0; i < NumMembers; ++i) {

      CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset);

      Address LoadAddr =

        CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset);

      LoadAddr = LoadAddr.withElementType(BaseTy);

      Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Tmp, i);

      llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr);

      CGF.Builder.CreateStore(Elem, StoreAddr);

    }

    RegAddr = Tmp.withElementType(MemTy);

  } else {

    // Otherwise the object is contiguous in memory.

    // It might be right-aligned in its slot.

    CharUnits SlotSize = BaseAddr.getAlignment();

    if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&

        (IsHFA || !isAggregateTypeForABI(Ty)) &&

        TySize < SlotSize) {

      CharUnits Offset = SlotSize - TySize;

      BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset);

    }

    RegAddr = BaseAddr.withElementType(MemTy);

  }

  CGF.EmitBranch(ContBlock);

  //=======================================

  // Argument was on the stack

  //=======================================

  CGF.EmitBlock(OnStackBlock);

  Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p");

  llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack");

  // Again, stack arguments may need realignment. In this case both integer and

  // floating-point ones might be affected.

  if (!IsIndirect && TyAlign.getQuantity() > 8) {

    int Align = TyAlign.getQuantity();

    OnStackPtr = CGF.Builder.CreatePtrToInt(OnStackPtr, CGF.Int64Ty);

    OnStackPtr = CGF.Builder.CreateAdd(

        OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, Align - 1),

        "align_stack");

    OnStackPtr = CGF.Builder.CreateAnd(

        OnStackPtr, llvm::ConstantInt::get(CGF.Int64Ty, -Align),

        "align_stack");

    OnStackPtr = CGF.Builder.CreateIntToPtr(OnStackPtr, CGF.Int8PtrTy);

  }

  Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,

                                std::max(CharUnits::fromQuantity(8), TyAlign));

  // All stack slots are multiples of 8 bytes.

  CharUnits StackSlotSize = CharUnits::fromQuantity(8);

  CharUnits StackSize;

  if (IsIndirect)

    StackSize = StackSlotSize;

  else

    StackSize = TySize.alignTo(StackSlotSize);

  llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize);

  llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(

      CGF.Int8Ty, OnStackPtr, StackSizeC, "new_stack");

  // Write the new value of __stack for the next call to va_arg

  CGF.Builder.CreateStore(NewStack, stack_p);

  if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) &&

      TySize < StackSlotSize) {

    CharUnits Offset = StackSlotSize - TySize;

    OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset);

  }

  OnStackAddr = OnStackAddr.withElementType(MemTy);

  CGF.EmitBranch(ContBlock);

  //=======================================

  // Tidy up

  //=======================================

  CGF.EmitBlock(ContBlock);

  Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, OnStackAddr,

                                 OnStackBlock, "vaargs.addr");

  if (IsIndirect)

    return Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy,

                   TyAlign);

  return ResAddr;

}

Address AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,

                                        CodeGenFunction &CGF) const {

  // The backend's lowering doesn't support va_arg for aggregates or

  // illegal vector types.  Lower VAArg here for these cases and use

  // the LLVM va_arg instruction for everything else.

  if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty))

    return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());

  uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8;

  CharUnits SlotSize = CharUnits::fromQuantity(PointerSize);

  // Empty records are ignored for parameter passing purposes.

  if (isEmptyRecord(getContext(), Ty, true))

    return Address(CGF.Builder.CreateLoad(VAListAddr, "ap.cur"),

                   CGF.ConvertTypeForMem(Ty), SlotSize);

  // The size of the actual thing passed, which might end up just

  // being a pointer for indirect types.

  auto TyInfo = getContext().getTypeInfoInChars(Ty);

  // Arguments bigger than 16 bytes which aren't homogeneous

  // aggregates should be passed indirectly.

  bool IsIndirect = false;

  if (TyInfo.Width.getQuantity() > 16) {

    const Type *Base = nullptr;

    uint64_t Members = 0;

    IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);

  }

  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,

                          TyInfo, SlotSize, /*AllowHigherAlign*/ true);

}

Address AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,

                                    QualType Ty) const {

  bool IsIndirect = false;

  // Composites larger than 16 bytes are passed by reference.

  if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) > 128)

    IsIndirect = true;

  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,

                          CGF.getContext().getTypeInfoInChars(Ty),

                          CharUnits::fromQuantity(8),

                          /*allowHigherAlign*/ false);

}

std::unique_ptr<TargetCodeGenInfo>

CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM,

                                        AArch64ABIKind Kind) {

  return std::make_unique<AArch64TargetCodeGenInfo>(CGM.getTypes(), Kind);

}

std::unique_ptr<TargetCodeGenInfo>

CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM,

                                               AArch64ABIKind K) {

  return std::make_unique<WindowsAArch64TargetCodeGenInfo>(CGM.getTypes(), K);

}