//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// This file contains the code for emitting atomic operations.

//

//===----------------------------------------------------------------------===//

#include "CGCall.h"

#include "CGRecordLayout.h"

#include "CodeGenFunction.h"

#include "CodeGenModule.h"

#include "TargetInfo.h"

#include "clang/AST/ASTContext.h"

#include "clang/CodeGen/CGFunctionInfo.h"

#include "clang/Frontend/FrontendDiagnostic.h"

#include "llvm/ADT/DenseMap.h"

#include "llvm/IR/DataLayout.h"

#include "llvm/IR/Intrinsics.h"

#include "llvm/IR/Operator.h"

using namespace clang;

using namespace CodeGen;

namespace {

  class AtomicInfo {

    CodeGenFunction &CGF;

    QualType AtomicTy;

    QualType ValueTy;

    uint64_t AtomicSizeInBits;

    uint64_t ValueSizeInBits;

    CharUnits AtomicAlign;

    CharUnits ValueAlign;

    TypeEvaluationKind EvaluationKind;

    bool UseLibcall;

    LValue LVal;

    CGBitFieldInfo BFI;

  public:

    AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)

        : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),

          EvaluationKind(TEK_Scalar), UseLibcall(true) {

      assert(!lvalue.isGlobalReg());

      ASTContext &C = CGF.getContext();

      if (lvalue.isSimple()) {

        AtomicTy = lvalue.getType();

        if (auto *ATy = AtomicTy->getAs<AtomicType>())

          ValueTy = ATy->getValueType();

        else

          ValueTy = AtomicTy;

        EvaluationKind = CGF.getEvaluationKind(ValueTy);

        uint64_t ValueAlignInBits;

        uint64_t AtomicAlignInBits;

        TypeInfo ValueTI = C.getTypeInfo(ValueTy);

        ValueSizeInBits = ValueTI.Width;

        ValueAlignInBits = ValueTI.Align;

        TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);

        AtomicSizeInBits = AtomicTI.Width;

        AtomicAlignInBits = AtomicTI.Align;

        assert(ValueSizeInBits <= AtomicSizeInBits);

        assert(ValueAlignInBits <= AtomicAlignInBits);

        AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);

        ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);

        if (lvalue.getAlignment().isZero())

          lvalue.setAlignment(AtomicAlign);

        LVal = lvalue;

      } else if (lvalue.isBitField()) {

        ValueTy = lvalue.getType();

        ValueSizeInBits = C.getTypeSize(ValueTy);

        auto &OrigBFI = lvalue.getBitFieldInfo();

        auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());

        AtomicSizeInBits = C.toBits(

            C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)

                .alignTo(lvalue.getAlignment()));

        llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer();

        auto OffsetInChars =

            (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *

            lvalue.getAlignment();

        llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(

            CGF.Int8Ty, BitFieldPtr, OffsetInChars.getQuantity());

        StoragePtr = CGF.Builder.CreateAddrSpaceCast(

            StoragePtr, CGF.UnqualPtrTy, "atomic_bitfield_base");

        BFI = OrigBFI;

        BFI.Offset = Offset;

        BFI.StorageSize = AtomicSizeInBits;

        BFI.StorageOffset += OffsetInChars;

        llvm::Type *StorageTy = CGF.Builder.getIntNTy(AtomicSizeInBits);

        LVal = LValue::MakeBitfield(

            Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,

            lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());

        AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);

        if (AtomicTy.isNull()) {

          llvm::APInt Size(

              /*numBits=*/32,

              C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());

          AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr,

                                            ArraySizeModifier::Normal,

                                            /*IndexTypeQuals=*/0);

        }

        AtomicAlign = ValueAlign = lvalue.getAlignment();

      } else if (lvalue.isVectorElt()) {

        ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();

        ValueSizeInBits = C.getTypeSize(ValueTy);

        AtomicTy = lvalue.getType();

        AtomicSizeInBits = C.getTypeSize(AtomicTy);

        AtomicAlign = ValueAlign = lvalue.getAlignment();

        LVal = lvalue;

      } else {

        assert(lvalue.isExtVectorElt());

        ValueTy = lvalue.getType();

        ValueSizeInBits = C.getTypeSize(ValueTy);

        AtomicTy = ValueTy = CGF.getContext().getExtVectorType(

            lvalue.getType(), cast<llvm::FixedVectorType>(

                                  lvalue.getExtVectorAddress().getElementType())

                                  ->getNumElements());

        AtomicSizeInBits = C.getTypeSize(AtomicTy);

        AtomicAlign = ValueAlign = lvalue.getAlignment();

        LVal = lvalue;

      }

      UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(

          AtomicSizeInBits, C.toBits(lvalue.getAlignment()));

    }

    QualType getAtomicType() const { return AtomicTy; }

    QualType getValueType() const { return ValueTy; }

    CharUnits getAtomicAlignment() const { return AtomicAlign; }

    uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }

    uint64_t getValueSizeInBits() const { return ValueSizeInBits; }

    TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }

    bool shouldUseLibcall() const { return UseLibcall; }

    const LValue &getAtomicLValue() const { return LVal; }

    llvm::Value *getAtomicPointer() const {

      if (LVal.isSimple())

        return LVal.getPointer(CGF);

      else if (LVal.isBitField())

        return LVal.getBitFieldPointer();

      else if (LVal.isVectorElt())

        return LVal.getVectorPointer();

      assert(LVal.isExtVectorElt());

      return LVal.getExtVectorPointer();

    }

    Address getAtomicAddress() const {

      llvm::Type *ElTy;

      if (LVal.isSimple())

        ElTy = LVal.getAddress(CGF).getElementType();

      else if (LVal.isBitField())

        ElTy = LVal.getBitFieldAddress().getElementType();

      else if (LVal.isVectorElt())

        ElTy = LVal.getVectorAddress().getElementType();

      else

        ElTy = LVal.getExtVectorAddress().getElementType();

      return Address(getAtomicPointer(), ElTy, getAtomicAlignment());

    }

    Address getAtomicAddressAsAtomicIntPointer() const {

      return castToAtomicIntPointer(getAtomicAddress());

    }

    /// Is the atomic size larger than the underlying value type?

    ///

    /// Note that the absence of padding does not mean that atomic

    /// objects are completely interchangeable with non-atomic

    /// objects: we might have promoted the alignment of a type

    /// without making it bigger.

    bool hasPadding() const {

      return (ValueSizeInBits != AtomicSizeInBits);

    }

    bool emitMemSetZeroIfNecessary() const;

    llvm::Value *getAtomicSizeValue() const {

      CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);

      return CGF.CGM.getSize(size);

    }

    /// Cast the given pointer to an integer pointer suitable for atomic

    /// operations if the source.

    Address castToAtomicIntPointer(Address Addr) const;

    /// If Addr is compatible with the iN that will be used for an atomic

    /// operation, bitcast it. Otherwise, create a temporary that is suitable

    /// and copy the value across.

    Address convertToAtomicIntPointer(Address Addr) const;

    /// Turn an atomic-layout object into an r-value.

    RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,

                                     SourceLocation loc, bool AsValue) const;

    /// Converts a rvalue to integer value.

    llvm::Value *convertRValueToInt(RValue RVal) const;

    RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,

                                     AggValueSlot ResultSlot,

                                     SourceLocation Loc, bool AsValue) const;

    /// Copy an atomic r-value into atomic-layout memory.

    void emitCopyIntoMemory(RValue rvalue) const;

    /// Project an l-value down to the value field.

    LValue projectValue() const {

      assert(LVal.isSimple());

      Address addr = getAtomicAddress();

      if (hasPadding())

        addr = CGF.Builder.CreateStructGEP(addr, 0);

      return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),

                              LVal.getBaseInfo(), LVal.getTBAAInfo());

    }

    /// Emits atomic load.

    /// \returns Loaded value.

    RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,

                          bool AsValue, llvm::AtomicOrdering AO,

                          bool IsVolatile);

    /// Emits atomic compare-and-exchange sequence.

    /// \param Expected Expected value.

    /// \param Desired Desired value.

    /// \param Success Atomic ordering for success operation.

    /// \param Failure Atomic ordering for failed operation.

    /// \param IsWeak true if atomic operation is weak, false otherwise.

    /// \returns Pair of values: previous value from storage (value type) and

    /// boolean flag (i1 type) with true if success and false otherwise.

    std::pair<RValue, llvm::Value *>

    EmitAtomicCompareExchange(RValue Expected, RValue Desired,

                              llvm::AtomicOrdering Success =

                                  llvm::AtomicOrdering::SequentiallyConsistent,

                              llvm::AtomicOrdering Failure =

                                  llvm::AtomicOrdering::SequentiallyConsistent,

                              bool IsWeak = false);

    /// Emits atomic update.

    /// \param AO Atomic ordering.

    /// \param UpdateOp Update operation for the current lvalue.

    void EmitAtomicUpdate(llvm::AtomicOrdering AO,

                          const llvm::function_ref<RValue(RValue)> &UpdateOp,

                          bool IsVolatile);

    /// Emits atomic update.

    /// \param AO Atomic ordering.

    void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,

                          bool IsVolatile);

    /// Materialize an atomic r-value in atomic-layout memory.

    Address materializeRValue(RValue rvalue) const;

    /// Creates temp alloca for intermediate operations on atomic value.

    Address CreateTempAlloca() const;

  private:

    bool requiresMemSetZero(llvm::Type *type) const;

    /// Emits atomic load as a libcall.

    void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,

                               llvm::AtomicOrdering AO, bool IsVolatile);

    /// Emits atomic load as LLVM instruction.

    llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);

    /// Emits atomic compare-and-exchange op as a libcall.

    llvm::Value *EmitAtomicCompareExchangeLibcall(

        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,

        llvm::AtomicOrdering Success =

            llvm::AtomicOrdering::SequentiallyConsistent,

        llvm::AtomicOrdering Failure =

            llvm::AtomicOrdering::SequentiallyConsistent);

    /// Emits atomic compare-and-exchange op as LLVM instruction.

    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(

        llvm::Value *ExpectedVal, llvm::Value *DesiredVal,

        llvm::AtomicOrdering Success =

            llvm::AtomicOrdering::SequentiallyConsistent,

        llvm::AtomicOrdering Failure =

            llvm::AtomicOrdering::SequentiallyConsistent,

        bool IsWeak = false);

    /// Emit atomic update as libcalls.

    void

    EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,

                            const llvm::function_ref<RValue(RValue)> &UpdateOp,

                            bool IsVolatile);

    /// Emit atomic update as LLVM instructions.

    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,

                            const llvm::function_ref<RValue(RValue)> &UpdateOp,

                            bool IsVolatile);

    /// Emit atomic update as libcalls.

    void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,

                                 bool IsVolatile);

    /// Emit atomic update as LLVM instructions.

    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,

                            bool IsVolatile);

  };

}

Address AtomicInfo::CreateTempAlloca() const {

  Address TempAlloca = CGF.CreateMemTemp(

      (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy

                                                                : AtomicTy,

      getAtomicAlignment(),

      "atomic-temp");

  // Cast to pointer to value type for bitfields.

  if (LVal.isBitField())

    return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(

        TempAlloca, getAtomicAddress().getType(),

        getAtomicAddress().getElementType());

  return TempAlloca;

}

static RValue emitAtomicLibcall(CodeGenFunction &CGF,

                                StringRef fnName,

                                QualType resultType,

                                CallArgList &args) {

  const CGFunctionInfo &fnInfo =

    CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);

  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);

  llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());

  fnAttrB.addAttribute(llvm::Attribute::NoUnwind);

  fnAttrB.addAttribute(llvm::Attribute::WillReturn);

  llvm::AttributeList fnAttrs = llvm::AttributeList::get(

      CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);

  llvm::FunctionCallee fn =

      CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);

  auto callee = CGCallee::forDirect(fn);

  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);

}

/// Does a store of the given IR type modify the full expected width?

static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,

                           uint64_t expectedSize) {

  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);

}

/// Does the atomic type require memsetting to zero before initialization?

///

/// The IR type is provided as a way of making certain queries faster.

bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {

  // If the atomic type has size padding, we definitely need a memset.

  if (hasPadding()) return true;

  // Otherwise, do some simple heuristics to try to avoid it:

  switch (getEvaluationKind()) {

  // For scalars and complexes, check whether the store size of the

  // type uses the full size.

  case TEK_Scalar:

    return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);

  case TEK_Complex:

    return !isFullSizeType(CGF.CGM, type->getStructElementType(0),

                           AtomicSizeInBits / 2);

  // Padding in structs has an undefined bit pattern.  User beware.

  case TEK_Aggregate:

    return false;

  }

  llvm_unreachable("bad evaluation kind");

}

bool AtomicInfo::emitMemSetZeroIfNecessary() const {

  assert(LVal.isSimple());

  Address addr = LVal.getAddress(CGF);

  if (!requiresMemSetZero(addr.getElementType()))

    return false;

  CGF.Builder.CreateMemSet(

      addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),

      CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),

      LVal.getAlignment().getAsAlign());

  return true;

}

static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,

                              Address Dest, Address Ptr,

                              Address Val1, Address Val2,

                              uint64_t Size,

                              llvm::AtomicOrdering SuccessOrder,

                              llvm::AtomicOrdering FailureOrder,

                              llvm::SyncScope::ID Scope) {

  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.

  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);

  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);

  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(

      Ptr, Expected, Desired, SuccessOrder, FailureOrder, Scope);

  Pair->setVolatile(E->isVolatile());

  Pair->setWeak(IsWeak);

  // Cmp holds the result of the compare-exchange operation: true on success,

  // false on failure.

  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);

  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);

  // This basic block is used to hold the store instruction if the operation

  // failed.

  llvm::BasicBlock *StoreExpectedBB =

      CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);

  // This basic block is the exit point of the operation, we should end up

  // here regardless of whether or not the operation succeeded.

  llvm::BasicBlock *ContinueBB =

      CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);

  // Update Expected if Expected isn't equal to Old, otherwise branch to the

  // exit point.

  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);

  CGF.Builder.SetInsertPoint(StoreExpectedBB);

  // Update the memory at Expected with Old's value.

  CGF.Builder.CreateStore(Old, Val1);

  // Finally, branch to the exit point.

  CGF.Builder.CreateBr(ContinueBB);

  CGF.Builder.SetInsertPoint(ContinueBB);

  // Update the memory at Dest with Cmp's value.

  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));

}

/// Given an ordering required on success, emit all possible cmpxchg

/// instructions to cope with the provided (but possibly only dynamically known)

/// FailureOrder.

static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,

                                        bool IsWeak, Address Dest, Address Ptr,

                                        Address Val1, Address Val2,

                                        llvm::Value *FailureOrderVal,

                                        uint64_t Size,

                                        llvm::AtomicOrdering SuccessOrder,

                                        llvm::SyncScope::ID Scope) {

  llvm::AtomicOrdering FailureOrder;

  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {

    auto FOS = FO->getSExtValue();

    if (!llvm::isValidAtomicOrderingCABI(FOS))

      FailureOrder = llvm::AtomicOrdering::Monotonic;

    else

      switch ((llvm::AtomicOrderingCABI)FOS) {

      case llvm::AtomicOrderingCABI::relaxed:

      // 31.7.2.18: "The failure argument shall not be memory_order_release

      // nor memory_order_acq_rel". Fallback to monotonic.

      case llvm::AtomicOrderingCABI::release:

      case llvm::AtomicOrderingCABI::acq_rel:

        FailureOrder = llvm::AtomicOrdering::Monotonic;

        break;

      case llvm::AtomicOrderingCABI::consume:

      case llvm::AtomicOrderingCABI::acquire:

        FailureOrder = llvm::AtomicOrdering::Acquire;

        break;

      case llvm::AtomicOrderingCABI::seq_cst:

        FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;

        break;

      }

    // Prior to c++17, "the failure argument shall be no stronger than the

    // success argument". This condition has been lifted and the only

    // precondition is 31.7.2.18. Effectively treat this as a DR and skip

    // language version checks.

    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,

                      FailureOrder, Scope);

    return;

  }

  // Create all the relevant BB's

  auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);

  auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);

  auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);

  auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);

  // MonotonicBB is arbitrarily chosen as the default case; in practice, this

  // doesn't matter unless someone is crazy enough to use something that

  // doesn't fold to a constant for the ordering.

  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);

  // Implemented as acquire, since it's the closest in LLVM.

  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),

              AcquireBB);

  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),

              AcquireBB);

  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),

              SeqCstBB);

  // Emit all the different atomics

  CGF.Builder.SetInsertPoint(MonotonicBB);

  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,

                    Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);

  CGF.Builder.CreateBr(ContBB);

  CGF.Builder.SetInsertPoint(AcquireBB);

  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,

                    llvm::AtomicOrdering::Acquire, Scope);

  CGF.Builder.CreateBr(ContBB);

  CGF.Builder.SetInsertPoint(SeqCstBB);

  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,

                    llvm::AtomicOrdering::SequentiallyConsistent, Scope);

  CGF.Builder.CreateBr(ContBB);

  CGF.Builder.SetInsertPoint(ContBB);

}

/// Duplicate the atomic min/max operation in conventional IR for the builtin

/// variants that return the new rather than the original value.

static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,

                                         AtomicExpr::AtomicOp Op,

                                         bool IsSigned,

                                         llvm::Value *OldVal,

                                         llvm::Value *RHS) {

  llvm::CmpInst::Predicate Pred;

  switch (Op) {

  default:

    llvm_unreachable("Unexpected min/max operation");

  case AtomicExpr::AO__atomic_max_fetch:

  case AtomicExpr::AO__scoped_atomic_max_fetch:

    Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;

    break;

  case AtomicExpr::AO__atomic_min_fetch:

  case AtomicExpr::AO__scoped_atomic_min_fetch:

    Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;

    break;

  }

  llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");

  return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");

}

static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,

                         Address Ptr, Address Val1, Address Val2,

                         llvm::Value *IsWeak, llvm::Value *FailureOrder,

                         uint64_t Size, llvm::AtomicOrdering Order,

                         llvm::SyncScope::ID Scope) {

  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;

  bool PostOpMinMax = false;

  unsigned PostOp = 0;

  switch (E->getOp()) {

  case AtomicExpr::AO__c11_atomic_init:

  case AtomicExpr::AO__opencl_atomic_init:

    llvm_unreachable("Already handled!");

  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:

  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:

  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:

    emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,

                                FailureOrder, Size, Order, Scope);

    return;

  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:

  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:

  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:

    emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,

                                FailureOrder, Size, Order, Scope);

    return;

  case AtomicExpr::AO__atomic_compare_exchange:

  case AtomicExpr::AO__atomic_compare_exchange_n:

  case AtomicExpr::AO__scoped_atomic_compare_exchange:

  case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {

    if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {

      emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,

                                  Val1, Val2, FailureOrder, Size, Order, Scope);

    } else {

      // Create all the relevant BB's

      llvm::BasicBlock *StrongBB =

          CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);

      llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);

      llvm::BasicBlock *ContBB =

          CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);

      llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);

      SI->addCase(CGF.Builder.getInt1(false), StrongBB);

      CGF.Builder.SetInsertPoint(StrongBB);

      emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,

                                  FailureOrder, Size, Order, Scope);

      CGF.Builder.CreateBr(ContBB);

      CGF.Builder.SetInsertPoint(WeakBB);

      emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,

                                  FailureOrder, Size, Order, Scope);

      CGF.Builder.CreateBr(ContBB);

      CGF.Builder.SetInsertPoint(ContBB);

    }

    return;

  }

  case AtomicExpr::AO__c11_atomic_load:

  case AtomicExpr::AO__opencl_atomic_load:

  case AtomicExpr::AO__hip_atomic_load:

  case AtomicExpr::AO__atomic_load_n:

  case AtomicExpr::AO__atomic_load:

  case AtomicExpr::AO__scoped_atomic_load_n:

  case AtomicExpr::AO__scoped_atomic_load: {

    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);

    Load->setAtomic(Order, Scope);

    Load->setVolatile(E->isVolatile());

    CGF.Builder.CreateStore(Load, Dest);

    return;

  }

  case AtomicExpr::AO__c11_atomic_store:

  case AtomicExpr::AO__opencl_atomic_store:

  case AtomicExpr::AO__hip_atomic_store:

  case AtomicExpr::AO__atomic_store:

  case AtomicExpr::AO__atomic_store_n:

  case AtomicExpr::AO__scoped_atomic_store:

  case AtomicExpr::AO__scoped_atomic_store_n: {

    llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);

    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);

    Store->setAtomic(Order, Scope);

    Store->setVolatile(E->isVolatile());

    return;

  }

  case AtomicExpr::AO__c11_atomic_exchange:

  case AtomicExpr::AO__hip_atomic_exchange:

  case AtomicExpr::AO__opencl_atomic_exchange:

  case AtomicExpr::AO__atomic_exchange_n:

  case AtomicExpr::AO__atomic_exchange:

  case AtomicExpr::AO__scoped_atomic_exchange_n:

  case AtomicExpr::AO__scoped_atomic_exchange:

    Op = llvm::AtomicRMWInst::Xchg;

    break;

  case AtomicExpr::AO__atomic_add_fetch:

  case AtomicExpr::AO__scoped_atomic_add_fetch:

    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd

                                                 : llvm::Instruction::Add;

    [[fallthrough]];

  case AtomicExpr::AO__c11_atomic_fetch_add:

  case AtomicExpr::AO__hip_atomic_fetch_add:

  case AtomicExpr::AO__opencl_atomic_fetch_add:

  case AtomicExpr::AO__atomic_fetch_add:

  case AtomicExpr::AO__scoped_atomic_fetch_add:

    Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd

                                             : llvm::AtomicRMWInst::Add;

    break;

  case AtomicExpr::AO__atomic_sub_fetch:

  case AtomicExpr::AO__scoped_atomic_sub_fetch:

    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub

                                                 : llvm::Instruction::Sub;

    [[fallthrough]];

  case AtomicExpr::AO__c11_atomic_fetch_sub:

  case AtomicExpr::AO__hip_atomic_fetch_sub:

  case AtomicExpr::AO__opencl_atomic_fetch_sub:

  case AtomicExpr::AO__atomic_fetch_sub:

  case AtomicExpr::AO__scoped_atomic_fetch_sub:

    Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub

                                             : llvm::AtomicRMWInst::Sub;

    break;

  case AtomicExpr::AO__atomic_min_fetch:

  case AtomicExpr::AO__scoped_atomic_min_fetch:

    PostOpMinMax = true;

    [[fallthrough]];

  case AtomicExpr::AO__c11_atomic_fetch_min:

  case AtomicExpr::AO__hip_atomic_fetch_min:

  case AtomicExpr::AO__opencl_atomic_fetch_min:

  case AtomicExpr::AO__atomic_fetch_min:

  case AtomicExpr::AO__scoped_atomic_fetch_min:

    Op = E->getValueType()->isFloatingType()

             ? llvm::AtomicRMWInst::FMin

             : (E->getValueType()->isSignedIntegerType()

                    ? llvm::AtomicRMWInst::Min

                    : llvm::AtomicRMWInst::UMin);

    break;

  case AtomicExpr::AO__atomic_max_fetch:

  case AtomicExpr::AO__scoped_atomic_max_fetch:

    PostOpMinMax = true;

    [[fallthrough]];

  case AtomicExpr::AO__c11_atomic_fetch_max:

  case AtomicExpr::AO__hip_atomic_fetch_max:

  case AtomicExpr::AO__opencl_atomic_fetch_max:

  case AtomicExpr::AO__atomic_fetch_max:

  case AtomicExpr::AO__scoped_atomic_fetch_max:

    Op = E->getValueType()->isFloatingType()

             ? llvm::AtomicRMWInst::FMax

             : (E->getValueType()->isSignedIntegerType()

                    ? llvm::AtomicRMWInst::Max

                    : llvm::AtomicRMWInst::UMax);

    break;

  case AtomicExpr::AO__atomic_and_fetch:

  case AtomicExpr::AO__scoped_atomic_and_fetch:

    PostOp = llvm::Instruction::And;

    [[fallthrough]];

  case AtomicExpr::AO__c11_atomic_fetch_and:

  case AtomicExpr::AO__hip_atomic_fetch_and:

  case AtomicExpr::AO__opencl_atomic_fetch_and:

  case AtomicExpr::AO__atomic_fetch_and:

  case AtomicExpr::AO__scoped_atomic_fetch_and:

    Op = llvm::AtomicRMWInst::And;

    break;

  case AtomicExpr::AO__atomic_or_fetch:

  case AtomicExpr::AO__scoped_atomic_or_fetch:

    PostOp = llvm::Instruction::Or;

    [[fallthrough]];

  case AtomicExpr::AO__c11_atomic_fetch_or:

  case AtomicExpr::AO__hip_atomic_fetch_or:

  case AtomicExpr::AO__opencl_atomic_fetch_or:

  case AtomicExpr::AO__atomic_fetch_or:

  case AtomicExpr::AO__scoped_atomic_fetch_or:

    Op = llvm::AtomicRMWInst::Or;

    break;

  case AtomicExpr::AO__atomic_xor_fetch:

  case AtomicExpr::AO__scoped_atomic_xor_fetch:

    PostOp = llvm::Instruction::Xor;

    [[fallthrough]];

  case AtomicExpr::AO__c11_atomic_fetch_xor:

  case AtomicExpr::AO__hip_atomic_fetch_xor:

  case AtomicExpr::AO__opencl_atomic_fetch_xor:

  case AtomicExpr::AO__atomic_fetch_xor:

  case AtomicExpr::AO__scoped_atomic_fetch_xor:

    Op = llvm::AtomicRMWInst::Xor;

    break;

  case AtomicExpr::AO__atomic_nand_fetch:

  case AtomicExpr::AO__scoped_atomic_nand_fetch:

    PostOp = llvm::Instruction::And; // the NOT is special cased below

    [[fallthrough]];

  case AtomicExpr::AO__c11_atomic_fetch_nand:

  case AtomicExpr::AO__atomic_fetch_nand:

  case AtomicExpr::AO__scoped_atomic_fetch_nand:

    Op = llvm::AtomicRMWInst::Nand;

    break;

  }

  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);

  llvm::AtomicRMWInst *RMWI =

      CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order, Scope);

  RMWI->setVolatile(E->isVolatile());

  // For __atomic_*_fetch operations, perform the operation again to

  // determine the value which was written.

  llvm::Value *Result = RMWI;

  if (PostOpMinMax)

    Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),

                                  E->getValueType()->isSignedIntegerType(),

                                  RMWI, LoadVal1);

  else if (PostOp)

    Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,

                                     LoadVal1);

  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||

      E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)

    Result = CGF.Builder.CreateNot(Result);

  CGF.Builder.CreateStore(Result, Dest);

}

// This function emits any expression (scalar, complex, or aggregate)

// into a temporary alloca.

static Address

EmitValToTemp(CodeGenFunction &CGF, Expr *E) {

  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");

  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),

                       /*Init*/ true);

  return DeclPtr;

}

static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,

                         Address Ptr, Address Val1, Address Val2,

                         llvm::Value *IsWeak, llvm::Value *FailureOrder,

                         uint64_t Size, llvm::AtomicOrdering Order,

                         llvm::Value *Scope) {

  auto ScopeModel = Expr->getScopeModel();

  // LLVM atomic instructions always have synch scope. If clang atomic

  // expression has no scope operand, use default LLVM synch scope.

  if (!ScopeModel) {

    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,

                 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));

    return;

  }

  // Handle constant scope.

  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {

    auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(

        CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),

        Order, CGF.CGM.getLLVMContext());

    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,

                 Order, SCID);

    return;

  }

  // Handle non-constant scope.

  auto &Builder = CGF.Builder;

  auto Scopes = ScopeModel->getRuntimeValues();

  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;

  for (auto S : Scopes)

    BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);

  llvm::BasicBlock *ContBB =

      CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);

  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);

  // If unsupported synch scope is encountered at run time, assume a fallback

  // synch scope value.

  auto FallBack = ScopeModel->getFallBackValue();

  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);

  for (auto S : Scopes) {

    auto *B = BB[S];

    if (S != FallBack)

      SI->addCase(Builder.getInt32(S), B);

    Builder.SetInsertPoint(B);

    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,

                 Order,

                 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),

                                                         ScopeModel->map(S),

                                                         Order,

                                                         CGF.getLLVMContext()));

    Builder.CreateBr(ContBB);

  }

  Builder.SetInsertPoint(ContBB);

}

static void

AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,

                  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,

                  SourceLocation Loc, CharUnits SizeInChars) {

  if (UseOptimizedLibcall) {

    // Load value and pass it to the function directly.

    CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);

    int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);

    ValTy =

        CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);

    llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);

    Address Ptr = Address(Val, ITy, Align);

    Val = CGF.EmitLoadOfScalar(Ptr, false,

                               CGF.getContext().getPointerType(ValTy),

                               Loc);

    // Coerce the value into an appropriately sized integer type.

    Args.add(RValue::get(Val), ValTy);

  } else {

    // Non-optimized functions always take a reference.

    Args.add(RValue::get(Val), CGF.getContext().VoidPtrTy);

  }

}

RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {

  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();

  QualType MemTy = AtomicTy;

  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())

    MemTy = AT->getValueType();

  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;

  Address Val1 = Address::invalid();

  Address Val2 = Address::invalid();

  Address Dest = Address::invalid();

  Address Ptr = EmitPointerWithAlignment(E->getPtr());

  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||

      E->getOp() == AtomicExpr::AO__opencl_atomic_init) {

    LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);

    EmitAtomicInit(E->getVal1(), lvalue);

    return RValue::get(nullptr);

  }

  auto TInfo = getContext().getTypeInfoInChars(AtomicTy);

  uint64_t Size = TInfo.Width.getQuantity();

  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();

  bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;

  bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;

  bool UseLibcall = Misaligned | Oversized;

  bool ShouldCastToIntPtrTy = true;

  CharUnits MaxInlineWidth =

      getContext().toCharUnitsFromBits(MaxInlineWidthInBits);

  DiagnosticsEngine &Diags = CGM.getDiags();

  if (Misaligned) {

    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)

        << (int)TInfo.Width.getQuantity()

        << (int)Ptr.getAlignment().getQuantity();

  }

  if (Oversized) {

    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)

        << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();

  }

  llvm::Value *Order = EmitScalarExpr(E->getOrder());

  llvm::Value *Scope =

      E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;

  switch (E->getOp()) {

  case AtomicExpr::AO__c11_atomic_init:

  case AtomicExpr::AO__opencl_atomic_init:

    llvm_unreachable("Already handled above with EmitAtomicInit!");

  case AtomicExpr::AO__atomic_load_n:

  case AtomicExpr::AO__scoped_atomic_load_n:

  case AtomicExpr::AO__c11_atomic_load:

  case AtomicExpr::AO__opencl_atomic_load:

  case AtomicExpr::AO__hip_atomic_load:

    break;

  case AtomicExpr::AO__atomic_load:

  case AtomicExpr::AO__scoped_atomic_load:

    Dest = EmitPointerWithAlignment(E->getVal1());

    break;

  case AtomicExpr::AO__atomic_store:

  case AtomicExpr::AO__scoped_atomic_store:

    Val1 = EmitPointerWithAlignment(E->getVal1());

    break;

  case AtomicExpr::AO__atomic_exchange:

  case AtomicExpr::AO__scoped_atomic_exchange:

    Val1 = EmitPointerWithAlignment(E->getVal1());

    Dest = EmitPointerWithAlignment(E->getVal2());

    break;

  case AtomicExpr::AO__atomic_compare_exchange:

  case AtomicExpr::AO__atomic_compare_exchange_n:

  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:

  case AtomicExpr::AO__c11_atomic_compare_exchange_strong: