//===--- Cuda.cpp - Cuda Tool and ToolChain Implementations -----*- C++ -*-===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

#include "Cuda.h"

#include "CommonArgs.h"

#include "clang/Basic/Cuda.h"

#include "clang/Config/config.h"

#include "clang/Driver/Compilation.h"

#include "clang/Driver/Distro.h"

#include "clang/Driver/Driver.h"

#include "clang/Driver/DriverDiagnostic.h"

#include "clang/Driver/InputInfo.h"

#include "clang/Driver/Options.h"

#include "llvm/ADT/StringExtras.h"

#include "llvm/Option/ArgList.h"

#include "llvm/Support/FileSystem.h"

#include "llvm/Support/FormatAdapters.h"

#include "llvm/Support/FormatVariadic.h"

#include "llvm/Support/Path.h"

#include "llvm/Support/Process.h"

#include "llvm/Support/Program.h"

#include "llvm/Support/VirtualFileSystem.h"

#include "llvm/TargetParser/Host.h"

#include "llvm/TargetParser/TargetParser.h"

#include <system_error>

using namespace clang::driver;

using namespace clang::driver::toolchains;

using namespace clang::driver::tools;

using namespace clang;

using namespace llvm::opt;

namespace {

CudaVersion getCudaVersion(uint32_t raw_version) {

  if (raw_version < 7050)

    return CudaVersion::CUDA_70;

  if (raw_version < 8000)

    return CudaVersion::CUDA_75;

  if (raw_version < 9000)

    return CudaVersion::CUDA_80;

  if (raw_version < 9010)

    return CudaVersion::CUDA_90;

  if (raw_version < 9020)

    return CudaVersion::CUDA_91;

  if (raw_version < 10000)

    return CudaVersion::CUDA_92;

  if (raw_version < 10010)

    return CudaVersion::CUDA_100;

  if (raw_version < 10020)

    return CudaVersion::CUDA_101;

  if (raw_version < 11000)

    return CudaVersion::CUDA_102;

  if (raw_version < 11010)

    return CudaVersion::CUDA_110;

  if (raw_version < 11020)

    return CudaVersion::CUDA_111;

  if (raw_version < 11030)

    return CudaVersion::CUDA_112;

  if (raw_version < 11040)

    return CudaVersion::CUDA_113;

  if (raw_version < 11050)

    return CudaVersion::CUDA_114;

  if (raw_version < 11060)

    return CudaVersion::CUDA_115;

  if (raw_version < 11070)

    return CudaVersion::CUDA_116;

  if (raw_version < 11080)

    return CudaVersion::CUDA_117;

  if (raw_version < 11090)

    return CudaVersion::CUDA_118;

  if (raw_version < 12010)

    return CudaVersion::CUDA_120;

  if (raw_version < 12020)

    return CudaVersion::CUDA_121;

  if (raw_version < 12030)

    return CudaVersion::CUDA_122;

  if (raw_version < 12040)

    return CudaVersion::CUDA_123;

  return CudaVersion::NEW;

}

CudaVersion parseCudaHFile(llvm::StringRef Input) {

  // Helper lambda which skips the words if the line starts with them or returns

  // std::nullopt otherwise.

  auto StartsWithWords =

      [](llvm::StringRef Line,

         const SmallVector<StringRef, 3> words) -> std::optional<StringRef> {

    for (StringRef word : words) {

      if (!Line.consume_front(word))

        return {};

      Line = Line.ltrim();

    }

    return Line;

  };

  Input = Input.ltrim();

  while (!Input.empty()) {

    if (auto Line =

            StartsWithWords(Input.ltrim(), {"#", "define", "CUDA_VERSION"})) {

      uint32_t RawVersion;

      Line->consumeInteger(10, RawVersion);

      return getCudaVersion(RawVersion);

    }

    // Find next non-empty line.

    Input = Input.drop_front(Input.find_first_of("\n\r")).ltrim();

  }

  return CudaVersion::UNKNOWN;

}

} // namespace

void CudaInstallationDetector::WarnIfUnsupportedVersion() {

  if (Version > CudaVersion::PARTIALLY_SUPPORTED) {

    std::string VersionString = CudaVersionToString(Version);

    if (!VersionString.empty())

      VersionString.insert(0, " ");

    D.Diag(diag::warn_drv_new_cuda_version)

        << VersionString

        << (CudaVersion::PARTIALLY_SUPPORTED != CudaVersion::FULLY_SUPPORTED)

        << CudaVersionToString(CudaVersion::PARTIALLY_SUPPORTED);

  } else if (Version > CudaVersion::FULLY_SUPPORTED)

    D.Diag(diag::warn_drv_partially_supported_cuda_version)

        << CudaVersionToString(Version);

}

CudaInstallationDetector::CudaInstallationDetector(

    const Driver &D, const llvm::Triple &HostTriple,

    const llvm::opt::ArgList &Args)

    : D(D) {

  struct Candidate {

    std::string Path;

    bool StrictChecking;

    Candidate(std::string Path, bool StrictChecking = false)

        : Path(Path), StrictChecking(StrictChecking) {}

  };

  SmallVector<Candidate, 4> Candidates;

  // In decreasing order so we prefer newer versions to older versions.

  std::initializer_list<const char *> Versions = {"8.0", "7.5", "7.0"};

  auto &FS = D.getVFS();

  if (Args.hasArg(clang::driver::options::OPT_cuda_path_EQ)) {

    Candidates.emplace_back(

        Args.getLastArgValue(clang::driver::options::OPT_cuda_path_EQ).str());

  } else if (HostTriple.isOSWindows()) {

    for (const char *Ver : Versions)

      Candidates.emplace_back(

          D.SysRoot + "/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v" +

          Ver);

  } else {

    if (!Args.hasArg(clang::driver::options::OPT_cuda_path_ignore_env)) {

      // Try to find ptxas binary. If the executable is located in a directory

      // called 'bin/', its parent directory might be a good guess for a valid

      // CUDA installation.

      // However, some distributions might installs 'ptxas' to /usr/bin. In that

      // case the candidate would be '/usr' which passes the following checks

      // because '/usr/include' exists as well. To avoid this case, we always

      // check for the directory potentially containing files for libdevice,

      // even if the user passes -nocudalib.

      if (llvm::ErrorOr<std::string> ptxas =

              llvm::sys::findProgramByName("ptxas")) {

        SmallString<256> ptxasAbsolutePath;

        llvm::sys::fs::real_path(*ptxas, ptxasAbsolutePath);

        StringRef ptxasDir = llvm::sys::path::parent_path(ptxasAbsolutePath);

        if (llvm::sys::path::filename(ptxasDir) == "bin")

          Candidates.emplace_back(

              std::string(llvm::sys::path::parent_path(ptxasDir)),

              /*StrictChecking=*/true);

      }

    }

    Candidates.emplace_back(D.SysRoot + "/usr/local/cuda");

    for (const char *Ver : Versions)

      Candidates.emplace_back(D.SysRoot + "/usr/local/cuda-" + Ver);

    Distro Dist(FS, llvm::Triple(llvm::sys::getProcessTriple()));

    if (Dist.IsDebian() || Dist.IsUbuntu())

      // Special case for Debian to have nvidia-cuda-toolkit work

      // out of the box. More info on http://bugs.debian.org/882505

      Candidates.emplace_back(D.SysRoot + "/usr/lib/cuda");

  }

  bool NoCudaLib = Args.hasArg(options::OPT_nogpulib);

  for (const auto &Candidate : Candidates) {

    InstallPath = Candidate.Path;

    if (InstallPath.empty() || !FS.exists(InstallPath))

      continue;

    BinPath = InstallPath + "/bin";

    IncludePath = InstallPath + "/include";

    LibDevicePath = InstallPath + "/nvvm/libdevice";

    if (!(FS.exists(IncludePath) && FS.exists(BinPath)))

      continue;

    bool CheckLibDevice = (!NoCudaLib || Candidate.StrictChecking);

    if (CheckLibDevice && !FS.exists(LibDevicePath))

      continue;

    Version = CudaVersion::UNKNOWN;

    if (auto CudaHFile = FS.getBufferForFile(InstallPath + "/include/cuda.h"))

      Version = parseCudaHFile((*CudaHFile)->getBuffer());

    // As the last resort, make an educated guess between CUDA-7.0, which had

    // old-style libdevice bitcode, and an unknown recent CUDA version.

    if (Version == CudaVersion::UNKNOWN) {

      Version = FS.exists(LibDevicePath + "/libdevice.10.bc")

                    ? CudaVersion::NEW

                    : CudaVersion::CUDA_70;

    }

    if (Version >= CudaVersion::CUDA_90) {

      // CUDA-9+ uses single libdevice file for all GPU variants.

      std::string FilePath = LibDevicePath + "/libdevice.10.bc";

      if (FS.exists(FilePath)) {

        for (int Arch = (int)CudaArch::SM_30, E = (int)CudaArch::LAST; Arch < E;

             ++Arch) {

          CudaArch GpuArch = static_cast<CudaArch>(Arch);

          if (!IsNVIDIAGpuArch(GpuArch))

            continue;

          std::string GpuArchName(CudaArchToString(GpuArch));

          LibDeviceMap[GpuArchName] = FilePath;

        }

      }

    } else {

      std::error_code EC;

      for (llvm::vfs::directory_iterator LI = FS.dir_begin(LibDevicePath, EC),

                                         LE;

           !EC && LI != LE; LI = LI.increment(EC)) {

        StringRef FilePath = LI->path();

        StringRef FileName = llvm::sys::path::filename(FilePath);

        // Process all bitcode filenames that look like

        // libdevice.compute_XX.YY.bc

        const StringRef LibDeviceName = "libdevice.";

        if (!(FileName.starts_with(LibDeviceName) && FileName.ends_with(".bc")))

          continue;

        StringRef GpuArch = FileName.slice(

            LibDeviceName.size(), FileName.find('.', LibDeviceName.size()));

        LibDeviceMap[GpuArch] = FilePath.str();

        // Insert map entries for specific devices with this compute

        // capability. NVCC's choice of the libdevice library version is

        // rather peculiar and depends on the CUDA version.

        if (GpuArch == "compute_20") {

          LibDeviceMap["sm_20"] = std::string(FilePath);

          LibDeviceMap["sm_21"] = std::string(FilePath);

          LibDeviceMap["sm_32"] = std::string(FilePath);

        } else if (GpuArch == "compute_30") {

          LibDeviceMap["sm_30"] = std::string(FilePath);

          if (Version < CudaVersion::CUDA_80) {

            LibDeviceMap["sm_50"] = std::string(FilePath);

            LibDeviceMap["sm_52"] = std::string(FilePath);

            LibDeviceMap["sm_53"] = std::string(FilePath);

          }

          LibDeviceMap["sm_60"] = std::string(FilePath);

          LibDeviceMap["sm_61"] = std::string(FilePath);

          LibDeviceMap["sm_62"] = std::string(FilePath);

        } else if (GpuArch == "compute_35") {

          LibDeviceMap["sm_35"] = std::string(FilePath);

          LibDeviceMap["sm_37"] = std::string(FilePath);

        } else if (GpuArch == "compute_50") {

          if (Version >= CudaVersion::CUDA_80) {

            LibDeviceMap["sm_50"] = std::string(FilePath);

            LibDeviceMap["sm_52"] = std::string(FilePath);

            LibDeviceMap["sm_53"] = std::string(FilePath);

          }

        }

      }

    }

    // Check that we have found at least one libdevice that we can link in if

    // -nocudalib hasn't been specified.

    if (LibDeviceMap.empty() && !NoCudaLib)

      continue;

    IsValid = true;

    break;

  }

}

void CudaInstallationDetector::AddCudaIncludeArgs(

    const ArgList &DriverArgs, ArgStringList &CC1Args) const {

  if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {

    // Add cuda_wrappers/* to our system include path.  This lets us wrap

    // standard library headers.

    SmallString<128> P(D.ResourceDir);

    llvm::sys::path::append(P, "include");

    llvm::sys::path::append(P, "cuda_wrappers");

    CC1Args.push_back("-internal-isystem");

    CC1Args.push_back(DriverArgs.MakeArgString(P));

  }

  if (DriverArgs.hasArg(options::OPT_nogpuinc))

    return;

  if (!isValid()) {

    D.Diag(diag::err_drv_no_cuda_installation);

    return;

  }

  CC1Args.push_back("-include");

  CC1Args.push_back("__clang_cuda_runtime_wrapper.h");

}

void CudaInstallationDetector::CheckCudaVersionSupportsArch(

    CudaArch Arch) const {

  if (Arch == CudaArch::UNKNOWN || Version == CudaVersion::UNKNOWN ||

      ArchsWithBadVersion[(int)Arch])

    return;

  auto MinVersion = MinVersionForCudaArch(Arch);

  auto MaxVersion = MaxVersionForCudaArch(Arch);

  if (Version < MinVersion || Version > MaxVersion) {

    ArchsWithBadVersion[(int)Arch] = true;

    D.Diag(diag::err_drv_cuda_version_unsupported)

        << CudaArchToString(Arch) << CudaVersionToString(MinVersion)

        << CudaVersionToString(MaxVersion) << InstallPath

        << CudaVersionToString(Version);

  }

}

void CudaInstallationDetector::print(raw_ostream &OS) const {

  if (isValid())

    OS << "Found CUDA installation: " << InstallPath << ", version "

       << CudaVersionToString(Version) << "\n";

}

namespace {

/// Debug info level for the NVPTX devices. We may need to emit different debug

/// info level for the host and for the device itselfi. This type controls

/// emission of the debug info for the devices. It either prohibits disable info

/// emission completely, or emits debug directives only, or emits same debug

/// info as for the host.

enum DeviceDebugInfoLevel {

  DisableDebugInfo,        /// Do not emit debug info for the devices.

  DebugDirectivesOnly,     /// Emit only debug directives.

  EmitSameDebugInfoAsHost, /// Use the same debug info level just like for the

                           /// host.

};

} // anonymous namespace

/// Define debug info level for the NVPTX devices. If the debug info for both

/// the host and device are disabled (-g0/-ggdb0 or no debug options at all). If

/// only debug directives are requested for the both host and device

/// (-gline-directvies-only), or the debug info only for the device is disabled

/// (optimization is on and --cuda-noopt-device-debug was not specified), the

/// debug directves only must be emitted for the device. Otherwise, use the same

/// debug info level just like for the host (with the limitations of only

/// supported DWARF2 standard).

static DeviceDebugInfoLevel mustEmitDebugInfo(const ArgList &Args) {

  const Arg *A = Args.getLastArg(options::OPT_O_Group);

  bool IsDebugEnabled = !A || A->getOption().matches(options::OPT_O0) ||

                        Args.hasFlag(options::OPT_cuda_noopt_device_debug,

                                     options::OPT_no_cuda_noopt_device_debug,

                                     /*Default=*/false);

  if (const Arg *A = Args.getLastArg(options::OPT_g_Group)) {

    const Option &Opt = A->getOption();

    if (Opt.matches(options::OPT_gN_Group)) {

      if (Opt.matches(options::OPT_g0) || Opt.matches(options::OPT_ggdb0))

        return DisableDebugInfo;

      if (Opt.matches(options::OPT_gline_directives_only))

        return DebugDirectivesOnly;

    }

    return IsDebugEnabled ? EmitSameDebugInfoAsHost : DebugDirectivesOnly;

  }

  return willEmitRemarks(Args) ? DebugDirectivesOnly : DisableDebugInfo;

}

void NVPTX::Assembler::ConstructJob(Compilation &C, const JobAction &JA,

                                    const InputInfo &Output,

                                    const InputInfoList &Inputs,

                                    const ArgList &Args,

                                    const char *LinkingOutput) const {

  const auto &TC =

      static_cast<const toolchains::NVPTXToolChain &>(getToolChain());

  assert(TC.getTriple().isNVPTX() && "Wrong platform");

  StringRef GPUArchName;

  // If this is a CUDA action we need to extract the device architecture

  // from the Job's associated architecture, otherwise use the -march=arch

  // option. This option may come from -Xopenmp-target flag or the default

  // value.

  if (JA.isDeviceOffloading(Action::OFK_Cuda)) {

    GPUArchName = JA.getOffloadingArch();

  } else {

    GPUArchName = Args.getLastArgValue(options::OPT_march_EQ);

    assert(!GPUArchName.empty() && "Must have an architecture passed in.");

  }

  // Obtain architecture from the action.

  CudaArch gpu_arch = StringToCudaArch(GPUArchName);

  assert(gpu_arch != CudaArch::UNKNOWN &&

         "Device action expected to have an architecture.");

  // Check that our installation's ptxas supports gpu_arch.

  if (!Args.hasArg(options::OPT_no_cuda_version_check)) {

    TC.CudaInstallation.CheckCudaVersionSupportsArch(gpu_arch);

  }

  ArgStringList CmdArgs;

  CmdArgs.push_back(TC.getTriple().isArch64Bit() ? "-m64" : "-m32");

  DeviceDebugInfoLevel DIKind = mustEmitDebugInfo(Args);

  if (DIKind == EmitSameDebugInfoAsHost) {

    // ptxas does not accept -g option if optimization is enabled, so

    // we ignore the compiler's -O* options if we want debug info.

    CmdArgs.push_back("-g");

    CmdArgs.push_back("--dont-merge-basicblocks");

    CmdArgs.push_back("--return-at-end");

  } else if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {

    // Map the -O we received to -O{0,1,2,3}.

    //

    // TODO: Perhaps we should map host -O2 to ptxas -O3. -O3 is ptxas's

    // default, so it may correspond more closely to the spirit of clang -O2.

    // -O3 seems like the least-bad option when -Osomething is specified to

    // clang but it isn't handled below.

    StringRef OOpt = "3";

    if (A->getOption().matches(options::OPT_O4) ||

        A->getOption().matches(options::OPT_Ofast))

      OOpt = "3";

    else if (A->getOption().matches(options::OPT_O0))

      OOpt = "0";

    else if (A->getOption().matches(options::OPT_O)) {

      // -Os, -Oz, and -O(anything else) map to -O2, for lack of better options.

      OOpt = llvm::StringSwitch<const char *>(A->getValue())

                 .Case("1", "1")

                 .Case("2", "2")

                 .Case("3", "3")

                 .Case("s", "2")

                 .Case("z", "2")

                 .Default("2");

    }

    CmdArgs.push_back(Args.MakeArgString(llvm::Twine("-O") + OOpt));

  } else {

    // If no -O was passed, pass -O0 to ptxas -- no opt flag should correspond

    // to no optimizations, but ptxas's default is -O3.

    CmdArgs.push_back("-O0");

  }

  if (DIKind == DebugDirectivesOnly)

    CmdArgs.push_back("-lineinfo");

  // Pass -v to ptxas if it was passed to the driver.

  if (Args.hasArg(options::OPT_v))

    CmdArgs.push_back("-v");

  CmdArgs.push_back("--gpu-name");

  CmdArgs.push_back(Args.MakeArgString(CudaArchToString(gpu_arch)));

  CmdArgs.push_back("--output-file");

  std::string OutputFileName = TC.getInputFilename(Output);

  // If we are invoking `nvlink` internally we need to output a `.cubin` file.

  // FIXME: This should hopefully be removed if NVIDIA updates their tooling.

  if (!C.getInputArgs().getLastArg(options::OPT_c)) {

    SmallString<256> Filename(Output.getFilename());

    llvm::sys::path::replace_extension(Filename, "cubin");

    OutputFileName = Filename.str();

  }

  if (Output.isFilename() && OutputFileName != Output.getFilename())

    C.addTempFile(Args.MakeArgString(OutputFileName));

  CmdArgs.push_back(Args.MakeArgString(OutputFileName));

  for (const auto &II : Inputs)

    CmdArgs.push_back(Args.MakeArgString(II.getFilename()));

  for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_ptxas))

    CmdArgs.push_back(Args.MakeArgString(A));

  bool Relocatable;

  if (JA.isOffloading(Action::OFK_OpenMP))

    // In OpenMP we need to generate relocatable code.

    Relocatable = Args.hasFlag(options::OPT_fopenmp_relocatable_target,

                               options::OPT_fnoopenmp_relocatable_target,

                               /*Default=*/true);

  else if (JA.isOffloading(Action::OFK_Cuda))

    // In CUDA we generate relocatable code by default.

    Relocatable = Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc,

                               /*Default=*/false);

  else

    // Otherwise, we are compiling directly and should create linkable output.

    Relocatable = true;

  if (Relocatable)

    CmdArgs.push_back("-c");

  const char *Exec;

  if (Arg *A = Args.getLastArg(options::OPT_ptxas_path_EQ))

    Exec = A->getValue();

  else

    Exec = Args.MakeArgString(TC.GetProgramPath("ptxas"));

  C.addCommand(std::make_unique<Command>(

      JA, *this,

      ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,

                          "--options-file"},

      Exec, CmdArgs, Inputs, Output));

}

static bool shouldIncludePTX(const ArgList &Args, const char *gpu_arch) {

  bool includePTX = true;

  for (Arg *A : Args) {

    if (!(A->getOption().matches(options::OPT_cuda_include_ptx_EQ) ||

          A->getOption().matches(options::OPT_no_cuda_include_ptx_EQ)))

      continue;

    A->claim();

    const StringRef ArchStr = A->getValue();

    if (ArchStr == "all" || ArchStr == gpu_arch) {

      includePTX = A->getOption().matches(options::OPT_cuda_include_ptx_EQ);

      continue;

    }

  }

  return includePTX;

}

// All inputs to this linker must be from CudaDeviceActions, as we need to look

// at the Inputs' Actions in order to figure out which GPU architecture they

// correspond to.

void NVPTX::FatBinary::ConstructJob(Compilation &C, const JobAction &JA,

                                    const InputInfo &Output,

                                    const InputInfoList &Inputs,

                                    const ArgList &Args,

                                    const char *LinkingOutput) const {

  const auto &TC =

      static_cast<const toolchains::CudaToolChain &>(getToolChain());

  assert(TC.getTriple().isNVPTX() && "Wrong platform");

  ArgStringList CmdArgs;

  if (TC.CudaInstallation.version() <= CudaVersion::CUDA_100)

    CmdArgs.push_back("--cuda");

  CmdArgs.push_back(TC.getTriple().isArch64Bit() ? "-64" : "-32");

  CmdArgs.push_back(Args.MakeArgString("--create"));

  CmdArgs.push_back(Args.MakeArgString(Output.getFilename()));

  if (mustEmitDebugInfo(Args) == EmitSameDebugInfoAsHost)

    CmdArgs.push_back("-g");

  for (const auto &II : Inputs) {

    auto *A = II.getAction();

    assert(A->getInputs().size() == 1 &&

           "Device offload action is expected to have a single input");

    const char *gpu_arch_str = A->getOffloadingArch();

    assert(gpu_arch_str &&

           "Device action expected to have associated a GPU architecture!");

    CudaArch gpu_arch = StringToCudaArch(gpu_arch_str);

    if (II.getType() == types::TY_PP_Asm &&

        !shouldIncludePTX(Args, gpu_arch_str))

      continue;

    // We need to pass an Arch of the form "sm_XX" for cubin files and

    // "compute_XX" for ptx.

    const char *Arch = (II.getType() == types::TY_PP_Asm)

                           ? CudaArchToVirtualArchString(gpu_arch)

                           : gpu_arch_str;

    CmdArgs.push_back(

        Args.MakeArgString(llvm::Twine("--image=profile=") + Arch +

                           ",file=" + getToolChain().getInputFilename(II)));

  }

  for (const auto &A : Args.getAllArgValues(options::OPT_Xcuda_fatbinary))

    CmdArgs.push_back(Args.MakeArgString(A));

  const char *Exec = Args.MakeArgString(TC.GetProgramPath("fatbinary"));

  C.addCommand(std::make_unique<Command>(

      JA, *this,

      ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,

                          "--options-file"},

      Exec, CmdArgs, Inputs, Output));

}

void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,

                                 const InputInfo &Output,

                                 const InputInfoList &Inputs,

                                 const ArgList &Args,

                                 const char *LinkingOutput) const {

  const auto &TC =

      static_cast<const toolchains::NVPTXToolChain &>(getToolChain());

  ArgStringList CmdArgs;

  assert(TC.getTriple().isNVPTX() && "Wrong platform");

  assert((Output.isFilename() || Output.isNothing()) && "Invalid output.");

  if (Output.isFilename()) {

    CmdArgs.push_back("-o");

    CmdArgs.push_back(Output.getFilename());

  }

  if (mustEmitDebugInfo(Args) == EmitSameDebugInfoAsHost)

    CmdArgs.push_back("-g");

  if (Args.hasArg(options::OPT_v))

    CmdArgs.push_back("-v");

  StringRef GPUArch = Args.getLastArgValue(options::OPT_march_EQ);

  assert(!GPUArch.empty() && "At least one GPU Arch required for nvlink.");

  CmdArgs.push_back("-arch");

  CmdArgs.push_back(Args.MakeArgString(GPUArch));

  // Add paths specified in LIBRARY_PATH environment variable as -L options.

  addDirectoryList(Args, CmdArgs, "-L", "LIBRARY_PATH");

  // Add paths for the default clang library path.

  SmallString<256> DefaultLibPath =

      llvm::sys::path::parent_path(TC.getDriver().Dir);

  llvm::sys::path::append(DefaultLibPath, CLANG_INSTALL_LIBDIR_BASENAME);

  CmdArgs.push_back(Args.MakeArgString(Twine("-L") + DefaultLibPath));

  for (const auto &II : Inputs) {

    if (II.getType() == types::TY_LLVM_IR || II.getType() == types::TY_LTO_IR ||

        II.getType() == types::TY_LTO_BC || II.getType() == types::TY_LLVM_BC) {

      C.getDriver().Diag(diag::err_drv_no_linker_llvm_support)

          << getToolChain().getTripleString();

      continue;

    }

    // Currently, we only pass the input files to the linker, we do not pass

    // any libraries that may be valid only for the host.

    if (!II.isFilename())

      continue;

    // The 'nvlink' application performs RDC-mode linking when given a '.o'

    // file and device linking when given a '.cubin' file. We always want to

    // perform device linking, so just rename any '.o' files.

    // FIXME: This should hopefully be removed if NVIDIA updates their tooling.

    auto InputFile = getToolChain().getInputFilename(II);

    if (llvm::sys::path::extension(InputFile) != ".cubin") {

      // If there are no actions above this one then this is direct input and we

      // can copy it. Otherwise the input is internal so a `.cubin` file should

      // exist.

      if (II.getAction() && II.getAction()->getInputs().size() == 0) {

        const char *CubinF =

            Args.MakeArgString(getToolChain().getDriver().GetTemporaryPath(

                llvm::sys::path::stem(InputFile), "cubin"));

        if (llvm::sys::fs::copy_file(InputFile, C.addTempFile(CubinF)))

          continue;

        CmdArgs.push_back(CubinF);

      } else {

        SmallString<256> Filename(InputFile);

        llvm::sys::path::replace_extension(Filename, "cubin");

        CmdArgs.push_back(Args.MakeArgString(Filename));

      }

    } else {

      CmdArgs.push_back(Args.MakeArgString(InputFile));

    }

  }

  C.addCommand(std::make_unique<Command>(

      JA, *this,

      ResponseFileSupport{ResponseFileSupport::RF_Full, llvm::sys::WEM_UTF8,

                          "--options-file"},

      Args.MakeArgString(getToolChain().GetProgramPath("nvlink")), CmdArgs,

      Inputs, Output));

}

void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple,

                                   const llvm::opt::ArgList &Args,

                                   std::vector<StringRef> &Features) {

  if (Args.hasArg(options::OPT_cuda_feature_EQ)) {

    StringRef PtxFeature =

        Args.getLastArgValue(options::OPT_cuda_feature_EQ, "+ptx42");

    Features.push_back(Args.MakeArgString(PtxFeature));

    return;

  }

  CudaInstallationDetector CudaInstallation(D, Triple, Args);

  // New CUDA versions often introduce new instructions that are only supported

  // by new PTX version, so we need to raise PTX level to enable them in NVPTX

  // back-end.

  const char *PtxFeature = nullptr;

  switch (CudaInstallation.version()) {

#define CASE_CUDA_VERSION(CUDA_VER, PTX_VER)                                   \

  case CudaVersion::CUDA_##CUDA_VER:                                           \

    PtxFeature = "+ptx" #PTX_VER;                                              \

    break;

    CASE_CUDA_VERSION(123, 83);

    CASE_CUDA_VERSION(122, 82);

    CASE_CUDA_VERSION(121, 81);

    CASE_CUDA_VERSION(120, 80);

    CASE_CUDA_VERSION(118, 78);

    CASE_CUDA_VERSION(117, 77);

    CASE_CUDA_VERSION(116, 76);

    CASE_CUDA_VERSION(115, 75);

    CASE_CUDA_VERSION(114, 74);

    CASE_CUDA_VERSION(113, 73);

    CASE_CUDA_VERSION(112, 72);

    CASE_CUDA_VERSION(111, 71);

    CASE_CUDA_VERSION(110, 70);

    CASE_CUDA_VERSION(102, 65);

    CASE_CUDA_VERSION(101, 64);

    CASE_CUDA_VERSION(100, 63);

    CASE_CUDA_VERSION(92, 61);

    CASE_CUDA_VERSION(91, 61);

    CASE_CUDA_VERSION(90, 60);

#undef CASE_CUDA_VERSION

  default:

    PtxFeature = "+ptx42";

  }

  Features.push_back(PtxFeature);

}

/// NVPTX toolchain. Our assembler is ptxas, and our linker is nvlink. This

/// operates as a stand-alone version of the NVPTX tools without the host

/// toolchain.

NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,

                               const llvm::Triple &HostTriple,

                               const ArgList &Args, bool Freestanding = false)

    : ToolChain(D, Triple, Args), CudaInstallation(D, HostTriple, Args),

      Freestanding(Freestanding) {

  if (CudaInstallation.isValid())

    getProgramPaths().push_back(std::string(CudaInstallation.getBinPath()));

  // Lookup binaries into the driver directory, this is used to

  // discover the 'nvptx-arch' executable.

  getProgramPaths().push_back(getDriver().Dir);

}

/// We only need the host triple to locate the CUDA binary utilities, use the

/// system's default triple if not provided.

NVPTXToolChain::NVPTXToolChain(const Driver &D, const llvm::Triple &Triple,

                               const ArgList &Args)

    : NVPTXToolChain(D, Triple, llvm::Triple(LLVM_HOST_TRIPLE), Args,

                     /*Freestanding=*/true) {}

llvm::opt::DerivedArgList *

NVPTXToolChain::TranslateArgs(const llvm::opt::DerivedArgList &Args,

                              StringRef BoundArch,

                              Action::OffloadKind DeviceOffloadKind) const {

  DerivedArgList *DAL =

      ToolChain::TranslateArgs(Args, BoundArch, DeviceOffloadKind);

  if (!DAL)

    DAL = new DerivedArgList(Args.getBaseArgs());

  const OptTable &Opts = getDriver().getOpts();

  for (Arg *A : Args)

    if (!llvm::is_contained(*DAL, A))

      DAL->append(A);

  if (!DAL->hasArg(options::OPT_march_EQ))

    DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_march_EQ),

                      CudaArchToString(CudaArch::CudaDefault));

  return DAL;

}

void NVPTXToolChain::addClangTargetOptions(

    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,

    Action::OffloadKind DeviceOffloadingKind) const {

  // If we are compiling with a standalone NVPTX toolchain we want to try to

  // mimic a standard environment as much as possible. So we enable lowering

  // ctor / dtor functions to global symbols that can be registered.

  if (Freestanding)

    CC1Args.append({"-mllvm", "--nvptx-lower-global-ctor-dtor"});

}

bool NVPTXToolChain::supportsDebugInfoOption(const llvm::opt::Arg *A) const {

  const Option &O = A->getOption();

  return (O.matches(options::OPT_gN_Group) &&

          !O.matches(options::OPT_gmodules)) ||

         O.matches(options::OPT_g_Flag) ||

         O.matches(options::OPT_ggdbN_Group) || O.matches(options::OPT_ggdb) ||

         O.matches(options::OPT_gdwarf) || O.matches(options::OPT_gdwarf_2) ||

         O.matches(options::OPT_gdwarf_3) || O.matches(options::OPT_gdwarf_4) ||

         O.matches(options::OPT_gdwarf_5) ||

         O.matches(options::OPT_gcolumn_info);

}

void NVPTXToolChain::adjustDebugInfoKind(

    llvm::codegenoptions::DebugInfoKind &DebugInfoKind,

    const ArgList &Args) const {

  switch (mustEmitDebugInfo(Args)) {

  case DisableDebugInfo:

    DebugInfoKind = llvm::codegenoptions::NoDebugInfo;

    break;

  case DebugDirectivesOnly:

    DebugInfoKind = llvm::codegenoptions::DebugDirectivesOnly;

    break;

  case EmitSameDebugInfoAsHost:

    // Use same debug info level as the host.

    break;

  }

}

/// CUDA toolchain.  Our assembler is ptxas, and our "linker" is fatbinary,

/// which isn't properly a linker but nonetheless performs the step of stitching

/// together object files from the assembler into a single blob.

CudaToolChain::CudaToolChain(const Driver &D, const llvm::Triple &Triple,

                             const ToolChain &HostTC, const ArgList &Args)

    : NVPTXToolChain(D, Triple, HostTC.getTriple(), Args), HostTC(HostTC) {}

void CudaToolChain::addClangTargetOptions(

    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,

    Action::OffloadKind DeviceOffloadingKind) const {

  HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind);

  StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ);

  assert(!GpuArch.empty() && "Must have an explicit GPU arch.");

  assert((DeviceOffloadingKind == Action::OFK_OpenMP ||

          DeviceOffloadingKind == Action::OFK_Cuda) &&

         "Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs.");

  if (DeviceOffloadingKind == Action::OFK_Cuda) {

    CC1Args.append(

        {"-fcuda-is-device", "-mllvm", "-enable-memcpyopt-without-libcalls"});

    // Unsized function arguments used for variadics were introduced in CUDA-9.0

    // We still do not support generating code that actually uses variadic

    // arguments yet, but we do need to allow parsing them as recent CUDA

    // headers rely on that. https://github.com/llvm/llvm-project/issues/58410

    if (CudaInstallation.version() >= CudaVersion::CUDA_90)

      CC1Args.push_back("-fcuda-allow-variadic-functions");

  }

  if (DriverArgs.hasArg(options::OPT_nogpulib))

    return;

  if (DeviceOffloadingKind == Action::OFK_OpenMP &&

      DriverArgs.hasArg(options::OPT_S))

    return;

  std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch);

  if (LibDeviceFile.empty()) {

    getDriver().Diag(diag::err_drv_no_cuda_libdevice) << GpuArch;

    return;

  }

  CC1Args.push_back("-mlink-builtin-bitcode");

  CC1Args.push_back(DriverArgs.MakeArgString(LibDeviceFile));

  clang::CudaVersion CudaInstallationVersion = CudaInstallation.version();

  if (DriverArgs.hasFlag(options::OPT_fcuda_short_ptr,

                         options::OPT_fno_cuda_short_ptr, false))

    CC1Args.append({"-mllvm", "--nvptx-short-ptr"});

  if (CudaInstallationVersion >= CudaVersion::UNKNOWN)

    CC1Args.push_back(

        DriverArgs.MakeArgString(Twine("-target-sdk-version=") +

                                 CudaVersionToString(CudaInstallationVersion)));

  if (DeviceOffloadingKind == Action::OFK_OpenMP) {

    if (CudaInstallationVersion < CudaVersion::CUDA_92) {

      getDriver().Diag(

          diag::err_drv_omp_offload_target_cuda_version_not_support)

          << CudaVersionToString(CudaInstallationVersion);

      return;

    }

    // Link the bitcode library late if we're using device LTO.

    if (getDriver().isUsingLTO(/* IsOffload */ true))

      return;

    addOpenMPDeviceRTL(getDriver(), DriverArgs, CC1Args, GpuArch.str(),

                       getTriple());

  }

}

llvm::DenormalMode CudaToolChain::getDefaultDenormalModeForType(

    const llvm::opt::ArgList &DriverArgs, const JobAction &JA,

    const llvm::fltSemantics *FPType) const {

  if (JA.getOffloadingDeviceKind() == Action::OFK_Cuda) {

    if (FPType && FPType == &llvm::APFloat::IEEEsingle() &&

        DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero,

                           options::OPT_fno_gpu_flush_denormals_to_zero, false))

      return llvm::DenormalMode::getPreserveSign();