//===--- PPMacroExpansion.cpp - Top level Macro Expansion -----------------===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// This file implements the top level handling of macro expansion for the

// preprocessor.

//

//===----------------------------------------------------------------------===//

#include "clang/Basic/AttributeCommonInfo.h"

#include "clang/Basic/Attributes.h"

#include "clang/Basic/Builtins.h"

#include "clang/Basic/FileManager.h"

#include "clang/Basic/IdentifierTable.h"

#include "clang/Basic/LLVM.h"

#include "clang/Basic/LangOptions.h"

#include "clang/Basic/ObjCRuntime.h"

#include "clang/Basic/SourceLocation.h"

#include "clang/Basic/TargetInfo.h"

#include "clang/Lex/CodeCompletionHandler.h"

#include "clang/Lex/DirectoryLookup.h"

#include "clang/Lex/ExternalPreprocessorSource.h"

#include "clang/Lex/HeaderSearch.h"

#include "clang/Lex/LexDiagnostic.h"

#include "clang/Lex/LiteralSupport.h"

#include "clang/Lex/MacroArgs.h"

#include "clang/Lex/MacroInfo.h"

#include "clang/Lex/Preprocessor.h"

#include "clang/Lex/PreprocessorLexer.h"

#include "clang/Lex/PreprocessorOptions.h"

#include "clang/Lex/Token.h"

#include "llvm/ADT/ArrayRef.h"

#include "llvm/ADT/DenseMap.h"

#include "llvm/ADT/DenseSet.h"

#include "llvm/ADT/FoldingSet.h"

#include "llvm/ADT/STLExtras.h"

#include "llvm/ADT/SmallString.h"

#include "llvm/ADT/SmallVector.h"

#include "llvm/ADT/StringRef.h"

#include "llvm/ADT/StringSwitch.h"

#include "llvm/Support/Casting.h"

#include "llvm/Support/ErrorHandling.h"

#include "llvm/Support/Format.h"

#include "llvm/Support/Path.h"

#include "llvm/Support/raw_ostream.h"

#include <algorithm>

#include <cassert>

#include <cstddef>

#include <cstring>

#include <ctime>

#include <optional>

#include <string>

#include <tuple>

#include <utility>

using namespace clang;

MacroDirective *

Preprocessor::getLocalMacroDirectiveHistory(const IdentifierInfo *II) const {

  if (!II->hadMacroDefinition())

    return nullptr;

  auto Pos = CurSubmoduleState->Macros.find(II);

  return Pos == CurSubmoduleState->Macros.end() ? nullptr

                                                : Pos->second.getLatest();

}

void Preprocessor::appendMacroDirective(IdentifierInfo *II, MacroDirective *MD){

  assert(MD && "MacroDirective should be non-zero!");

  assert(!MD->getPrevious() && "Already attached to a MacroDirective history.");

  MacroState &StoredMD = CurSubmoduleState->Macros[II];

  auto *OldMD = StoredMD.getLatest();

  MD->setPrevious(OldMD);

  StoredMD.setLatest(MD);

  StoredMD.overrideActiveModuleMacros(*this, II);

  if (needModuleMacros()) {

    // Track that we created a new macro directive, so we know we should

    // consider building a ModuleMacro for it when we get to the end of

    // the module.

    PendingModuleMacroNames.push_back(II);

  }

  // Set up the identifier as having associated macro history.

  II->setHasMacroDefinition(true);

  if (!MD->isDefined() && !LeafModuleMacros.contains(II))

    II->setHasMacroDefinition(false);

  if (II->isFromAST())

    II->setChangedSinceDeserialization();

}

void Preprocessor::setLoadedMacroDirective(IdentifierInfo *II,

                                           MacroDirective *ED,

                                           MacroDirective *MD) {

  // Normally, when a macro is defined, it goes through appendMacroDirective()

  // above, which chains a macro to previous defines, undefs, etc.

  // However, in a pch, the whole macro history up to the end of the pch is

  // stored, so ASTReader goes through this function instead.

  // However, built-in macros are already registered in the Preprocessor

  // ctor, and ASTWriter stops writing the macro chain at built-in macros,

  // so in that case the chain from the pch needs to be spliced to the existing

  // built-in.

  assert(II && MD);

  MacroState &StoredMD = CurSubmoduleState->Macros[II];

  if (auto *OldMD = StoredMD.getLatest()) {

    // shouldIgnoreMacro() in ASTWriter also stops at macros from the

    // predefines buffer in module builds. However, in module builds, modules

    // are loaded completely before predefines are processed, so StoredMD

    // will be nullptr for them when they're loaded. StoredMD should only be

    // non-nullptr for builtins read from a pch file.

    assert(OldMD->getMacroInfo()->isBuiltinMacro() &&

           "only built-ins should have an entry here");

    assert(!OldMD->getPrevious() && "builtin should only have a single entry");

    ED->setPrevious(OldMD);

    StoredMD.setLatest(MD);

  } else {

    StoredMD = MD;

  }

  // Setup the identifier as having associated macro history.

  II->setHasMacroDefinition(true);

  if (!MD->isDefined() && !LeafModuleMacros.contains(II))

    II->setHasMacroDefinition(false);

}

ModuleMacro *Preprocessor::addModuleMacro(Module *Mod, IdentifierInfo *II,

                                          MacroInfo *Macro,

                                          ArrayRef<ModuleMacro *> Overrides,

                                          bool &New) {

  llvm::FoldingSetNodeID ID;

  ModuleMacro::Profile(ID, Mod, II);

  void *InsertPos;

  if (auto *MM = ModuleMacros.FindNodeOrInsertPos(ID, InsertPos)) {

    New = false;

    return MM;

  }

  auto *MM = ModuleMacro::create(*this, Mod, II, Macro, Overrides);

  ModuleMacros.InsertNode(MM, InsertPos);

  // Each overridden macro is now overridden by one more macro.

  bool HidAny = false;

  for (auto *O : Overrides) {

    HidAny |= (O->NumOverriddenBy == 0);

    ++O->NumOverriddenBy;

  }

  // If we were the first overrider for any macro, it's no longer a leaf.

  auto &LeafMacros = LeafModuleMacros[II];

  if (HidAny) {

    llvm::erase_if(LeafMacros,

                   [](ModuleMacro *MM) { return MM->NumOverriddenBy != 0; });

  }

  // The new macro is always a leaf macro.

  LeafMacros.push_back(MM);

  // The identifier now has defined macros (that may or may not be visible).

  II->setHasMacroDefinition(true);

  New = true;

  return MM;

}

ModuleMacro *Preprocessor::getModuleMacro(Module *Mod,

                                          const IdentifierInfo *II) {

  llvm::FoldingSetNodeID ID;

  ModuleMacro::Profile(ID, Mod, II);

  void *InsertPos;

  return ModuleMacros.FindNodeOrInsertPos(ID, InsertPos);

}

void Preprocessor::updateModuleMacroInfo(const IdentifierInfo *II,

                                         ModuleMacroInfo &Info) {

  assert(Info.ActiveModuleMacrosGeneration !=

             CurSubmoduleState->VisibleModules.getGeneration() &&

         "don't need to update this macro name info");

  Info.ActiveModuleMacrosGeneration =

      CurSubmoduleState->VisibleModules.getGeneration();

  auto Leaf = LeafModuleMacros.find(II);

  if (Leaf == LeafModuleMacros.end()) {

    // No imported macros at all: nothing to do.

    return;

  }

  Info.ActiveModuleMacros.clear();

  // Every macro that's locally overridden is overridden by a visible macro.

  llvm::DenseMap<ModuleMacro *, int> NumHiddenOverrides;

  for (auto *O : Info.OverriddenMacros)

    NumHiddenOverrides[O] = -1;

  // Collect all macros that are not overridden by a visible macro.

  llvm::SmallVector<ModuleMacro *, 16> Worklist;

  for (auto *LeafMM : Leaf->second) {

    assert(LeafMM->getNumOverridingMacros() == 0 && "leaf macro overridden");

    if (NumHiddenOverrides.lookup(LeafMM) == 0)

      Worklist.push_back(LeafMM);

  }

  while (!Worklist.empty()) {

    auto *MM = Worklist.pop_back_val();

    if (CurSubmoduleState->VisibleModules.isVisible(MM->getOwningModule())) {

      // We only care about collecting definitions; undefinitions only act

      // to override other definitions.

      if (MM->getMacroInfo())

        Info.ActiveModuleMacros.push_back(MM);

    } else {

      for (auto *O : MM->overrides())

        if ((unsigned)++NumHiddenOverrides[O] == O->getNumOverridingMacros())

          Worklist.push_back(O);

    }

  }

  // Our reverse postorder walk found the macros in reverse order.

  std::reverse(Info.ActiveModuleMacros.begin(), Info.ActiveModuleMacros.end());

  // Determine whether the macro name is ambiguous.

  MacroInfo *MI = nullptr;

  bool IsSystemMacro = true;

  bool IsAmbiguous = false;

  if (auto *MD = Info.MD) {

    while (MD && isa<VisibilityMacroDirective>(MD))

      MD = MD->getPrevious();

    if (auto *DMD = dyn_cast_or_null<DefMacroDirective>(MD)) {

      MI = DMD->getInfo();

      IsSystemMacro &= SourceMgr.isInSystemHeader(DMD->getLocation());

    }

  }

  for (auto *Active : Info.ActiveModuleMacros) {

    auto *NewMI = Active->getMacroInfo();

    // Before marking the macro as ambiguous, check if this is a case where

    // both macros are in system headers. If so, we trust that the system

    // did not get it wrong. This also handles cases where Clang's own

    // headers have a different spelling of certain system macros:

    //   #define LONG_MAX __LONG_MAX__ (clang's limits.h)

    //   #define LONG_MAX 0x7fffffffffffffffL (system's limits.h)

    //

    // FIXME: Remove the defined-in-system-headers check. clang's limits.h

    // overrides the system limits.h's macros, so there's no conflict here.

    if (MI && NewMI != MI &&

        !MI->isIdenticalTo(*NewMI, *this, /*Syntactically=*/true))

      IsAmbiguous = true;

    IsSystemMacro &= Active->getOwningModule()->IsSystem ||

                     SourceMgr.isInSystemHeader(NewMI->getDefinitionLoc());

    MI = NewMI;

  }

  Info.IsAmbiguous = IsAmbiguous && !IsSystemMacro;

}

void Preprocessor::dumpMacroInfo(const IdentifierInfo *II) {

  ArrayRef<ModuleMacro*> Leaf;

  auto LeafIt = LeafModuleMacros.find(II);

  if (LeafIt != LeafModuleMacros.end())

    Leaf = LeafIt->second;

  const MacroState *State = nullptr;

  auto Pos = CurSubmoduleState->Macros.find(II);

  if (Pos != CurSubmoduleState->Macros.end())

    State = &Pos->second;

  llvm::errs() << "MacroState " << State << " " << II->getNameStart();

  if (State && State->isAmbiguous(*this, II))

    llvm::errs() << " ambiguous";

  if (State && !State->getOverriddenMacros().empty()) {

    llvm::errs() << " overrides";

    for (auto *O : State->getOverriddenMacros())

      llvm::errs() << " " << O->getOwningModule()->getFullModuleName();

  }

  llvm::errs() << "\n";

  // Dump local macro directives.

  for (auto *MD = State ? State->getLatest() : nullptr; MD;

       MD = MD->getPrevious()) {

    llvm::errs() << " ";

    MD->dump();

  }

  // Dump module macros.

  llvm::DenseSet<ModuleMacro*> Active;

  for (auto *MM :

       State ? State->getActiveModuleMacros(*this, II) : std::nullopt)

    Active.insert(MM);

  llvm::DenseSet<ModuleMacro*> Visited;

  llvm::SmallVector<ModuleMacro *, 16> Worklist(Leaf.begin(), Leaf.end());

  while (!Worklist.empty()) {

    auto *MM = Worklist.pop_back_val();

    llvm::errs() << " ModuleMacro " << MM << " "

                 << MM->getOwningModule()->getFullModuleName();

    if (!MM->getMacroInfo())

      llvm::errs() << " undef";

    if (Active.count(MM))

      llvm::errs() << " active";

    else if (!CurSubmoduleState->VisibleModules.isVisible(

                 MM->getOwningModule()))

      llvm::errs() << " hidden";

    else if (MM->getMacroInfo())

      llvm::errs() << " overridden";

    if (!MM->overrides().empty()) {

      llvm::errs() << " overrides";

      for (auto *O : MM->overrides()) {

        llvm::errs() << " " << O->getOwningModule()->getFullModuleName();

        if (Visited.insert(O).second)

          Worklist.push_back(O);

      }

    }

    llvm::errs() << "\n";

    if (auto *MI = MM->getMacroInfo()) {

      llvm::errs() << "  ";

      MI->dump();

      llvm::errs() << "\n";

    }

  }

}

/// RegisterBuiltinMacro - Register the specified identifier in the identifier

/// table and mark it as a builtin macro to be expanded.

static IdentifierInfo *RegisterBuiltinMacro(Preprocessor &PP, const char *Name){

  // Get the identifier.

  IdentifierInfo *Id = PP.getIdentifierInfo(Name);

  // Mark it as being a macro that is builtin.

  MacroInfo *MI = PP.AllocateMacroInfo(SourceLocation());

  MI->setIsBuiltinMacro();

  PP.appendDefMacroDirective(Id, MI);

  return Id;

}

/// RegisterBuiltinMacros - Register builtin macros, such as __LINE__ with the

/// identifier table.

void Preprocessor::RegisterBuiltinMacros() {

  Ident__LINE__ = RegisterBuiltinMacro(*this, "__LINE__");

  Ident__FILE__ = RegisterBuiltinMacro(*this, "__FILE__");

  Ident__DATE__ = RegisterBuiltinMacro(*this, "__DATE__");

  Ident__TIME__ = RegisterBuiltinMacro(*this, "__TIME__");

  Ident__COUNTER__ = RegisterBuiltinMacro(*this, "__COUNTER__");

  Ident_Pragma  = RegisterBuiltinMacro(*this, "_Pragma");

  Ident__FLT_EVAL_METHOD__ = RegisterBuiltinMacro(*this, "__FLT_EVAL_METHOD__");

  // C++ Standing Document Extensions.

  if (getLangOpts().CPlusPlus)

    Ident__has_cpp_attribute =

        RegisterBuiltinMacro(*this, "__has_cpp_attribute");

  else

    Ident__has_cpp_attribute = nullptr;

  // GCC Extensions.

  Ident__BASE_FILE__     = RegisterBuiltinMacro(*this, "__BASE_FILE__");

  Ident__INCLUDE_LEVEL__ = RegisterBuiltinMacro(*this, "__INCLUDE_LEVEL__");

  Ident__TIMESTAMP__     = RegisterBuiltinMacro(*this, "__TIMESTAMP__");

  // Microsoft Extensions.

  if (getLangOpts().MicrosoftExt) {

    Ident__identifier = RegisterBuiltinMacro(*this, "__identifier");

    Ident__pragma = RegisterBuiltinMacro(*this, "__pragma");

  } else {

    Ident__identifier = nullptr;

    Ident__pragma = nullptr;

  }

  // Clang Extensions.

  Ident__FILE_NAME__      = RegisterBuiltinMacro(*this, "__FILE_NAME__");

  Ident__has_feature      = RegisterBuiltinMacro(*this, "__has_feature");

  Ident__has_extension    = RegisterBuiltinMacro(*this, "__has_extension");

  Ident__has_builtin      = RegisterBuiltinMacro(*this, "__has_builtin");

  Ident__has_constexpr_builtin =

      RegisterBuiltinMacro(*this, "__has_constexpr_builtin");

  Ident__has_attribute    = RegisterBuiltinMacro(*this, "__has_attribute");

  if (!getLangOpts().CPlusPlus)

    Ident__has_c_attribute = RegisterBuiltinMacro(*this, "__has_c_attribute");

  else

    Ident__has_c_attribute = nullptr;

  Ident__has_declspec = RegisterBuiltinMacro(*this, "__has_declspec_attribute");

  Ident__has_include      = RegisterBuiltinMacro(*this, "__has_include");

  Ident__has_include_next = RegisterBuiltinMacro(*this, "__has_include_next");

  Ident__has_warning      = RegisterBuiltinMacro(*this, "__has_warning");

  Ident__is_identifier    = RegisterBuiltinMacro(*this, "__is_identifier");

  Ident__is_target_arch   = RegisterBuiltinMacro(*this, "__is_target_arch");

  Ident__is_target_vendor = RegisterBuiltinMacro(*this, "__is_target_vendor");

  Ident__is_target_os     = RegisterBuiltinMacro(*this, "__is_target_os");

  Ident__is_target_environment =

      RegisterBuiltinMacro(*this, "__is_target_environment");

  Ident__is_target_variant_os =

      RegisterBuiltinMacro(*this, "__is_target_variant_os");

  Ident__is_target_variant_environment =

      RegisterBuiltinMacro(*this, "__is_target_variant_environment");

  // Modules.

  Ident__building_module  = RegisterBuiltinMacro(*this, "__building_module");

  if (!getLangOpts().CurrentModule.empty())

    Ident__MODULE__ = RegisterBuiltinMacro(*this, "__MODULE__");

  else

    Ident__MODULE__ = nullptr;

}

/// isTrivialSingleTokenExpansion - Return true if MI, which has a single token

/// in its expansion, currently expands to that token literally.

static bool isTrivialSingleTokenExpansion(const MacroInfo *MI,

                                          const IdentifierInfo *MacroIdent,

                                          Preprocessor &PP) {

  IdentifierInfo *II = MI->getReplacementToken(0).getIdentifierInfo();

  // If the token isn't an identifier, it's always literally expanded.

  if (!II) return true;

  // If the information about this identifier is out of date, update it from

  // the external source.

  if (II->isOutOfDate())

    PP.getExternalSource()->updateOutOfDateIdentifier(*II);

  // If the identifier is a macro, and if that macro is enabled, it may be

  // expanded so it's not a trivial expansion.

  if (auto *ExpansionMI = PP.getMacroInfo(II))

    if (ExpansionMI->isEnabled() &&

        // Fast expanding "#define X X" is ok, because X would be disabled.

        II != MacroIdent)

      return false;

  // If this is an object-like macro invocation, it is safe to trivially expand

  // it.

  if (MI->isObjectLike()) return true;

  // If this is a function-like macro invocation, it's safe to trivially expand

  // as long as the identifier is not a macro argument.

  return !llvm::is_contained(MI->params(), II);

}

/// isNextPPTokenLParen - Determine whether the next preprocessor token to be

/// lexed is a '('.  If so, consume the token and return true, if not, this

/// method should have no observable side-effect on the lexed tokens.

bool Preprocessor::isNextPPTokenLParen() {

  // Do some quick tests for rejection cases.

  unsigned Val;

  if (CurLexer)

    Val = CurLexer->isNextPPTokenLParen();

  else

    Val = CurTokenLexer->isNextTokenLParen();

  if (Val == 2) {

    // We have run off the end.  If it's a source file we don't

    // examine enclosing ones (C99 5.1.1.2p4).  Otherwise walk up the

    // macro stack.

    if (CurPPLexer)

      return false;

    for (const IncludeStackInfo &Entry : llvm::reverse(IncludeMacroStack)) {

      if (Entry.TheLexer)

        Val = Entry.TheLexer->isNextPPTokenLParen();

      else

        Val = Entry.TheTokenLexer->isNextTokenLParen();

      if (Val != 2)

        break;

      // Ran off the end of a source file?

      if (Entry.ThePPLexer)

        return false;

    }

  }

  // Okay, if we know that the token is a '(', lex it and return.  Otherwise we

  // have found something that isn't a '(' or we found the end of the

  // translation unit.  In either case, return false.

  return Val == 1;

}

/// HandleMacroExpandedIdentifier - If an identifier token is read that is to be

/// expanded as a macro, handle it and return the next token as 'Identifier'.

bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,

                                                 const MacroDefinition &M) {

  emitMacroExpansionWarnings(Identifier);

  MacroInfo *MI = M.getMacroInfo();

  // If this is a macro expansion in the "#if !defined(x)" line for the file,

  // then the macro could expand to different things in other contexts, we need

  // to disable the optimization in this case.

  if (CurPPLexer) CurPPLexer->MIOpt.ExpandedMacro();

  // If this is a builtin macro, like __LINE__ or _Pragma, handle it specially.

  if (MI->isBuiltinMacro()) {

    if (Callbacks)

      Callbacks->MacroExpands(Identifier, M, Identifier.getLocation(),

                              /*Args=*/nullptr);

    ExpandBuiltinMacro(Identifier);

    return true;

  }

  /// Args - If this is a function-like macro expansion, this contains,

  /// for each macro argument, the list of tokens that were provided to the

  /// invocation.

  MacroArgs *Args = nullptr;

  // Remember where the end of the expansion occurred.  For an object-like

  // macro, this is the identifier.  For a function-like macro, this is the ')'.

  SourceLocation ExpansionEnd = Identifier.getLocation();

  // If this is a function-like macro, read the arguments.

  if (MI->isFunctionLike()) {

    // Remember that we are now parsing the arguments to a macro invocation.

    // Preprocessor directives used inside macro arguments are not portable, and

    // this enables the warning.

    InMacroArgs = true;

    ArgMacro = &Identifier;

    Args = ReadMacroCallArgumentList(Identifier, MI, ExpansionEnd);

    // Finished parsing args.

    InMacroArgs = false;

    ArgMacro = nullptr;

    // If there was an error parsing the arguments, bail out.

    if (!Args) return true;

    ++NumFnMacroExpanded;

  } else {

    ++NumMacroExpanded;

  }

  // Notice that this macro has been used.

  markMacroAsUsed(MI);

  // Remember where the token is expanded.

  SourceLocation ExpandLoc = Identifier.getLocation();

  SourceRange ExpansionRange(ExpandLoc, ExpansionEnd);

  if (Callbacks) {

    if (InMacroArgs) {

      // We can have macro expansion inside a conditional directive while

      // reading the function macro arguments. To ensure, in that case, that

      // MacroExpands callbacks still happen in source order, queue this

      // callback to have it happen after the function macro callback.

      DelayedMacroExpandsCallbacks.push_back(

          MacroExpandsInfo(Identifier, M, ExpansionRange));

    } else {

      Callbacks->MacroExpands(Identifier, M, ExpansionRange, Args);

      if (!DelayedMacroExpandsCallbacks.empty()) {

        for (const MacroExpandsInfo &Info : DelayedMacroExpandsCallbacks) {

          // FIXME: We lose macro args info with delayed callback.

          Callbacks->MacroExpands(Info.Tok, Info.MD, Info.Range,

                                  /*Args=*/nullptr);

        }

        DelayedMacroExpandsCallbacks.clear();

      }

    }

  }

  // If the macro definition is ambiguous, complain.

  if (M.isAmbiguous()) {

    Diag(Identifier, diag::warn_pp_ambiguous_macro)

      << Identifier.getIdentifierInfo();

    Diag(MI->getDefinitionLoc(), diag::note_pp_ambiguous_macro_chosen)

      << Identifier.getIdentifierInfo();

    M.forAllDefinitions([&](const MacroInfo *OtherMI) {

      if (OtherMI != MI)

        Diag(OtherMI->getDefinitionLoc(), diag::note_pp_ambiguous_macro_other)

          << Identifier.getIdentifierInfo();

    });

  }

  // If we started lexing a macro, enter the macro expansion body.

  // If this macro expands to no tokens, don't bother to push it onto the

  // expansion stack, only to take it right back off.

  if (MI->getNumTokens() == 0) {

    // No need for arg info.

    if (Args) Args->destroy(*this);

    // Propagate whitespace info as if we had pushed, then popped,

    // a macro context.

    Identifier.setFlag(Token::LeadingEmptyMacro);

    PropagateLineStartLeadingSpaceInfo(Identifier);

    ++NumFastMacroExpanded;

    return false;

  } else if (MI->getNumTokens() == 1 &&

             isTrivialSingleTokenExpansion(MI, Identifier.getIdentifierInfo(),

                                           *this)) {

    // Otherwise, if this macro expands into a single trivially-expanded

    // token: expand it now.  This handles common cases like

    // "#define VAL 42".

    // No need for arg info.

    if (Args) Args->destroy(*this);

    // Propagate the isAtStartOfLine/hasLeadingSpace markers of the macro

    // identifier to the expanded token.

    bool isAtStartOfLine = Identifier.isAtStartOfLine();

    bool hasLeadingSpace = Identifier.hasLeadingSpace();

    // Replace the result token.

    Identifier = MI->getReplacementToken(0);

    // Restore the StartOfLine/LeadingSpace markers.

    Identifier.setFlagValue(Token::StartOfLine , isAtStartOfLine);

    Identifier.setFlagValue(Token::LeadingSpace, hasLeadingSpace);

    // Update the tokens location to include both its expansion and physical

    // locations.

    SourceLocation Loc =

      SourceMgr.createExpansionLoc(Identifier.getLocation(), ExpandLoc,

                                   ExpansionEnd,Identifier.getLength());

    Identifier.setLocation(Loc);

    // If this is a disabled macro or #define X X, we must mark the result as

    // unexpandable.

    if (IdentifierInfo *NewII = Identifier.getIdentifierInfo()) {

      if (MacroInfo *NewMI = getMacroInfo(NewII))

        if (!NewMI->isEnabled() || NewMI == MI) {

          Identifier.setFlag(Token::DisableExpand);

          // Don't warn for "#define X X" like "#define bool bool" from

          // stdbool.h.

          if (NewMI != MI || MI->isFunctionLike())

            Diag(Identifier, diag::pp_disabled_macro_expansion);

        }

    }

    // Since this is not an identifier token, it can't be macro expanded, so

    // we're done.

    ++NumFastMacroExpanded;

    return true;

  }

  // Start expanding the macro.

  EnterMacro(Identifier, ExpansionEnd, MI, Args);

  return false;

}

enum Bracket {

  Brace,

  Paren

};

/// CheckMatchedBrackets - Returns true if the braces and parentheses in the

/// token vector are properly nested.

static bool CheckMatchedBrackets(const SmallVectorImpl<Token> &Tokens) {

  SmallVector<Bracket, 8> Brackets;

  for (SmallVectorImpl<Token>::const_iterator I = Tokens.begin(),

                                              E = Tokens.end();

       I != E; ++I) {

    if (I->is(tok::l_paren)) {

      Brackets.push_back(Paren);

    } else if (I->is(tok::r_paren)) {

      if (Brackets.empty() || Brackets.back() == Brace)

        return false;

      Brackets.pop_back();

    } else if (I->is(tok::l_brace)) {

      Brackets.push_back(Brace);

    } else if (I->is(tok::r_brace)) {

      if (Brackets.empty() || Brackets.back() == Paren)

        return false;

      Brackets.pop_back();

    }

  }

  return Brackets.empty();

}

/// GenerateNewArgTokens - Returns true if OldTokens can be converted to a new

/// vector of tokens in NewTokens.  The new number of arguments will be placed

/// in NumArgs and the ranges which need to surrounded in parentheses will be

/// in ParenHints.

/// Returns false if the token stream cannot be changed.  If this is because

/// of an initializer list starting a macro argument, the range of those

/// initializer lists will be place in InitLists.

static bool GenerateNewArgTokens(Preprocessor &PP,

                                 SmallVectorImpl<Token> &OldTokens,

                                 SmallVectorImpl<Token> &NewTokens,

                                 unsigned &NumArgs,

                                 SmallVectorImpl<SourceRange> &ParenHints,

                                 SmallVectorImpl<SourceRange> &InitLists) {

  if (!CheckMatchedBrackets(OldTokens))

    return false;

  // Once it is known that the brackets are matched, only a simple count of the

  // braces is needed.

  unsigned Braces = 0;

  // First token of a new macro argument.

  SmallVectorImpl<Token>::iterator ArgStartIterator = OldTokens.begin();

  // First closing brace in a new macro argument.  Used to generate

  // SourceRanges for InitLists.

  SmallVectorImpl<Token>::iterator ClosingBrace = OldTokens.end();

  NumArgs = 0;

  Token TempToken;

  // Set to true when a macro separator token is found inside a braced list.

  // If true, the fixed argument spans multiple old arguments and ParenHints

  // will be updated.

  bool FoundSeparatorToken = false;

  for (SmallVectorImpl<Token>::iterator I = OldTokens.begin(),

                                        E = OldTokens.end();

       I != E; ++I) {

    if (I->is(tok::l_brace)) {

      ++Braces;

    } else if (I->is(tok::r_brace)) {

      --Braces;

      if (Braces == 0 && ClosingBrace == E && FoundSeparatorToken)

        ClosingBrace = I;

    } else if (I->is(tok::eof)) {

      // EOF token is used to separate macro arguments

      if (Braces != 0) {

        // Assume comma separator is actually braced list separator and change

        // it back to a comma.

        FoundSeparatorToken = true;

        I->setKind(tok::comma);

        I->setLength(1);

      } else { // Braces == 0

        // Separator token still separates arguments.

        ++NumArgs;

        // If the argument starts with a brace, it can't be fixed with

        // parentheses.  A different diagnostic will be given.

        if (FoundSeparatorToken && ArgStartIterator->is(tok::l_brace)) {

          InitLists.push_back(

              SourceRange(ArgStartIterator->getLocation(),

                          PP.getLocForEndOfToken(ClosingBrace->getLocation())));

          ClosingBrace = E;

        }

        // Add left paren

        if (FoundSeparatorToken) {

          TempToken.startToken();

          TempToken.setKind(tok::l_paren);

          TempToken.setLocation(ArgStartIterator->getLocation());

          TempToken.setLength(0);

          NewTokens.push_back(TempToken);

        }

        // Copy over argument tokens

        NewTokens.insert(NewTokens.end(), ArgStartIterator, I);

        // Add right paren and store the paren locations in ParenHints

        if (FoundSeparatorToken) {

          SourceLocation Loc = PP.getLocForEndOfToken((I - 1)->getLocation());

          TempToken.startToken();

          TempToken.setKind(tok::r_paren);

          TempToken.setLocation(Loc);

          TempToken.setLength(0);

          NewTokens.push_back(TempToken);

          ParenHints.push_back(SourceRange(ArgStartIterator->getLocation(),

                                           Loc));

        }

        // Copy separator token

        NewTokens.push_back(*I);

        // Reset values

        ArgStartIterator = I + 1;

        FoundSeparatorToken = false;

      }

    }

  }

  return !ParenHints.empty() && InitLists.empty();

}

/// ReadFunctionLikeMacroArgs - After reading "MACRO" and knowing that the next

/// token is the '(' of the macro, this method is invoked to read all of the

/// actual arguments specified for the macro invocation.  This returns null on

/// error.

MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName,

                                                   MacroInfo *MI,

                                                   SourceLocation &MacroEnd) {

  // The number of fixed arguments to parse.

  unsigned NumFixedArgsLeft = MI->getNumParams();

  bool isVariadic = MI->isVariadic();

  // Outer loop, while there are more arguments, keep reading them.

  Token Tok;

  // Read arguments as unexpanded tokens.  This avoids issues, e.g., where

  // an argument value in a macro could expand to ',' or '(' or ')'.

  LexUnexpandedToken(Tok);

  assert(Tok.is(tok::l_paren) && "Error computing l-paren-ness?");

  // ArgTokens - Build up a list of tokens that make up each argument.  Each

  // argument is separated by an EOF token.  Use a SmallVector so we can avoid

  // heap allocations in the common case.

  SmallVector<Token, 64> ArgTokens;

  bool ContainsCodeCompletionTok = false;

  bool FoundElidedComma = false;

  SourceLocation TooManyArgsLoc;

  unsigned NumActuals = 0;

  while (Tok.isNot(tok::r_paren)) {

    if (ContainsCodeCompletionTok && Tok.isOneOf(tok::eof, tok::eod))

      break;

    assert(Tok.isOneOf(tok::l_paren, tok::comma) &&

           "only expect argument separators here");

    size_t ArgTokenStart = ArgTokens.size();

    SourceLocation ArgStartLoc = Tok.getLocation();

    // C99 6.10.3p11: Keep track of the number of l_parens we have seen.  Note

    // that we already consumed the first one.

    unsigned NumParens = 0;

    while (true) {

      // Read arguments as unexpanded tokens.  This avoids issues, e.g., where

      // an argument value in a macro could expand to ',' or '(' or ')'.

      LexUnexpandedToken(Tok);

      if (Tok.isOneOf(tok::eof, tok::eod)) { // "#if f(<eof>" & "#if f(\n"

        if (!ContainsCodeCompletionTok) {

          Diag(MacroName, diag::err_unterm_macro_invoc);

          Diag(MI->getDefinitionLoc(), diag::note_macro_here)

            << MacroName.getIdentifierInfo();

          // Do not lose the EOF/EOD.  Return it to the client.

          MacroName = Tok;

          return nullptr;

        }

        // Do not lose the EOF/EOD.

        auto Toks = std::make_unique<Token[]>(1);

        Toks[0] = Tok;

        EnterTokenStream(std::move(Toks), 1, true, /*IsReinject*/ false);

        break;

      } else if (Tok.is(tok::r_paren)) {

        // If we found the ) token, the macro arg list is done.

        if (NumParens-- == 0) {

          MacroEnd = Tok.getLocation();

          if (!ArgTokens.empty() &&

              ArgTokens.back().commaAfterElided()) {

            FoundElidedComma = true;

          }

          break;

        }

      } else if (Tok.is(tok::l_paren)) {

        ++NumParens;

      } else if (Tok.is(tok::comma)) {

        // In Microsoft-compatibility mode, single commas from nested macro

        // expansions should not be considered as argument separators. We test

        // for this with the IgnoredComma token flag.

        if (Tok.getFlags() & Token::IgnoredComma) {

          // However, in MSVC's preprocessor, subsequent expansions do treat

          // these commas as argument separators. This leads to a common

          // workaround used in macros that need to work in both MSVC and

          // compliant preprocessors. Therefore, the IgnoredComma flag can only

          // apply once to any given token.

          Tok.clearFlag(Token::IgnoredComma);

        } else if (NumParens == 0) {

          // Comma ends this argument if there are more fixed arguments

          // expected. However, if this is a variadic macro, and this is part of

          // the variadic part, then the comma is just an argument token.

          if (!isVariadic)

            break;

          if (NumFixedArgsLeft > 1)

            break;

        }

      } else if (Tok.is(tok::comment) && !KeepMacroComments) {

        // If this is a comment token in the argument list and we're just in

        // -C mode (not -CC mode), discard the comment.

        continue;

      } else if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {

        // Reading macro arguments can cause macros that we are currently

        // expanding from to be popped off the expansion stack.  Doing so causes

        // them to be reenabled for expansion.  Here we record whether any

        // identifiers we lex as macro arguments correspond to disabled macros.

        // If so, we mark the token as noexpand.  This is a subtle aspect of

        // C99 6.10.3.4p2.

        if (MacroInfo *MI = getMacroInfo(Tok.getIdentifierInfo()))

          if (!MI->isEnabled())

            Tok.setFlag(Token::DisableExpand);

      } else if (Tok.is(tok::code_completion)) {

        ContainsCodeCompletionTok = true;

        if (CodeComplete)

          CodeComplete->CodeCompleteMacroArgument(MacroName.getIdentifierInfo(),

                                                  MI, NumActuals);

        // Don't mark that we reached the code-completion point because the

        // parser is going to handle the token and there will be another

        // code-completion callback.

      }

      ArgTokens.push_back(Tok);

    }

    // If this was an empty argument list foo(), don't add this as an empty

    // argument.

    if (ArgTokens.empty() && Tok.getKind() == tok::r_paren)

      break;

    // If this is not a variadic macro, and too many args were specified, emit

    // an error.

    if (!isVariadic && NumFixedArgsLeft == 0 && TooManyArgsLoc.isInvalid()) {

      if (ArgTokens.size() != ArgTokenStart)

        TooManyArgsLoc = ArgTokens[ArgTokenStart].getLocation();

      else

        TooManyArgsLoc = ArgStartLoc;

    }

    // Empty arguments are standard in C99 and C++0x, and are supported as an

    // extension in other modes.

    if (ArgTokens.size() == ArgTokenStart && !getLangOpts().C99)

      Diag(Tok, getLangOpts().CPlusPlus11

                    ? diag::warn_cxx98_compat_empty_fnmacro_arg

                    : diag::ext_empty_fnmacro_arg);

    // Add a marker EOF token to the end of the token list for this argument.

    Token EOFTok;

    EOFTok.startToken();

    EOFTok.setKind(tok::eof);

    EOFTok.setLocation(Tok.getLocation());

    EOFTok.setLength(0);

    ArgTokens.push_back(EOFTok);

    ++NumActuals;

    if (!ContainsCodeCompletionTok && NumFixedArgsLeft != 0)

      --NumFixedArgsLeft;

  }

  // Okay, we either found the r_paren.  Check to see if we parsed too few

  // arguments.

  unsigned MinArgsExpected = MI->getNumParams();

  // If this is not a variadic macro, and too many args were specified, emit

  // an error.

  if (!isVariadic && NumActuals > MinArgsExpected &&

      !ContainsCodeCompletionTok) {

    // Emit the diagnostic at the macro name in case there is a missing ).