/src/llvm-project/llvm/lib/Analysis/InlineOrder.cpp

Source (jump to first uncovered line)
//===- InlineOrder.cpp - Inlining order abstraction -*- C++ ---*-----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "llvm/Analysis/InlineOrder.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/CommandLine.h"

using namespace llvm;

#define DEBUG_TYPE "inline-order"

enum class InlinePriorityMode : int { Size, Cost, CostBenefit, ML };

static cl::opt<InlinePriorityMode> UseInlinePriority(
    "inline-priority-mode", cl::init(InlinePriorityMode::Size), cl::Hidden,
    cl::desc("Choose the priority mode to use in module inline"),
    cl::values(clEnumValN(InlinePriorityMode::Size, "size",
                          "Use callee size priority."),
               clEnumValN(InlinePriorityMode::Cost, "cost",
                          "Use inline cost priority."),
               clEnumValN(InlinePriorityMode::CostBenefit, "cost-benefit",
                          "Use cost-benefit ratio."),
               clEnumValN(InlinePriorityMode::ML, "ml", "Use ML.")));

static cl::opt<int> ModuleInlinerTopPriorityThreshold(
    "module-inliner-top-priority-threshold", cl::Hidden, cl::init(0),
    cl::desc("The cost threshold for call sites that get inlined without the "
             "cost-benefit analysis"));

namespace {

llvm::InlineCost getInlineCostWrapper(CallBase &CB,
                                      FunctionAnalysisManager &FAM,
                                      const InlineParams &Params) {
  Function &Caller = *CB.getCaller();
  ProfileSummaryInfo *PSI =
      FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)
          .getCachedResult<ProfileSummaryAnalysis>(
              *CB.getParent()->getParent()->getParent());

  auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
  auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
    return FAM.getResult<AssumptionAnalysis>(F);
  };
  auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
    return FAM.getResult<BlockFrequencyAnalysis>(F);
  };
  auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
    return FAM.getResult<TargetLibraryAnalysis>(F);
  };

  Function &Callee = *CB.getCalledFunction();
  auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
  bool RemarksEnabled =
      Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
          DEBUG_TYPE);
  return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
                       GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
}

class SizePriority {
public:
  SizePriority() = default;
  SizePriority(const CallBase *CB, FunctionAnalysisManager &,
               const InlineParams &) {
    Function *Callee = CB->getCalledFunction();
    Size = Callee->getInstructionCount();
  }

  static bool isMoreDesirable(const SizePriority &P1, const SizePriority &P2) {
    return P1.Size < P2.Size;
  }

private:
  unsigned Size = UINT_MAX;
};

class CostPriority {
public:
  CostPriority() = default;
  CostPriority(const CallBase *CB, FunctionAnalysisManager &FAM,
               const InlineParams &Params) {
    auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);
    if (IC.isVariable())
      Cost = IC.getCost();
    else
      Cost = IC.isNever() ? INT_MAX : INT_MIN;
  }

  static bool isMoreDesirable(const CostPriority &P1, const CostPriority &P2) {
    return P1.Cost < P2.Cost;
  }

private:
  int Cost = INT_MAX;
};

class CostBenefitPriority {
public:
  CostBenefitPriority() = default;
  CostBenefitPriority(const CallBase *CB, FunctionAnalysisManager &FAM,
                      const InlineParams &Params) {
    auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);
    Cost = IC.getCost();
    StaticBonusApplied = IC.getStaticBonusApplied();
    CostBenefit = IC.getCostBenefit();
  }

  static bool isMoreDesirable(const CostBenefitPriority &P1,
                              const CostBenefitPriority &P2) {
    // We prioritize call sites in the dictionary order of the following
    // priorities:
    //
    // 1. Those call sites that are expected to reduce the caller size when
    //    inlined.  Within them, we prioritize those call sites with bigger
    //    reduction.
    //
    // 2. Those call sites that have gone through the cost-benefit analysis.
    //    Currently, they are limited to hot call sites.  Within them, we
    //    prioritize those call sites with higher benefit-to-cost ratios.
    //
    // 3. Remaining call sites are prioritized according to their costs.

    // We add back StaticBonusApplied to determine whether we expect the caller
    // to shrink (even if we don't delete the callee).
    bool P1ReducesCallerSize =
        P1.Cost + P1.StaticBonusApplied < ModuleInlinerTopPriorityThreshold;
    bool P2ReducesCallerSize =
        P2.Cost + P2.StaticBonusApplied < ModuleInlinerTopPriorityThreshold;
    if (P1ReducesCallerSize || P2ReducesCallerSize) {
      // If one reduces the caller size while the other doesn't, then return
      // true iff P1 reduces the caller size.
      if (P1ReducesCallerSize != P2ReducesCallerSize)
        return P1ReducesCallerSize;

      // If they both reduce the caller size, pick the one with the smaller
      // cost.
      return P1.Cost < P2.Cost;
    }

    bool P1HasCB = P1.CostBenefit.has_value();
    bool P2HasCB = P2.CostBenefit.has_value();
    if (P1HasCB || P2HasCB) {
      // If one has undergone the cost-benefit analysis while the other hasn't,
      // then return true iff P1 has.
      if (P1HasCB != P2HasCB)
        return P1HasCB;

      // If they have undergone the cost-benefit analysis, then pick the one
      // with a higher benefit-to-cost ratio.
      APInt LHS = P1.CostBenefit->getBenefit() * P2.CostBenefit->getCost();
      APInt RHS = P2.CostBenefit->getBenefit() * P1.CostBenefit->getCost();
      return LHS.ugt(RHS);
    }

    // Remaining call sites are ordered according to their costs.
    return P1.Cost < P2.Cost;
  }

private:
  int Cost = INT_MAX;
  int StaticBonusApplied = 0;
  std::optional<CostBenefitPair> CostBenefit;
};

class MLPriority {
public:
  MLPriority() = default;
  MLPriority(const CallBase *CB, FunctionAnalysisManager &FAM,
             const InlineParams &Params) {
    auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);
    if (IC.isVariable())
      Cost = IC.getCost();
    else
      Cost = IC.isNever() ? INT_MAX : INT_MIN;
  }

  static bool isMoreDesirable(const MLPriority &P1, const MLPriority &P2) {
    return P1.Cost < P2.Cost;
  }

private:
  int Cost = INT_MAX;
};

template <typename PriorityT>
class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
  using T = std::pair<CallBase *, int>;

  bool hasLowerPriority(const CallBase *L, const CallBase *R) const {
    const auto I1 = Priorities.find(L);
    const auto I2 = Priorities.find(R);
    assert(I1 != Priorities.end() && I2 != Priorities.end());
    return PriorityT::isMoreDesirable(I2->second, I1->second);
  }

  bool updateAndCheckDecreased(const CallBase *CB) {
    auto It = Priorities.find(CB);
    const auto OldPriority = It->second;
    It->second = PriorityT(CB, FAM, Params);
    const auto NewPriority = It->second;
    return PriorityT::isMoreDesirable(OldPriority, NewPriority);
  }

  // A call site could become less desirable for inlining because of the size
  // growth from prior inlining into the callee. This method is used to lazily
  // update the desirability of a call site if it's decreasing. It is only
  // called on pop(), not every time the desirability changes. When the
  // desirability of the front call site decreases, an updated one would be
  // pushed right back into the heap. For simplicity, those cases where the
  // desirability of a call site increases are ignored here.
  void pop_heap_adjust() {
    std::pop_heap(Heap.begin(), Heap.end(), isLess);
    while (updateAndCheckDecreased(Heap.back())) {
      std::push_heap(Heap.begin(), Heap.end(), isLess);
      std::pop_heap(Heap.begin(), Heap.end(), isLess);
    }
  }

public:
  PriorityInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params)
      : FAM(FAM), Params(Params) {
    isLess = [&](const CallBase *L, const CallBase *R) {
      return hasLowerPriority(L, R);
    };
  }

  size_t size() override { return Heap.size(); }

  void push(const T &Elt) override {
    CallBase *CB = Elt.first;
    const int InlineHistoryID = Elt.second;

    Heap.push_back(CB);
    Priorities[CB] = PriorityT(CB, FAM, Params);
    std::push_heap(Heap.begin(), Heap.end(), isLess);
    InlineHistoryMap[CB] = InlineHistoryID;
  }

  T pop() override {
    assert(size() > 0);
    pop_heap_adjust();

    CallBase *CB = Heap.pop_back_val();
    T Result = std::make_pair(CB, InlineHistoryMap[CB]);
    InlineHistoryMap.erase(CB);
    return Result;
  }

  void erase_if(function_ref<bool(T)> Pred) override {
    auto PredWrapper = [=](CallBase *CB) -> bool {
      return Pred(std::make_pair(CB, 0));
    };
    llvm::erase_if(Heap, PredWrapper);
    std::make_heap(Heap.begin(), Heap.end(), isLess);
  }

private:
  SmallVector<CallBase *, 16> Heap;
  std::function<bool(const CallBase *L, const CallBase *R)> isLess;
  DenseMap<CallBase *, int> InlineHistoryMap;
  DenseMap<const CallBase *, PriorityT> Priorities;
  FunctionAnalysisManager &FAM;
  const InlineParams &Params;
};

} // namespace

AnalysisKey llvm::PluginInlineOrderAnalysis::Key;
bool llvm::PluginInlineOrderAnalysis::HasBeenRegistered;

std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
llvm::getDefaultInlineOrder(FunctionAnalysisManager &FAM,
                            const InlineParams &Params,
                            ModuleAnalysisManager &MAM, Module &M) {
  switch (UseInlinePriority) {
  case InlinePriorityMode::Size:
    LLVM_DEBUG(dbgs() << "    Current used priority: Size priority ---- \n");
    return std::make_unique<PriorityInlineOrder<SizePriority>>(FAM, Params);

  case InlinePriorityMode::Cost:
    LLVM_DEBUG(dbgs() << "    Current used priority: Cost priority ---- \n");
    return std::make_unique<PriorityInlineOrder<CostPriority>>(FAM, Params);

  case InlinePriorityMode::CostBenefit:
    LLVM_DEBUG(
        dbgs() << "    Current used priority: cost-benefit priority ---- \n");
    return std::make_unique<PriorityInlineOrder<CostBenefitPriority>>(FAM,
                                                                      Params);
  case InlinePriorityMode::ML:
    LLVM_DEBUG(dbgs() << "    Current used priority: ML priority ---- \n");
    return std::make_unique<PriorityInlineOrder<MLPriority>>(FAM, Params);
  }
  return nullptr;
}

std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
llvm::getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
                     ModuleAnalysisManager &MAM, Module &M) {
  if (llvm::PluginInlineOrderAnalysis::isRegistered()) {
    LLVM_DEBUG(dbgs() << "    Current used priority: plugin ---- \n");
    return MAM.getResult<PluginInlineOrderAnalysis>(M).Factory(FAM, Params, MAM,
                                                               M);
  }
  return getDefaultInlineOrder(FAM, Params, MAM, M);
}

Coverage Report

Created: 2024-01-17 10:31

Line	Count	Source (jump to first uncovered line)
1		//===- InlineOrder.cpp - Inlining order abstraction -- C++ --------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8
9		#include "llvm/Analysis/InlineOrder.h"
10		#include "llvm/Analysis/AssumptionCache.h"
11		#include "llvm/Analysis/BlockFrequencyInfo.h"
12		#include "llvm/Analysis/GlobalsModRef.h"
13		#include "llvm/Analysis/InlineAdvisor.h"
14		#include "llvm/Analysis/InlineCost.h"
15		#include "llvm/Analysis/OptimizationRemarkEmitter.h"
16		#include "llvm/Analysis/ProfileSummaryInfo.h"
17		#include "llvm/Analysis/TargetLibraryInfo.h"
18		#include "llvm/Analysis/TargetTransformInfo.h"
19		#include "llvm/Support/CommandLine.h"
20
21		using namespace llvm;
22
23	0	#define DEBUG_TYPE "inline-order"
24
25		enum class InlinePriorityMode : int { Size, Cost, CostBenefit, ML };
26
27		static cl::opt<InlinePriorityMode> UseInlinePriority(
28		"inline-priority-mode", cl::init(InlinePriorityMode::Size), cl::Hidden,
29		cl::desc("Choose the priority mode to use in module inline"),
30		cl::values(clEnumValN(InlinePriorityMode::Size, "size",
31		"Use callee size priority."),
32		clEnumValN(InlinePriorityMode::Cost, "cost",
33		"Use inline cost priority."),
34		clEnumValN(InlinePriorityMode::CostBenefit, "cost-benefit",
35		"Use cost-benefit ratio."),
36		clEnumValN(InlinePriorityMode::ML, "ml", "Use ML.")));
37
38		static cl::opt<int> ModuleInlinerTopPriorityThreshold(
39		"module-inliner-top-priority-threshold", cl::Hidden, cl::init(0),
40		cl::desc("The cost threshold for call sites that get inlined without the "
41		"cost-benefit analysis"));
42
43		namespace {
44
45		llvm::InlineCost getInlineCostWrapper(CallBase &CB,
46		FunctionAnalysisManager &FAM,
47	0	const InlineParams &Params) {
48	0	Function &Caller = *CB.getCaller();
49	0	ProfileSummaryInfo *PSI =
50	0	FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)
51	0	.getCachedResult<ProfileSummaryAnalysis>(
52	0	*CB.getParent()->getParent()->getParent());
53
54	0	auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
55	0	auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
56	0	return FAM.getResult<AssumptionAnalysis>(F);
57	0	};
58	0	auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
59	0	return FAM.getResult<BlockFrequencyAnalysis>(F);
60	0	};
61	0	auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
62	0	return FAM.getResult<TargetLibraryAnalysis>(F);
63	0	};
64
65	0	Function &Callee = *CB.getCalledFunction();
66	0	auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
67	0	bool RemarksEnabled =
68	0	Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
69	0	DEBUG_TYPE);
70	0	return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
71	0	GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
72	0	}
73
74		class SizePriority {
75		public:
76	0	SizePriority() = default;
77		SizePriority(const CallBase *CB, FunctionAnalysisManager &,
78	0	const InlineParams &) {
79	0	Function *Callee = CB->getCalledFunction();
80	0	Size = Callee->getInstructionCount();
81	0	}
82
83	0	static bool isMoreDesirable(const SizePriority &P1, const SizePriority &P2) {
84	0	return P1.Size < P2.Size;
85	0	}
86
87		private:
88		unsigned Size = UINT_MAX;
89		};
90
91		class CostPriority {
92		public:
93	0	CostPriority() = default;
94		CostPriority(const CallBase *CB, FunctionAnalysisManager &FAM,
95	0	const InlineParams &Params) {
96	0	auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);
97	0	if (IC.isVariable())
98	0	Cost = IC.getCost();
99	0	else
100	0	Cost = IC.isNever() ? INT_MAX : INT_MIN;
101	0	}
102
103	0	static bool isMoreDesirable(const CostPriority &P1, const CostPriority &P2) {
104	0	return P1.Cost < P2.Cost;
105	0	}
106
107		private:
108		int Cost = INT_MAX;
109		};
110
111		class CostBenefitPriority {
112		public:
113	0	CostBenefitPriority() = default;
114		CostBenefitPriority(const CallBase *CB, FunctionAnalysisManager &FAM,
115	0	const InlineParams &Params) {
116	0	auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);
117	0	Cost = IC.getCost();
118	0	StaticBonusApplied = IC.getStaticBonusApplied();
119	0	CostBenefit = IC.getCostBenefit();
120	0	}
121
122		static bool isMoreDesirable(const CostBenefitPriority &P1,
123	0	const CostBenefitPriority &P2) {
124		// We prioritize call sites in the dictionary order of the following
125		// priorities:
126		//
127		// 1. Those call sites that are expected to reduce the caller size when
128		// inlined. Within them, we prioritize those call sites with bigger
129		// reduction.
130		//
131		// 2. Those call sites that have gone through the cost-benefit analysis.
132		// Currently, they are limited to hot call sites. Within them, we
133		// prioritize those call sites with higher benefit-to-cost ratios.
134		//
135		// 3. Remaining call sites are prioritized according to their costs.
136
137		// We add back StaticBonusApplied to determine whether we expect the caller
138		// to shrink (even if we don't delete the callee).
139	0	bool P1ReducesCallerSize =
140	0	P1.Cost + P1.StaticBonusApplied < ModuleInlinerTopPriorityThreshold;
141	0	bool P2ReducesCallerSize =
142	0	P2.Cost + P2.StaticBonusApplied < ModuleInlinerTopPriorityThreshold;
143	0	if (P1ReducesCallerSize \|\| P2ReducesCallerSize) {
144		// If one reduces the caller size while the other doesn't, then return
145		// true iff P1 reduces the caller size.
146	0	if (P1ReducesCallerSize != P2ReducesCallerSize)
147	0	return P1ReducesCallerSize;
148
149		// If they both reduce the caller size, pick the one with the smaller
150		// cost.
151	0	return P1.Cost < P2.Cost;
152	0	}
153
154	0	bool P1HasCB = P1.CostBenefit.has_value();
155	0	bool P2HasCB = P2.CostBenefit.has_value();
156	0	if (P1HasCB \|\| P2HasCB) {
157		// If one has undergone the cost-benefit analysis while the other hasn't,
158		// then return true iff P1 has.
159	0	if (P1HasCB != P2HasCB)
160	0	return P1HasCB;
161
162		// If they have undergone the cost-benefit analysis, then pick the one
163		// with a higher benefit-to-cost ratio.
164	0	APInt LHS = P1.CostBenefit->getBenefit() * P2.CostBenefit->getCost();
165	0	APInt RHS = P2.CostBenefit->getBenefit() * P1.CostBenefit->getCost();
166	0	return LHS.ugt(RHS);
167	0	}
168
169		// Remaining call sites are ordered according to their costs.
170	0	return P1.Cost < P2.Cost;
171	0	}
172
173		private:
174		int Cost = INT_MAX;
175		int StaticBonusApplied = 0;
176		std::optional<CostBenefitPair> CostBenefit;
177		};
178
179		class MLPriority {
180		public:
181	0	MLPriority() = default;
182		MLPriority(const CallBase *CB, FunctionAnalysisManager &FAM,
183	0	const InlineParams &Params) {
184	0	auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);
185	0	if (IC.isVariable())
186	0	Cost = IC.getCost();
187	0	else
188	0	Cost = IC.isNever() ? INT_MAX : INT_MIN;
189	0	}
190
191	0	static bool isMoreDesirable(const MLPriority &P1, const MLPriority &P2) {
192	0	return P1.Cost < P2.Cost;
193	0	}
194
195		private:
196		int Cost = INT_MAX;
197		};
198
199		template <typename PriorityT>
200		class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {
201		using T = std::pair<CallBase *, int>;
202
203	0	bool hasLowerPriority(const CallBase L, const CallBase R) const {
204	0	const auto I1 = Priorities.find(L);
205	0	const auto I2 = Priorities.find(R);
206	0	assert(I1 != Priorities.end() && I2 != Priorities.end());
207	0	return PriorityT::isMoreDesirable(I2->second, I1->second);
208	0	} Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::hasLowerPriority(llvm::CallBase const, llvm::CallBase const) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::hasLowerPriority(llvm::CallBase const, llvm::CallBase const) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::hasLowerPriority(llvm::CallBase const, llvm::CallBase const) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::hasLowerPriority(llvm::CallBase const, llvm::CallBase const) const
209
210	0	bool updateAndCheckDecreased(const CallBase *CB) {
211	0	auto It = Priorities.find(CB);
212	0	const auto OldPriority = It->second;
213	0	It->second = PriorityT(CB, FAM, Params);
214	0	const auto NewPriority = It->second;
215	0	return PriorityT::isMoreDesirable(OldPriority, NewPriority);
216	0	} Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::updateAndCheckDecreased(llvm::CallBase const) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::updateAndCheckDecreased(llvm::CallBase const) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::updateAndCheckDecreased(llvm::CallBase const) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::updateAndCheckDecreased(llvm::CallBase const)
217
218		// A call site could become less desirable for inlining because of the size
219		// growth from prior inlining into the callee. This method is used to lazily
220		// update the desirability of a call site if it's decreasing. It is only
221		// called on pop(), not every time the desirability changes. When the
222		// desirability of the front call site decreases, an updated one would be
223		// pushed right back into the heap. For simplicity, those cases where the
224		// desirability of a call site increases are ignored here.
225	0	void pop_heap_adjust() {
226	0	std::pop_heap(Heap.begin(), Heap.end(), isLess);
227	0	while (updateAndCheckDecreased(Heap.back())) {
228	0	std::push_heap(Heap.begin(), Heap.end(), isLess);
229	0	std::pop_heap(Heap.begin(), Heap.end(), isLess);
230	0	}
231	0	} Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::pop_heap_adjust() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::pop_heap_adjust() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::pop_heap_adjust() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::pop_heap_adjust()
232
233		public:
234		PriorityInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params)
235	0	: FAM(FAM), Params(Params) {
236	0	isLess = [&](const CallBase L, const CallBase R) {
237	0	return hasLowerPriority(L, R);
238	0	}; Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::PriorityInlineOrder(llvm::AnalysisManager<llvm::Function>&, llvm::InlineParams const&)::{lambda(llvm::CallBase const, llvm::CallBase const)#1}::operator()(llvm::CallBase const, llvm::CallBase const) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::PriorityInlineOrder(llvm::AnalysisManager<llvm::Function>&, llvm::InlineParams const&)::{lambda(llvm::CallBase const, llvm::CallBase const)#1}::operator()(llvm::CallBase const, llvm::CallBase const) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::PriorityInlineOrder(llvm::AnalysisManager<llvm::Function>&, llvm::InlineParams const&)::{lambda(llvm::CallBase const, llvm::CallBase const)#1}::operator()(llvm::CallBase const, llvm::CallBase const) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::PriorityInlineOrder(llvm::AnalysisManager<llvm::Function>&, llvm::InlineParams const&)::{lambda(llvm::CallBase const, llvm::CallBase const)#1}::operator()(llvm::CallBase const, llvm::CallBase const) const
239	0	} Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::PriorityInlineOrder(llvm::AnalysisManager<llvm::Function>&, llvm::InlineParams const&) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::PriorityInlineOrder(llvm::AnalysisManager<llvm::Function>&, llvm::InlineParams const&) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::PriorityInlineOrder(llvm::AnalysisManager<llvm::Function>&, llvm::InlineParams const&) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::PriorityInlineOrder(llvm::AnalysisManager<llvm::Function>&, llvm::InlineParams const&)
240
241	0	size_t size() override { return Heap.size(); } Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::size() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::size() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::size() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::size()
242
243	0	void push(const T &Elt) override {
244	0	CallBase *CB = Elt.first;
245	0	const int InlineHistoryID = Elt.second;
246
247	0	Heap.push_back(CB);
248	0	Priorities[CB] = PriorityT(CB, FAM, Params);
249	0	std::push_heap(Heap.begin(), Heap.end(), isLess);
250	0	InlineHistoryMap[CB] = InlineHistoryID;
251	0	} Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::push(std::__1::pair<llvm::CallBase, int> const&) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::push(std::__1::pair<llvm::CallBase, int> const&) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::push(std::__1::pair<llvm::CallBase, int> const&) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::push(std::__1::pair<llvm::CallBase, int> const&)
252
253	0	T pop() override {
254	0	assert(size() > 0);
255	0	pop_heap_adjust();
256
257	0	CallBase *CB = Heap.pop_back_val();
258	0	T Result = std::make_pair(CB, InlineHistoryMap[CB]);
259	0	InlineHistoryMap.erase(CB);
260	0	return Result;
261	0	} Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::pop() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::pop() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::pop() Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::pop()
262
263	0	void erase_if(function_ref<bool(T)> Pred) override {
264	0	auto PredWrapper = [=](CallBase *CB) -> bool {
265	0	return Pred(std::make_pair(CB, 0));
266	0	}; Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::erase_if(llvm::function_ref<bool (std::__1::pair<llvm::CallBase, int>)>)::{lambda(llvm::CallBase)#1}::operator()(llvm::CallBase) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::erase_if(llvm::function_ref<bool (std::__1::pair<llvm::CallBase, int>)>)::{lambda(llvm::CallBase)#1}::operator()(llvm::CallBase) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::erase_if(llvm::function_ref<bool (std::__1::pair<llvm::CallBase, int>)>)::{lambda(llvm::CallBase)#1}::operator()(llvm::CallBase) const Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::erase_if(llvm::function_ref<bool (std::__1::pair<llvm::CallBase, int>)>)::{lambda(llvm::CallBase)#1}::operator()(llvm::CallBase) const
267	0	llvm::erase_if(Heap, PredWrapper);
268	0	std::make_heap(Heap.begin(), Heap.end(), isLess);
269	0	} Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::SizePriority>::erase_if(llvm::function_ref<bool (std::__1::pair<llvm::CallBase, int>)>) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostPriority>::erase_if(llvm::function_ref<bool (std::__1::pair<llvm::CallBase, int>)>) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::CostBenefitPriority>::erase_if(llvm::function_ref<bool (std::__1::pair<llvm::CallBase, int>)>) Unexecuted instantiation: InlineOrder.cpp:(anonymous namespace)::PriorityInlineOrder<(anonymous namespace)::MLPriority>::erase_if(llvm::function_ref<bool (std::__1::pair<llvm::CallBase, int>)>)
270
271		private:
272		SmallVector<CallBase *, 16> Heap;
273		std::function<bool(const CallBase L, const CallBase R)> isLess;
274		DenseMap<CallBase *, int> InlineHistoryMap;
275		DenseMap<const CallBase *, PriorityT> Priorities;
276		FunctionAnalysisManager &FAM;
277		const InlineParams &Params;
278		};
279
280		} // namespace
281
282		AnalysisKey llvm::PluginInlineOrderAnalysis::Key;
283		bool llvm::PluginInlineOrderAnalysis::HasBeenRegistered;
284
285		std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
286		llvm::getDefaultInlineOrder(FunctionAnalysisManager &FAM,
287		const InlineParams &Params,
288	0	ModuleAnalysisManager &MAM, Module &M) {
289	0	switch (UseInlinePriority) {
290	0	case InlinePriorityMode::Size:
291	0	LLVM_DEBUG(dbgs() << " Current used priority: Size priority ---- \n");
292	0	return std::make_unique<PriorityInlineOrder<SizePriority>>(FAM, Params);
293
294	0	case InlinePriorityMode::Cost:
295	0	LLVM_DEBUG(dbgs() << " Current used priority: Cost priority ---- \n");
296	0	return std::make_unique<PriorityInlineOrder<CostPriority>>(FAM, Params);
297
298	0	case InlinePriorityMode::CostBenefit:
299	0	LLVM_DEBUG(
300	0	dbgs() << " Current used priority: cost-benefit priority ---- \n");
301	0	return std::make_unique<PriorityInlineOrder<CostBenefitPriority>>(FAM,
302	0	Params);
303	0	case InlinePriorityMode::ML:
304	0	LLVM_DEBUG(dbgs() << " Current used priority: ML priority ---- \n");
305	0	return std::make_unique<PriorityInlineOrder<MLPriority>>(FAM, Params);
306	0	}
307	0	return nullptr;
308	0	}
309
310		std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
311		llvm::getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params,
312	0	ModuleAnalysisManager &MAM, Module &M) {
313	0	if (llvm::PluginInlineOrderAnalysis::isRegistered()) {
314	0	LLVM_DEBUG(dbgs() << " Current used priority: plugin ---- \n");
315	0	return MAM.getResult<PluginInlineOrderAnalysis>(M).Factory(FAM, Params, MAM,
316	0	M);
317	0	}
318	0	return getDefaultInlineOrder(FAM, Params, MAM, M);
319	0	}