Fixing for Discarded GOs which are emitted during the second build.

llvm/lib/Transforms/IPO/RepoPruning.cpp

@@ -11,16 +11,17 @@
 #include "pstore/core/hamt_map.hpp"
 #include "pstore/core/index_types.hpp"
 #include "pstore/mcrepo/fragment.hpp"
+#include "pstore/mcrepo/section.hpp"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/RepoDefinition.h"
 #include "llvm/IR/RepoGlobals.h"
 #include "llvm/IR/RepoHashCalculator.h"
-#include "llvm/IR/RepoDefinition.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Error.h"
@@ -123,14 +124,128 @@ repodefinition::DigestType toDigestType(pstore::index::digest D) {
   return Digest;
 }
 
+template <pstore::repo::section_kind Kind>
+void collect(pstore::database const &Db, const pstore::repo::fragment &Fragment,
+             StringSet<> &XFixupNames) {
+
+  const auto *const Section = Fragment.atp<Kind>();
+  for (pstore::repo::external_fixup const &Xfx : Section->xfixups()) {
+    LLVM_DEBUG(dbgs() << "    Adding XFixup name: "
+                      << pstore::indirect_string::read(Db, Xfx.name).to_string()
+                      << '\n');
+    XFixupNames.insert(
+        std::move(pstore::indirect_string::read(Db, Xfx.name).to_string()));
+  }
+}
+
+template <>
+inline void collect<pstore::repo::section_kind::dependent>(
+    pstore::database const & /*Db*/,
+    const pstore::repo::fragment & /*Fragment*/,
+    StringSet<> & /*XFixupNames*/) {}
+
+static void collectFragmentXFixupNames(
+    const pstore::database &Repository,
+    std::shared_ptr<const pstore::repo::fragment> const &Fragment,
+    StringSet<> &XFixupNames) {
+  for (const pstore::repo::section_kind Kind : *Fragment) {
+    assert(Fragment->has_section(Kind));
+
+#define X(k)                                                                   \
+  case pstore::repo::section_kind::k: {                                        \
+    collect<pstore::repo::section_kind::k>(Repository, *Fragment,              \
+                                           XFixupNames);                       \
+    break;                                                                     \
+  }
+
+    switch (Kind) {
+      PSTORE_MCREPO_SECTION_KINDS
+    case pstore::repo::section_kind::last:
+      llvm_unreachable("Bad section kind");
+      break;
+    }
+#undef X
+  }
+}
+
+/// Two kinds of pruning exist in the repository level and the module level.
+/// The repository level pruning: if the global object is compiled before and
+/// exists in the repository, it will be pruned. The module level pruning: in
+/// the same compilation unit (module), some global objects have the same
+/// digests. The RepoPruning pass could prune them as well.
+///
+/// This function is used to collect the GO's pruning status which include three
+/// possible states: in the repository, in the ModuleFragments and not in both.
+/// If GO is in the repository, collect the GO's xfixup names. If it is not in
+/// repository and not in ModuleFragments, put it into the ModuleFragments.
+///
+/// \param Repository The database.
+/// \param FragmentIndex A database fragment index.
+/// \param ModuleFragments The collection of fragments that will appear in this
+///        compilation but not include the pruned entries.
+/// \param XFixupNames A set of xfixup's names.
+/// \param PrunedDiscardableIfUnusedGos A set of GOs if GOs exist in the
+///        repository and  may be discarded if it is not used.
+/// \returns a tuple containing 1) true if the GO is in the existing repository.
+///          2) true if the GO doesn't exist in the existing repository but is
+///          in the ModuleFragments, and 3) the GO's digest.
+static std::tuple<bool, bool, pstore::index::digest> hasExistingGO(
+    const pstore::database &Repository,
+    std::shared_ptr<const pstore::index::fragment_index> const &FragmentIndex,
+    std::map<pstore::index::digest, const GlobalObject *> &ModuleFragments,
+    GlobalObject &GO, StringSet<> &XFixupNames,
+    DenseSet<GlobalObject *> &PrunedDiscardableIfUnusedGos) {
+  auto const Result = repodefinition::get(&GO);
+  assert(!Result.second && "The repo_definitio metadata should be created by "
+                           "the RepoMetadataGeneration pass!");
+  auto const Digest =
+      pstore::index::digest{Result.first.high(), Result.first.low()};
+
+  //Case 1: the GO is in the existing repository.
+  if (FragmentIndex) {
+    auto It = FragmentIndex->find(Repository, Digest);
+    if (It != FragmentIndex->end(Repository)) {
+      LLVM_DEBUG(dbgs() << "Existing GO (in repository) name: " << GO.getName()
+                        << '\n');
+      if (!GO.isDiscardableIfUnused()) {
+        // Collect xfixup's names.
+        auto Fragment = pstore::repo::fragment::load(Repository, It->second);
+        collectFragmentXFixupNames(Repository, Fragment, XFixupNames);
+      } else {
+        PrunedDiscardableIfUnusedGos.insert(&GO);
+      }
+      return std::make_tuple(true, false, Digest);
+    }
+  }
+
+  // Case 2: the GO is in the ModuleFragments.
+  auto It = ModuleFragments.find(Digest);
+  if (It != ModuleFragments.end() && !It->second->isDiscardableIfUnused()) {
+    // The definition of some global objects may be discarded if not used.
+    // If a global has been pruned and its digest matches a discardable GO,
+    // a missing fragment error might be met during the assembler. To avoid
+    // this issue, this global object is put into the ModuleFragments only if
+    // it is not a discardable GO.
+    LLVM_DEBUG(dbgs() << "Existing GO (in new fragments) name: " << GO.getName()
+                      << '\n');
+    return std::make_tuple(false, true, Digest);
+  }
+
+  LLVM_DEBUG(dbgs() << "Putting new GO (named: " << GO.getName()
+                    << ") into ModuleFragments.\n");
+  ModuleFragments.emplace(Digest, &GO);
+  return std::make_tuple(false, false, Digest);
+}
+
 static void addDependentFragments(
     Module &M, StringSet<> &DependentFragments,
     std::shared_ptr<const pstore::index::fragment_index> const &Fragments,
-    const pstore::database &Repository, pstore::index::digest const &Digest) {
+    const pstore::database &Repository, StringSet<> &XFixupNames,
+    pstore::index::digest const &Digest) {
 
   auto It = Fragments->find(Repository, Digest);
   assert(It != Fragments->end(Repository));
-  // Create  the dependent fragments if existing in the repository.
+  // Load  the dependent fragments if existing in the repository.
   auto Fragment = pstore::repo::fragment::load(Repository, It->second);
   if (auto Dependents =
           Fragment->atp<pstore::repo::section_kind::dependent>()) {
@@ -150,11 +265,18 @@ static void addDependentFragments(
         NamedMDNode *const NMD = M.getOrInsertNamedMetadata("repo.definitions");
         assert(NMD && "NamedMDNode cannot be NULL!");
         NMD->addOperand(DMD);
+        // Collect the Dependent's XFixups.
+        auto DependentIt = Fragments->find(Repository, CM->digest);
+        assert(DependentIt != Fragments->end(Repository));
+        // Load  the dependent fragments if existing in the repository.
+        auto DependentFragment =
+            pstore::repo::fragment::load(Repository, DependentIt->second);
+        collectFragmentXFixupNames(Repository, DependentFragment, XFixupNames);
         // If function 'A' is dependent on function 'B' and 'B' is dependent on
         // function 'C', both RepoDefinition of 'B' and 'C' need to be added
         // into in the 'repo.definitions' during the pruning.
         addDependentFragments(M, DependentFragments, Fragments, Repository,
-                              CM->digest);
+                              XFixupNames, CM->digest);
       }
     }
   }
@@ -192,6 +314,20 @@ static void deleteGlobalObject(GlobalObject *GO) {
   }
 }
 
+static void pruning(GlobalObject &GO) {
+  GO.setComdat(nullptr);
+  GO.setDSOLocal(false);
+  RepoDefinition *MD =
+      dyn_cast<RepoDefinition>(GO.getMetadata(LLVMContext::MD_repo_definition));
+  MD->setPruned(true);
+
+  if (isSafeToPruneIntrinsicGV(GO) || GO.use_empty()) {
+    deleteGlobalObject(&GO);
+    return;
+  }
+  GO.setLinkage(GlobalValue::AvailableExternallyLinkage);
+}
+
 static bool doPruning(Module &M) {
   MDBuilder MDB(M.getContext());
   const pstore::database &Repository = getRepoDatabase();
@@ -205,64 +341,38 @@ static bool doPruning(Module &M) {
 
   std::map<pstore::index::digest, const GlobalObject *> ModuleFragments;
   StringSet<> DependentFragments; // Record all dependents.
+  StringSet<> XFixupNames;        // Record all xfixups' name for pruned GOs.
+  DenseSet<GlobalObject *> PrunedDiscardableIfUnusedGos;
 
-  // Erase the unchanged global objects.
+  // Erase the unchanged global objects which is not discardable if unused.
   auto EraseUnchangedGlobalObject =
-      [&ModuleFragments, &Fragments, &Repository, &M,
-       &DependentFragments](GlobalObject &GO) -> bool {
+      [&ModuleFragments, &Fragments, &Repository, &M, &DependentFragments,
+       &XFixupNames, &PrunedDiscardableIfUnusedGos](GlobalObject &GO) -> bool {
     if (GO.isDeclaration() || GO.hasAvailableExternallyLinkage() ||
         !isSafeToPrune(GO))
       return false;
-    auto const Result = repodefinition::get(&GO);
-    assert(!Result.second &&
-           "The repo_definition metadata should be created by "
-           "the RepoMetadataGeneration pass!");
-
-    auto const Key =
-        pstore::index::digest{Result.first.high(), Result.first.low()};
-
-    bool InRepository = true;
-    if (!Fragments) {
-      InRepository = false;
-    } else {
-      auto It = Fragments->find(Repository, Key);
-      if (It == Fragments->end(Repository)) {
-        LLVM_DEBUG(dbgs() << "New GO name: " << GO.getName() << '\n');
-        InRepository = false;
-      } else {
-        LLVM_DEBUG(dbgs() << "Prunning GO name: " << GO.getName() << '\n');
-        addDependentFragments(M, DependentFragments, Fragments, Repository,
-                              Key);
-      }
-    }
 
-    if (!InRepository) {
-      auto It = ModuleFragments.find(Key);
-      // The definition of some global objects may be discarded if not used.
-      // If a global has been pruned and its digest matches a discardable GO,
-      // a missing fragment error might be met during the assembler. To avoid
-      // this issue, this global object can't be pruned if the referenced
-      // global object is discardable.
-      if (It == ModuleFragments.end() || It->second->isDiscardableIfUnused()) {
-        LLVM_DEBUG(dbgs() << "Putting GO name into ModuleFragments: "
-                          << GO.getName() << '\n');
-        ModuleFragments.emplace(Key, &GO);
-        return false;
-      }
+    bool InRepository;
+    bool InNewFragments;
+    pstore::index::digest Key;
+    std::tie(InRepository, InNewFragments, Key) =
+        hasExistingGO(Repository, Fragments, ModuleFragments, GO, XFixupNames,
+                      PrunedDiscardableIfUnusedGos);
+
+    if (!InRepository && !InNewFragments) {
+      return false;
     }
 
-    GO.setComdat(nullptr);
-    GO.setDSOLocal(false);
-    RepoDefinition *MD = dyn_cast<RepoDefinition>(
-        GO.getMetadata(LLVMContext::MD_repo_definition));
-    MD->setPruned(true);
+    if (InRepository && GO.isDiscardableIfUnused()) {
+      return false;
+    }
 
-    if (isSafeToPruneIntrinsicGV(GO) || GO.use_empty()) {
-      deleteGlobalObject(&GO);
-      return true;
+    if (InRepository) {
+      addDependentFragments(M, DependentFragments, Fragments, Repository,
+                            XFixupNames, Key);
     }
 
-    GO.setLinkage(GlobalValue::AvailableExternallyLinkage);
+    pruning(GO);
     return true;
   };
 
@@ -273,6 +383,45 @@ static bool doPruning(Module &M) {
     }
   }
 
+  auto EraseUnchangedDiscardableGlobalObject =
+      [&XFixupNames, &Fragments, &Repository, &M,
+       &DependentFragments](GlobalObject *GO) -> bool {
+    if (XFixupNames.find(GO->getName()) != XFixupNames.end()) {
+      LLVM_DEBUG(dbgs() << "Pruning GO name (DiscardableIfUnused): "
+                        << GO->getName() << '\n');
+      auto const Result = repodefinition::get(GO);
+      auto const Digest =
+          pstore::index::digest{Result.first.high(), Result.first.low()};
+
+      addDependentFragments(M, DependentFragments, Fragments, Repository,
+                            XFixupNames, Digest);
+
+      auto It = Fragments->find(Repository, Digest);
+      auto Fragment = pstore::repo::fragment::load(Repository, It->second);
+      collectFragmentXFixupNames(Repository, Fragment, XFixupNames);
+
+      pruning(*GO);
+      return true;
+    }
+    return false;
+  };
+
+  bool IsPruned;
+  do {
+    IsPruned = false;
+    // Go through discardable and pruned GOs.
+    for (auto It = PrunedDiscardableIfUnusedGos.begin();
+         It != PrunedDiscardableIfUnusedGos.end();) {
+      auto CurIt = It;
+      ++It;
+      if (EraseUnchangedDiscardableGlobalObject(*CurIt)) {
+        IsPruned = true;
+        Changed = true;
+        PrunedDiscardableIfUnusedGos.erase(CurIt);
+      }
+    }
+  } while (IsPruned);
+
   return Changed;
 }
 

llvm/test/Feature/Repo/Inputs/repo_external_GO.c

@@ -0,0 +1,3 @@
+int f0() {
+  return 1;
+}

llvm/test/Feature/Repo/repo_discardable_GO.c

@@ -0,0 +1,15 @@
+// RUN: rm -rf %t.db
+// RUN: env REPOFILE=%t.db clang -c -O3 -target x86_64-pc-linux-gnu-repo %s -o %t.o
+// RUN: env REPOFILE=%t.db repo-ticket-dump %t.o > %t.log
+// RUN: env REPOFILE=%t.db clang -c -O3 -target x86_64-pc-linux-gnu-repo %S/Inputs/repo_external_GO.c -o %t1.o
+// RUN: env REPOFILE=%t.db clang -c -O3 -target x86_64-pc-linux-gnu-repo %s -o %t2.o
+// RUN: env REPOFILE=%t.db repo-ticket-dump %t2.o > %t2.log
+// RUN: diff %t.log %t2.log
+
+static int f0() {
+  return 1;
+}
+
+int f1() {
+  return f0();
+}