[CAS] Sync LLVMCAS library implementation to next

clang/tools/driver/cc1depscan_main.cpp

@@ -941,7 +941,7 @@ void ScanServer::start(bool Exclusive, ArrayRef<const char *> CASArgs) {
       return;
     SmallString<64> LLVMCasStorage;
     SmallString<64> CASPath;
-    CASOpts.getResolvedCASPath(CASPath);
+    ExitOnErr(CASOpts.getResolvedCASPath(CASPath));
     ExitOnErr(llvm::cas::validateOnDiskUnifiedCASDatabasesIfNeeded(
         CASPath, /*CheckHash=*/true,
         /*AllowRecovery=*/true,
@@ -961,7 +961,8 @@ void ScanServer::start(bool Exclusive, ArrayRef<const char *> CASArgs) {
 
     // Try to lock; failure means there's another daemon running.
     if (std::error_code EC = llvm::sys::fs::tryLockFile(
-            PidFD, std::chrono::milliseconds(0), /*Exclusive=*/true)) {
+            PidFD, std::chrono::milliseconds(0),
+            llvm::sys::fs::LockKind::Exclusive)) {
       if (Exclusive)
         reportError("another daemon using the base path");
       ::exit(0);

llvm/CMakeLists.txt

@@ -890,6 +890,7 @@ option (LLVM_ENABLE_SPHINX "Use Sphinx to generate llvm documentation." OFF)
 option (LLVM_ENABLE_OCAMLDOC "Build OCaml bindings documentation." ON)
 option (LLVM_ENABLE_BINDINGS "Build bindings." ON)
 option (LLVM_ENABLE_TELEMETRY "Enable the telemetry library. If set to OFF, library cannot be enabled after build (eg., at runtime)" ON)
+option (LLVM_ENABLE_ONDISK_CAS "Build OnDiskCAS." ON)
 
 if(CMAKE_SIZEOF_VOID_P GREATER_EQUAL 8)
   set(LLVM_ENABLE_ONDISK_CAS_default ON)

llvm/include/llvm/CAS/ActionCache.h

@@ -1,13 +1,19 @@
-//===- llvm/CAS/ActionCache.h -----------------------------------*- C++ -*-===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declaration of the ActionCache class, which is the
+/// base class for ActionCache implementations.
+///
+//===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CAS_CASACTIONCACHE_H
-#define LLVM_CAS_CASACTIONCACHE_H
+#ifndef LLVM_CAS_ACTIONCACHE_H
+#define LLVM_CAS_ACTIONCACHE_H
 
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/StringRef.h"
@@ -27,15 +33,11 @@ class ObjectProxy;
 /// for CAS types.
 class CacheKey {
 public:
-  LLVM_ABI StringRef getKey() const { return Key; }
+  StringRef getKey() const { return Key; }
 
-  // TODO: Support CacheKey other than a CASID but rather any array of bytes.
-  // To do that, ActionCache need to be able to rehash the key into the index,
-  // which then `getOrCompute` method can be used to avoid multiple calls to
-  // has function.
   LLVM_ABI CacheKey(const CASID &ID);
-  LLVM_ABI CacheKey(const ObjectProxy &Proxy);
-  LLVM_ABI CacheKey(const ObjectStore &CAS, const ObjectRef &Ref);
+  LLVM_ABI_FOR_TEST CacheKey(const ObjectProxy &Proxy);
+  CacheKey(const ObjectStore &CAS, const ObjectRef &Ref);
 
 private:
   std::string Key;
@@ -54,63 +56,67 @@ struct AsyncErrorValue {
   Error Value;
 };
 
-/// A cache from a key describing an action to the result of doing it.
+/// A cache from a key (that describes an action) to the result of performing
+/// that action.
 ///
-/// Actions are expected to be pure (collision is an error).
+/// Actions are expected to be pure. Storing mappings from one action to
+/// multiple results will result in error (cache poisoning).
 class ActionCache {
   virtual void anchor();
 
 public:
   /// Get a previously computed result for \p ActionKey.
   ///
-  /// \param Globally if true it is a hint to the underlying implementation that
-  /// the lookup is profitable to be done on a distributed caching level, not
-  /// just locally. The implementation is free to ignore this flag.
+  /// \param CanBeDistributed is a hint to the underlying implementation that if
+  /// it is true, the lookup is profitable to be done on a distributed caching
+  /// level, not just locally. The implementation is free to ignore this flag.
   Expected<std::optional<CASID>> get(const CacheKey &ActionKey,
-                                     bool Globally = false) const {
-    return getImpl(arrayRefFromStringRef(ActionKey.getKey()), Globally);
+                                     bool CanBeDistributed = false) const {
+    return getImpl(arrayRefFromStringRef(ActionKey.getKey()), CanBeDistributed);
   }
 
   /// Asynchronous version of \c get.
   std::future<AsyncCASIDValue> getFuture(const CacheKey &ActionKey,
-                                         bool Globally = false) const;
+                                         bool CanBeDistributed = false) const;
 
   /// Asynchronous version of \c get.
-  void getAsync(const CacheKey &ActionKey, bool Globally,
+  void getAsync(const CacheKey &ActionKey, bool CanBeDistributed,
                 unique_function<void(Expected<std::optional<CASID>>)> Callback,
                 std::unique_ptr<Cancellable> *CancelObj = nullptr) const {
-    return getImplAsync(arrayRefFromStringRef(ActionKey.getKey()), Globally,
+    return getImplAsync(arrayRefFromStringRef(ActionKey.getKey()), CanBeDistributed,
                         std::move(Callback), CancelObj);
   }
 
   /// Cache \p Result for the \p ActionKey computation.
   ///
-  /// \param Globally if true it is a hint to the underlying implementation that
-  /// the association is profitable to be done on a distributed caching level,
-  /// not just locally. The implementation is free to ignore this flag.
+  /// \param CanBeDistributed is a hint to the underlying implementation that if
+  /// it is true, the association is profitable to be done on a distributed
+  /// caching level, not just locally. The implementation is free to ignore this
+  /// flag.
   Error put(const CacheKey &ActionKey, const CASID &Result,
-            bool Globally = false) {
+            bool CanBeDistributed = false) {
     assert(Result.getContext().getHashSchemaIdentifier() ==
                getContext().getHashSchemaIdentifier() &&
            "Hash schema mismatch");
-    return putImpl(arrayRefFromStringRef(ActionKey.getKey()), Result, Globally);
+    return putImpl(arrayRefFromStringRef(ActionKey.getKey()), Result,
+                   CanBeDistributed);
   }
 
   /// Asynchronous version of \c put.
   std::future<AsyncErrorValue> putFuture(const CacheKey &ActionKey,
                                          const CASID &Result,
-                                         bool Globally = false);
+                                         bool CanBeDistributed = false);
 
   /// Asynchronous version of \c put.
   /// \param[out] CancelObj Optional pointer to receive a cancellation object.
-  void putAsync(const CacheKey &ActionKey, const CASID &Result, bool Globally,
+  void putAsync(const CacheKey &ActionKey, const CASID &Result, bool CanBeDistributed,
                 unique_function<void(Error)> Callback,
                 std::unique_ptr<Cancellable> *CancelObj = nullptr) {
     assert(Result.getContext().getHashSchemaIdentifier() ==
                getContext().getHashSchemaIdentifier() &&
            "Hash schema mismatch");
     return putImplAsync(arrayRefFromStringRef(ActionKey.getKey()), Result,
-                        Globally, std::move(Callback), CancelObj);
+                        CanBeDistributed, std::move(Callback), CancelObj);
   }
 
   /// Validate the ActionCache contents.
@@ -119,17 +125,21 @@ class ActionCache {
   virtual ~ActionCache() = default;
 
 protected:
-  virtual Expected<std::optional<CASID>> getImpl(ArrayRef<uint8_t> ResolvedKey,
-                                                 bool Globally) const = 0;
+  // Implementation detail for \p get method.
+  virtual Expected<std::optional<CASID>>
+  getImpl(ArrayRef<uint8_t> ResolvedKey, bool CanBeDistributed) const = 0;
+
   virtual void
-  getImplAsync(ArrayRef<uint8_t> ResolvedKey, bool Globally,
+  getImplAsync(ArrayRef<uint8_t> ResolvedKey, bool CanBeDistributed,
                unique_function<void(Expected<std::optional<CASID>>)> Callback,
                std::unique_ptr<Cancellable> *CancelObj) const;
 
+  // Implementation detail for \p put method.
   virtual Error putImpl(ArrayRef<uint8_t> ResolvedKey, const CASID &Result,
-                        bool Globally) = 0;
+                        bool CanBeDistributed) = 0;
+
   virtual void putImplAsync(ArrayRef<uint8_t> ResolvedKey, const CASID &Result,
-                            bool Globally,
+                            bool CanBeDistributed,
                             unique_function<void(Error)> Callback,
                             std::unique_ptr<Cancellable> *CancelObj);
 
@@ -146,12 +156,12 @@ LLVM_ABI std::unique_ptr<ActionCache> createInMemoryActionCache();
 
 /// Get a reasonable default on-disk path for a persistent ActionCache for the
 /// current user.
-LLVM_ABI std::string getDefaultOnDiskActionCachePath();
+std::string getDefaultOnDiskActionCachePath();
 
 /// Create an action cache on disk.
 LLVM_ABI Expected<std::unique_ptr<ActionCache>>
 createOnDiskActionCache(StringRef Path);
 
 } // end namespace llvm::cas
 
-#endif // LLVM_CAS_CASACTIONCACHE_H
+#endif // LLVM_CAS_ACTIONCACHE_H

llvm/include/llvm/CAS/BuiltinCASContext.h

@@ -54,6 +54,7 @@ namespace llvm::cas::builtin {
 using HasherT = BLAKE3;
 using HashType = decltype(HasherT::hash(std::declval<ArrayRef<uint8_t> &>()));
 
+/// CASContext for LLVM builtin CAS using BLAKE3 hash type.
 class BuiltinCASContext : public CASContext {
   void printIDImpl(raw_ostream &OS, const CASID &ID) const final;
   void anchor() override;

llvm/include/llvm/CAS/BuiltinObjectHasher.h

@@ -9,13 +9,12 @@
 #ifndef LLVM_CAS_BUILTINOBJECTHASHER_H
 #define LLVM_CAS_BUILTINOBJECTHASHER_H
 
-#include "llvm/ADT/StringRef.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Endian.h"
 
-namespace llvm {
-namespace cas {
+namespace llvm::cas {
 
+/// Hasher for stored objects in builtin CAS.
 template <class HasherT> class BuiltinObjectHasher {
 public:
   using HashT = decltype(HasherT::hash(std::declval<ArrayRef<uint8_t> &>()));
@@ -66,21 +65,20 @@ template <class HasherT> class BuiltinObjectHasher {
 
   void updateArray(ArrayRef<char> Bytes) {
     updateArray(ArrayRef(reinterpret_cast<const uint8_t *>(Bytes.data()),
-                             Bytes.size()));
+                         Bytes.size()));
   }
 
   void updateSize(uint64_t Size) {
     Size = support::endian::byte_swap(Size, endianness::little);
-    Hasher.update(ArrayRef(reinterpret_cast<const uint8_t *>(&Size),
-                                sizeof(Size)));
+    Hasher.update(
+        ArrayRef(reinterpret_cast<const uint8_t *>(&Size), sizeof(Size)));
   }
 
   BuiltinObjectHasher() = default;
   ~BuiltinObjectHasher() = default;
   HasherT Hasher;
 };
 
-} // namespace cas
-} // namespace llvm
+} // namespace llvm::cas
 
 #endif // LLVM_CAS_BUILTINOBJECTHASHER_H

llvm/include/llvm/CAS/BuiltinUnifiedCASDatabases.h

@@ -1,4 +1,4 @@
-//===- BuiltinUnifiedCASDatabases.h -----------------------------*- C++ -*-===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -31,7 +31,7 @@ createOnDiskUnifiedCASDatabases(StringRef Path);
 /// marking the files for garbage collection.
 /// \param ForceValidation Whether to force validation to occur even if it
 /// should not be necessary.
-/// \param LLVMCasBinary If provided, validation is performed out-of-process
+/// \param LLVMCasBinaryPath If provided, validation is performed out-of-process
 /// using the given \c llvm-cas executable which protects against crashes
 /// during validation. Otherwise validation is performed in-process.
 ///
@@ -42,7 +42,7 @@ createOnDiskUnifiedCASDatabases(StringRef Path);
 LLVM_ABI
 Expected<ValidationResult> validateOnDiskUnifiedCASDatabasesIfNeeded(
     StringRef Path, bool CheckHash, bool AllowRecovery, bool ForceValidation,
-    std::optional<StringRef> LLVMCasBinary);
+    std::optional<StringRef> LLVMCasBinaryPath);
 
 } // namespace llvm::cas
 

llvm/include/llvm/CAS/CASConfiguration.h

@@ -51,11 +51,7 @@ class CASConfiguration {
     return !(LHS == RHS);
   }
 
-  // Get resolved CASPath.
-  void getResolvedCASPath(llvm::SmallVectorImpl<char> &Result) const;
-
   // Create CASDatabase from the CASConfiguration.
-  LLVM_ABI
   llvm::Expected<std::pair<std::shared_ptr<llvm::cas::ObjectStore>,
                            std::shared_ptr<llvm::cas::ActionCache>>>
   createDatabases() const;
@@ -71,12 +67,14 @@ class CASConfiguration {
   ///
   /// Returns the path to configuration file and its corresponding
   /// CASConfiguration.
-  LLVM_ABI
   static std::optional<std::pair<std::string, CASConfiguration>>
   createFromSearchConfigFile(
       StringRef Path,
       llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = nullptr);
 
+  /// Get resolved CASPath.
+  Error getResolvedCASPath(llvm::SmallVectorImpl<char> &Result) const;
+
   /// DenseMap support \{
   static cas::CASConfiguration getDenseMapEmptyKey() { return {}; }
 

llvm/include/llvm/CAS/CASFileSystem.h

@@ -20,7 +20,7 @@ class CASID;
 /// Abstract class represents an open file backed by a CAS.
 class CASBackedFile : public RTTIExtends<CASBackedFile, vfs::File> {
 public:
-  LLVM_ABI static const char ID;
+  static const char ID;
   /// Get the CAS reference for the contents of the file.
   virtual cas::ObjectRef getObjectRefForContent() = 0;
 };
@@ -29,7 +29,7 @@ class CASBackedFile : public RTTIExtends<CASBackedFile, vfs::File> {
 class CASBackedFileSystem
     : public llvm::RTTIExtends<CASBackedFileSystem, vfs::FileSystem> {
 public:
-  LLVM_ABI static const char ID;
+  static const char ID;
 
   /// This is a convenience method that opens a file, gets its content and then
   /// closes the file. It returns MemoryBuffer and ObjectRef in one call to avoid
@@ -42,8 +42,7 @@ class CASBackedFileSystem
                                bool IsVolatile = false, bool IsText = true);
 
   /// Get ObjectRef of a file from its path.
-  LLVM_ABI llvm::Expected<cas::ObjectRef>
-  getObjectRefForFileContent(const Twine &Name);
+  llvm::Expected<cas::ObjectRef> getObjectRefForFileContent(const Twine &Name);
 
   /// Implementation for openFileForRead using CASBackedFile.
   ErrorOr<std::unique_ptr<vfs::File>>
@@ -63,11 +62,11 @@ class CASBackedFileSystem
   createThreadSafeProxyFS() = 0;
 };
 
-LLVM_ABI Expected<std::unique_ptr<vfs::FileSystem>>
+Expected<std::unique_ptr<vfs::FileSystem>>
 createCASFileSystem(std::shared_ptr<ObjectStore> DB, const CASID &RootID,
                     sys::path::Style PathStyle = sys::path::Style::native);
 
-LLVM_ABI Expected<std::unique_ptr<vfs::FileSystem>>
+Expected<std::unique_ptr<vfs::FileSystem>>
 createCASFileSystem(ObjectStore &DB, const CASID &RootID,
                     sys::path::Style PathStyle = sys::path::Style::native);
 

llvm/include/llvm/CAS/CASID.h

@@ -55,18 +55,21 @@ class CASContext {
 /// compared directly. If they are, then \a
 /// CASIDContext::getHashSchemaIdentifier() is compared to see if they can be
 /// compared by hash, in which case the result of \a getHash() is compared.
-///
-/// FIXME: Rename to ObjectID (and rename file to CASObjectID.h?).
 class CASID {
 public:
   void dump() const;
-  void print(raw_ostream &OS) const {
-    return getContext().printIDImpl(OS, *this);
-  }
+
   friend raw_ostream &operator<<(raw_ostream &OS, const CASID &ID) {
     ID.print(OS);
     return OS;
   }
+
+  /// Print CASID.
+  void print(raw_ostream &OS) const {
+    return getContext().printIDImpl(OS, *this);
+  }
+
+  /// Return a printable string for CASID.
   LLVM_ABI std::string toString() const;
 
   ArrayRef<uint8_t> getHash() const {
@@ -92,8 +95,7 @@ class CASID {
   }
 
   friend hash_code hash_value(const CASID &ID) {
-    ArrayRef<uint8_t> Hash = ID.getHash();
-    return hash_combine_range(Hash.begin(), Hash.end());
+    return hash_combine_range(ID.getHash());
   }
 
   const CASContext &getContext() const {
@@ -110,6 +112,7 @@ class CASID {
 
   CASID() = delete;
 
+  /// Create CASID from CASContext and raw hash bytes.
   static CASID create(const CASContext *Context, StringRef Hash) {
     return CASID(Context, Hash);
   }
@@ -137,7 +140,7 @@ template <typename T> struct AsyncValue {
 };
 
 class Cancellable {
-  LLVM_ABI virtual void anchor();
+  virtual void anchor();
 
 public:
   virtual ~Cancellable() {}