Introduce global git cache

CHANGELOG.md

@@ -13,6 +13,7 @@ Other improvements:
 - Fix output truncation with `--json-errors`, many warnings and build failure (#1199)
 - Update README with info about depending on a freshly added library
 - Fixed globbing issue where `/.spago` behaves differently than `.spago` in `.gitignore`
+- When a git repo is used for multiple dependencies, only clone it once
 - Fixed empty output for `--verbose-stats` when there are no errors or warnings.
 - Added support for `--package-set` options for `spago upgrade`.
 - `spago repl` now writes a `.purs-repl` file, unless already there, containing `import Prelude`.

src/Spago/Command/Fetch.purs

@@ -443,28 +443,64 @@ toAllDependencies = foldl (Map.unionWith (\l _ -> l)) Map.empty
 getGitPackageInLocalCache :: forall a. PackageName -> GitPackage -> Spago (Git.GitEnv a) Unit
 getGitPackageInLocalCache name package = do
   let localPackageLocation = Config.getPackageLocation name (GitPackage package)
-  tempDir <- mkTemp' (Just $ printJson Config.gitPackageCodec package)
-  logDebug $ "Cloning repo in " <> tempDir
-  Git.fetchRepo package tempDir >>= case _ of
-    Left err -> die err
-    Right _ -> do
-      logDebug $ "Repo cloned. Moving to " <> localPackageLocation
-      FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ]
-      FS.moveSync { src: tempDir, dst: localPackageLocation }
-
-      -- Note: the package might have been cloned with a tag, but we stick the commit hash in the lockfiles
-      -- so we need to make a copy to a location that has the commit hash too.
-      -- So we run getRef here and then do a copy if the ref is different than the original one
-      -- (since it might be a commit to start with)
-      logDebug $ "Checking if we need to copy the package to a commit hash location..."
-      Git.getRef (Just localPackageLocation) >>= case _ of
+  existsGitCache <- FS.exists gitCacheLocation
+  { globallyCached, sourceDir } <-
+    if existsGitCache then do
+      logDebug $ "Using global git cache in " <> gitCacheLocation
+      pure { globallyCached: true, sourceDir: gitCacheLocation }
+    else do
+      -- Not cached in the global cache, so we download it and then see if it can be cached
+      tempDir <- mkTemp' (Just $ printJson Config.gitPackageCodec package)
+      logDebug $ "Cloning repo in " <> tempDir
+      fetchRes <- Git.fetchRepo package tempDir
+      case fetchRes of
         Left err -> die err
-        Right ref -> do
-          when (ref /= package.ref) do
-            let commitHashLocation = Config.getPackageLocation name (GitPackage $ package { ref = ref })
-            logDebug $ "Copying the repo also to " <> commitHashLocation
-            FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ]
-            FS.copyTree { src: localPackageLocation, dst: commitHashLocation }
+        Right _ -> do
+          shouldCache <- not <$> Git.isBranch { ref: package.ref, path: tempDir }
+          if shouldCache then do
+            FS.mkdirp globalGitCache
+            try (FS.moveSync { src: tempDir, dst: gitCacheLocation }) >>= case _ of
+              Left _err -> logDebug $ "Couldn't move cloned repo to " <> gitCacheLocation
+              Right _ -> pure unit
+            pure { globallyCached: true, sourceDir: gitCacheLocation }
+          else do
+            pure { globallyCached: false, sourceDir: tempDir }
+
+  FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ]
+  if globallyCached then do
+    logDebug $ "Repo cloned. Copying to " <> localPackageLocation
+    FS.copyTree { src: sourceDir, dst: localPackageLocation }
+  else do
+    logDebug $ "Repo cloned. Moving to " <> localPackageLocation
+    FS.moveSync { src: sourceDir, dst: localPackageLocation }
+
+  -- Note: the package might have been cloned with a tag, but we stick the commit hash in the lockfiles
+  -- so we need to make a copy to a location that has the commit hash too.
+  -- So we run getRef here and then do a copy if the ref is different than the original one
+  -- (since it might be a commit to start with)
+  logDebug $ "Checking if we need to copy the package to a commit hash location..."
+  Git.getRef (Just localPackageLocation) >>= case _ of
+    Left err -> die err
+    Right ref -> do
+      when (ref /= package.ref) do
+        let commitHashLocation = Config.getPackageLocation name (GitPackage $ package { ref = ref })
+        logDebug $ "Copying the repo also to " <> commitHashLocation
+        FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ]
+        FS.copyTree { src: localPackageLocation, dst: commitHashLocation }
+  where
+  globalGitCacheElems =
+    [ Paths.globalCachePath
+    , "git"
+    , Config.fileSystemCharEscape package.git
+    ]
+
+  globalGitCache :: FilePath
+  globalGitCache =
+    Path.concat $ globalGitCacheElems
+
+  gitCacheLocation :: FilePath
+  gitCacheLocation =
+    Path.concat $ globalGitCacheElems <> [ Config.fileSystemCharEscape package.ref ]
 
 getPackageDependencies :: forall a. PackageName -> Package -> Spago (FetchEnv a) (Maybe (Map PackageName Range))
 getPackageDependencies packageName package = case package of

src/Spago/Git.purs

@@ -11,6 +11,7 @@ module Spago.Git
   , parseRemote
   , pushTag
   , tagCheckedOut
+  , isBranch
   ) where
 
 import Spago.Prelude
@@ -48,6 +49,12 @@ runGit args cwd = ExceptT do
     Right r -> Right r.stdout
     Left r -> Left r.stderr
 
+-- See https://stackoverflow.com/questions/18222634
+isBranch :: forall a b. { ref :: String, path :: FilePath | a } -> Spago (GitEnv b) Boolean
+isBranch { ref, path } = do
+  showRef <- Except.runExceptT $ runGit_ [ "show-ref", "--verify", "refs/heads/" <> ref ] (Just path)
+  pure $ isRight showRef
+
 fetchRepo :: forall a b. { git :: String, ref :: String | a } -> FilePath -> Spago (GitEnv b) (Either (Array String) Unit)
 fetchRepo { git, ref } path = do
   repoExists <- FS.exists path

test/Spago/Install.purs

@@ -98,12 +98,20 @@ spec = Spec.around withTempDir do
 
     Spec.it "installs a package in the set from a commit hash" \{ spago } -> do
       spago [ "init" ] >>= shouldBeSuccess
-      writeConfigWithEither
+      -- The commit for `either` is for the `v6.1.0` release
+      writeConfigWithEither "af655a04ed2fd694b6688af39ee20d7907ad0763"
       spago [ "install", "either" ] >>= shouldBeSuccess
 
+    Spec.it "can install cached dependencies if offline" \{ spago } -> do
+      spago [ "init" ] >>= shouldBeSuccess
+      -- The commit for `either` is for the `v6.1.0` release, as with the previous test
+      writeConfigWithEither "af655a04ed2fd694b6688af39ee20d7907ad0763"
+      spago [ "install", "--offline", "either" ] >>= shouldBeSuccess
+
     Spec.it "can't install (uncached) dependencies if offline" \{ spago, fixture } -> do
       spago [ "init" ] >>= shouldBeSuccess
-      writeConfigWithEither
+      -- The commit for `either` is for the `v6.0.0` release
+      writeConfigWithEither "5fbe43cb88e3784c8625c938cadcf61506edb3f4"
       spago [ "install", "--offline", "either" ] >>= shouldBeFailureErr (fixture "offline.txt")
 
     Spec.it "installs a package version by branch name with / in it" \{ spago, testCwd } -> do
@@ -234,9 +242,8 @@ spec = Spec.around withTempDir do
       -- Check that the lockfile is back to the original
       checkFixture "spago.lock" (fixture "spago.lock")
 
-writeConfigWithEither :: Aff Unit
-writeConfigWithEither = do
-  -- The commit for `either` is for the `v6.1.0` release
+writeConfigWithEither :: String -> Aff Unit
+writeConfigWithEither ref = do
   let
     conf = Init.defaultConfig
       { name: mkPackageName "eee"
@@ -252,7 +259,7 @@ writeConfigWithEither = do
                 { extraPackages = Just $ Map.fromFoldable
                     [ Tuple (mkPackageName "either") $ Config.ExtraRemotePackage $ Config.RemoteGitPackage
                         { git: "https://github.com/purescript/purescript-either.git"
-                        , ref: "af655a04ed2fd694b6688af39ee20d7907ad0763"
+                        , ref
                         , subdir: Nothing
                         , dependencies: Just $ mkDependencies [ "control", "invariant", "maybe", "prelude" ]
                         }