Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce global git cache #1240

Closed
wants to merge 14 commits into from
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ Other improvements:
- Fix output truncation with `--json-errors`, many warnings and build failure (#1199)
- Update README with info about depending on a freshly added library
- Fixed globbing issue where `/.spago` behaves differently than `.spago` in `.gitignore`
- When a git repo is used for multiple dependencies, only clone it once
- Fixed empty output for `--verbose-stats` when there are no errors or warnings.
- Added support for `--package-set` options for `spago upgrade`.
- `spago repl` now writes a `.purs-repl` file, unless already there, containing `import Prelude`.
Expand Down
78 changes: 57 additions & 21 deletions src/Spago/Command/Fetch.purs
Original file line number Diff line number Diff line change
Expand Up @@ -443,28 +443,64 @@ toAllDependencies = foldl (Map.unionWith (\l _ -> l)) Map.empty
getGitPackageInLocalCache :: forall a. PackageName -> GitPackage -> Spago (Git.GitEnv a) Unit
getGitPackageInLocalCache name package = do
let localPackageLocation = Config.getPackageLocation name (GitPackage package)
tempDir <- mkTemp' (Just $ printJson Config.gitPackageCodec package)
logDebug $ "Cloning repo in " <> tempDir
Git.fetchRepo package tempDir >>= case _ of
Left err -> die err
Right _ -> do
logDebug $ "Repo cloned. Moving to " <> localPackageLocation
FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ]
FS.moveSync { src: tempDir, dst: localPackageLocation }

-- Note: the package might have been cloned with a tag, but we stick the commit hash in the lockfiles
-- so we need to make a copy to a location that has the commit hash too.
-- So we run getRef here and then do a copy if the ref is different than the original one
-- (since it might be a commit to start with)
logDebug $ "Checking if we need to copy the package to a commit hash location..."
Git.getRef (Just localPackageLocation) >>= case _ of
existsGitCache <- FS.exists gitCacheLocation
{ globallyCached, sourceDir } <-
if existsGitCache then do
logDebug $ "Using global git cache in " <> gitCacheLocation
pure { globallyCached: true, sourceDir: gitCacheLocation }
else do
-- Not cached in the global cache, so we download it and then see if it can be cached
tempDir <- mkTemp' (Just $ printJson Config.gitPackageCodec package)
logDebug $ "Cloning repo in " <> tempDir
fetchRes <- Git.fetchRepo package tempDir
case fetchRes of
Left err -> die err
Right ref -> do
when (ref /= package.ref) do
let commitHashLocation = Config.getPackageLocation name (GitPackage $ package { ref = ref })
logDebug $ "Copying the repo also to " <> commitHashLocation
FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ]
FS.copyTree { src: localPackageLocation, dst: commitHashLocation }
Right _ -> do
shouldCache <- not <$> Git.isBranch { ref: package.ref, path: tempDir }
if shouldCache then do
FS.mkdirp globalGitCache
try (FS.moveSync { src: tempDir, dst: gitCacheLocation }) >>= case _ of
Left _err -> logDebug $ "Couldn't move cloned repo to " <> gitCacheLocation
Right _ -> pure unit
pure { globallyCached: true, sourceDir: gitCacheLocation }
else do
pure { globallyCached: false, sourceDir: tempDir }

FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ]
if globallyCached then do
logDebug $ "Repo cloned. Copying to " <> localPackageLocation
FS.copyTree { src: sourceDir, dst: localPackageLocation }
else do
logDebug $ "Repo cloned. Moving to " <> localPackageLocation
FS.moveSync { src: sourceDir, dst: localPackageLocation }

-- Note: the package might have been cloned with a tag, but we stick the commit hash in the lockfiles
-- so we need to make a copy to a location that has the commit hash too.
-- So we run getRef here and then do a copy if the ref is different than the original one
-- (since it might be a commit to start with)
logDebug $ "Checking if we need to copy the package to a commit hash location..."
Git.getRef (Just localPackageLocation) >>= case _ of
Left err -> die err
Right ref -> do
when (ref /= package.ref) do
let commitHashLocation = Config.getPackageLocation name (GitPackage $ package { ref = ref })
logDebug $ "Copying the repo also to " <> commitHashLocation
FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ]
FS.copyTree { src: localPackageLocation, dst: commitHashLocation }
where
globalGitCacheElems =
[ Paths.globalCachePath
, "git"
, Config.fileSystemCharEscape package.git
]

globalGitCache :: FilePath
globalGitCache =
Path.concat $ globalGitCacheElems

gitCacheLocation :: FilePath
gitCacheLocation =
Path.concat $ globalGitCacheElems <> [ Config.fileSystemCharEscape package.ref ]

getPackageDependencies :: forall a. PackageName -> Package -> Spago (FetchEnv a) (Maybe (Map PackageName Range))
getPackageDependencies packageName package = case package of
Expand Down
7 changes: 7 additions & 0 deletions src/Spago/Git.purs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ module Spago.Git
, parseRemote
, pushTag
, tagCheckedOut
, isBranch
) where

import Spago.Prelude
Expand Down Expand Up @@ -48,6 +49,12 @@ runGit args cwd = ExceptT do
Right r -> Right r.stdout
Left r -> Left r.stderr

-- See https://stackoverflow.com/questions/18222634
isBranch :: forall a b. { ref :: String, path :: FilePath | a } -> Spago (GitEnv b) Boolean
isBranch { ref, path } = do
showRef <- Except.runExceptT $ runGit_ [ "show-ref", "--verify", "refs/heads/" <> ref ] (Just path)
pure $ isRight showRef

fetchRepo :: forall a b. { git :: String, ref :: String | a } -> FilePath -> Spago (GitEnv b) (Either (Array String) Unit)
fetchRepo { git, ref } path = do
repoExists <- FS.exists path
Expand Down
19 changes: 13 additions & 6 deletions test/Spago/Install.purs
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,20 @@ spec = Spec.around withTempDir do

Spec.it "installs a package in the set from a commit hash" \{ spago } -> do
spago [ "init" ] >>= shouldBeSuccess
writeConfigWithEither
-- The commit for `either` is for the `v6.1.0` release
writeConfigWithEither "af655a04ed2fd694b6688af39ee20d7907ad0763"
spago [ "install", "either" ] >>= shouldBeSuccess

Spec.it "can install cached dependencies if offline" \{ spago } -> do
spago [ "init" ] >>= shouldBeSuccess
-- The commit for `either` is for the `v6.1.0` release, as with the previous test
writeConfigWithEither "af655a04ed2fd694b6688af39ee20d7907ad0763"
spago [ "install", "--offline", "either" ] >>= shouldBeSuccess

Spec.it "can't install (uncached) dependencies if offline" \{ spago, fixture } -> do
spago [ "init" ] >>= shouldBeSuccess
writeConfigWithEither
-- The commit for `either` is for the `v6.0.0` release
writeConfigWithEither "5fbe43cb88e3784c8625c938cadcf61506edb3f4"
spago [ "install", "--offline", "either" ] >>= shouldBeFailureErr (fixture "offline.txt")

Spec.it "installs a package version by branch name with / in it" \{ spago, testCwd } -> do
Expand Down Expand Up @@ -234,9 +242,8 @@ spec = Spec.around withTempDir do
-- Check that the lockfile is back to the original
checkFixture "spago.lock" (fixture "spago.lock")

writeConfigWithEither :: Aff Unit
writeConfigWithEither = do
-- The commit for `either` is for the `v6.1.0` release
writeConfigWithEither :: String -> Aff Unit
writeConfigWithEither ref = do
let
conf = Init.defaultConfig
{ name: mkPackageName "eee"
Expand All @@ -252,7 +259,7 @@ writeConfigWithEither = do
{ extraPackages = Just $ Map.fromFoldable
[ Tuple (mkPackageName "either") $ Config.ExtraRemotePackage $ Config.RemoteGitPackage
{ git: "https://github.com/purescript/purescript-either.git"
, ref: "af655a04ed2fd694b6688af39ee20d7907ad0763"
, ref
, subdir: Nothing
, dependencies: Just $ mkDependencies [ "control", "invariant", "maybe", "prelude" ]
}
Expand Down
Loading