Skip to content

Commit 7bcc712

Browse files
edolstraedolstra
authored
Merge pull request #278 from DeterminateSystems/detsys/eelcodolstra/fh-1150-legacy-eol-handling-v2
Backward compatibility hack for Nix < 2.20 Git inputs using git-archive
2 parents 3765b1a + 99b4669 commit 7bcc712

File tree

11 files changed

+245
-47
lines changed

11 files changed

+245
-47
lines changed

‎src/libfetchers-tests/git-utils.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ TEST_F(GitUtilsTest, sink_basic)
9292
// sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
9393

9494
auto result = repo->dereferenceSingletonDirectory(sink->flush());
95-
auto accessor = repo->getAccessor(result, false, getRepoName());
95+
auto accessor = repo->getAccessor(result, {}, getRepoName());
9696
auto entries = accessor->readDirectory(CanonPath::root);
9797
ASSERT_EQ(entries.size(), 5u);
9898
ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");

‎src/libfetchers/fetchers.cc‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,12 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
327327
auto makeStoreAccessor = [&]() -> std::pair<ref<SourceAccessor>, Input> {
328328
auto accessor = make_ref<SubstitutedSourceAccessor>(ref{store->getFSAccessor(*storePath)});
329329

330-
accessor->fingerprint = getFingerprint(store);
330+
// FIXME: use the NAR hash for fingerprinting Git trees that have a .gitattributes file, since we don't know if
331+
// we used `git archive` or libgit2 to fetch it.
332+
accessor->fingerprint = getType() == "git" && accessor->pathExists(CanonPath(".gitattributes"))
333+
? std::optional(storePath->hashPart())
334+
: getFingerprint(store);
335+
cachedFingerprint = accessor->fingerprint;
331336

332337
// Store a cache entry for the substituted tree so later fetches
333338
// can reuse the existing nar instead of copying the unpacked
@@ -357,10 +362,10 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
357362
try {
358363
auto [accessor, result] = scheme->getAccessor(settings, store, *this);
359364

360-
if (!accessor->fingerprint)
361-
accessor->fingerprint = result.getFingerprint(store);
365+
if (auto fp = accessor->getFingerprint(CanonPath::root).second)
366+
result.cachedFingerprint = *fp;
362367
else
363-
result.cachedFingerprint = accessor->fingerprint;
368+
accessor->fingerprint = result.getFingerprint(store);
364369

365370
return {accessor, std::move(result)};
366371
} catch (Error & e) {

‎src/libfetchers/git-utils.cc‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -550,14 +550,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
550550
}
551551

552552
/**
553-
* A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
553+
* A 'GitSourceAccessor' with no regard for export-ignore.
554554
*/
555-
ref<GitSourceAccessor> getRawAccessor(const Hash & rev, bool smudgeLfs = false);
555+
ref<GitSourceAccessor> getRawAccessor(const Hash & rev, const GitAccessorOptions & options);
556556

557557
ref<SourceAccessor>
558-
getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) override;
558+
getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) override;
559559

560-
ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
560+
ref<SourceAccessor>
561+
getAccessor(const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError e) override;
561562

562563
ref<GitFileSystemObjectSink> getFileSystemObjectSink() override;
563564

@@ -700,7 +701,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
700701

701702
Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
702703
{
703-
auto accessor = getAccessor(treeHash, false, "");
704+
auto accessor = getAccessor(treeHash, {}, "");
704705

705706
fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
706707

@@ -737,28 +738,35 @@ ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create,
737738
return make_ref<GitRepoImpl>(path, create, bare);
738739
}
739740

741+
std::string GitAccessorOptions::makeFingerprint(const Hash & rev) const
742+
{
743+
return "git:" + rev.gitRev() + (exportIgnore ? ";e" : "") + (smudgeLfs ? ";l" : "");
744+
}
745+
740746
/**
741747
* Raw git tree input accessor.
742748
*/
743-
744749
struct GitSourceAccessor : SourceAccessor
745750
{
746751
struct State
747752
{
748753
ref<GitRepoImpl> repo;
749754
Object root;
750755
std::optional<lfs::Fetch> lfsFetch = std::nullopt;
756+
GitAccessorOptions options;
751757
};
752758

753759
Sync<State> state_;
754760

755-
GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
761+
GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, const GitAccessorOptions & options)
756762
: state_{State{
757763
.repo = repo_,
758764
.root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
759-
.lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
765+
.lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
766+
.options = options,
760767
}}
761768
{
769+
fingerprint = options.makeFingerprint(rev);
762770
}
763771

764772
std::string readBlob(const CanonPath & path, bool symlink)
@@ -1307,26 +1315,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
13071315
}
13081316
};
13091317

1310-
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, bool smudgeLfs)
1318+
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
13111319
{
13121320
auto self = ref<GitRepoImpl>(shared_from_this());
1313-
return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
1321+
return make_ref<GitSourceAccessor>(self, rev, options);
13141322
}
13151323

13161324
ref<SourceAccessor>
1317-
GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs)
1325+
GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix)
13181326
{
13191327
auto self = ref<GitRepoImpl>(shared_from_this());
1320-
ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
1328+
ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, options);
13211329
rawGitAccessor->setPathDisplay(std::move(displayPrefix));
1322-
if (exportIgnore)
1330+
if (options.exportIgnore)
13231331
return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
13241332
else
13251333
return rawGitAccessor;
13261334
}
13271335

1328-
ref<SourceAccessor>
1329-
GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
1336+
ref<SourceAccessor> GitRepoImpl::getAccessor(
1337+
const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError)
13301338
{
13311339
auto self = ref<GitRepoImpl>(shared_from_this());
13321340
ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create(
@@ -1336,7 +1344,7 @@ GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllow
13361344
boost::unordered_flat_set<CanonPath>{CanonPath::root},
13371345
std::move(makeNotAllowedError))
13381346
.cast<SourceAccessor>();
1339-
if (exportIgnore)
1347+
if (options.exportIgnore)
13401348
fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
13411349
return fileAccessor;
13421350
}
@@ -1351,7 +1359,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
13511359
/* Read the .gitmodules files from this revision. */
13521360
CanonPath modulesFile(".gitmodules");
13531361

1354-
auto accessor = getAccessor(rev, exportIgnore, "");
1362+
auto accessor = getAccessor(rev, {.exportIgnore = exportIgnore}, "");
13551363
if (!accessor->pathExists(modulesFile))
13561364
return {};
13571365

@@ -1368,7 +1376,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
13681376

13691377
std::vector<std::tuple<Submodule, Hash>> result;
13701378

1371-
auto rawAccessor = getRawAccessor(rev);
1379+
auto rawAccessor = getRawAccessor(rev, {});
13721380

13731381
for (auto & submodule : parseSubmodules(pathTemp)) {
13741382
/* Filter out .gitmodules entries that don't exist or are not

‎src/libfetchers/git.cc‎

Lines changed: 122 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include "nix/util/json-utils.hh"
1717
#include "nix/util/archive.hh"
1818
#include "nix/util/mounted-source-accessor.hh"
19+
#include "nix/fetchers/fetch-to-store.hh"
1920

2021
#include <regex>
2122
#include <string.h>
@@ -637,8 +638,72 @@ struct GitInputScheme : InputScheme
637638
return shallow || input.getRevCount().has_value();
638639
}
639640

641+
GitAccessorOptions getGitAccessorOptions(const Input & input) const
642+
{
643+
return GitAccessorOptions{
644+
.exportIgnore = getExportIgnoreAttr(input),
645+
.smudgeLfs = getLfsAttr(input),
646+
.submodules = getSubmodulesAttr(input),
647+
};
648+
}
649+
650+
/**
651+
* Get a `SourceAccessor` for the given Git revision using Nix < 2.20 semantics, i.e. using `git archive` or `git
652+
* checkout`.
653+
*/
654+
ref<SourceAccessor> getLegacyGitAccessor(
655+
Store & store,
656+
RepoInfo & repoInfo,
657+
const std::filesystem::path & repoDir,
658+
const Hash & rev,
659+
GitAccessorOptions & options) const
660+
{
661+
auto tmpDir = createTempDir();
662+
AutoDelete delTmpDir(tmpDir, true);
663+
664+
auto storePath =
665+
options.submodules
666+
? [&]() {
667+
// Nix < 2.20 used `git checkout` for repos with submodules.
668+
runProgram2({.program = "git", .args = {"init", tmpDir}});
669+
runProgram2({.program = "git", .args = {"-C", tmpDir, "remote", "add", "origin", repoDir}});
670+
runProgram2({.program = "git", .args = {"-C", tmpDir, "fetch", "origin", rev.gitRev()}});
671+
runProgram2({.program = "git", .args = {"-C", tmpDir, "checkout", rev.gitRev()}});
672+
PathFilter filter = [&](const Path & path) { return baseNameOf(path) != ".git"; };
673+
return store.addToStore(
674+
"source",
675+
{getFSSourceAccessor(), CanonPath(tmpDir)},
676+
ContentAddressMethod::Raw::NixArchive,
677+
HashAlgorithm::SHA256,
678+
{},
679+
filter);
680+
}()
681+
: [&]() {
682+
// Nix < 2.20 used `git archive` for repos without submodules.
683+
options.exportIgnore = true;
684+
685+
auto source = sinkToSource([&](Sink & sink) {
686+
runProgram2(
687+
{.program = "git",
688+
.args = {"-C", repoDir, "--git-dir", repoInfo.gitDir, "archive", rev.gitRev()},
689+
.standardOut = &sink});
690+
});
691+
692+
unpackTarfile(*source, tmpDir);
693+
694+
return store.addToStore("source", {getFSSourceAccessor(), CanonPath(tmpDir)});
695+
}();
696+
697+
auto accessor = store.getFSAccessor(storePath);
698+
699+
accessor->fingerprint = options.makeFingerprint(rev) + ";legacy";
700+
701+
return ref{accessor};
702+
}
703+
640704
std::pair<ref<SourceAccessor>, Input>
641705
getAccessorFromCommit(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
706+
642707
{
643708
assert(!repoInfo.workdirInfo.isDirty);
644709

@@ -779,17 +844,59 @@ struct GitInputScheme : InputScheme
779844

780845
verifyCommit(input, repo);
781846

782-
bool exportIgnore = getExportIgnoreAttr(input);
783-
bool smudgeLfs = getLfsAttr(input);
784-
auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string(true) + "»", smudgeLfs);
847+
auto options = getGitAccessorOptions(input);
848+
849+
auto expectedNarHash = input.getNarHash();
850+
851+
auto accessor = repo->getAccessor(rev, options, "«" + input.to_string(true) + "»");
852+
853+
if (settings.nix219Compat && !options.smudgeLfs && accessor->pathExists(CanonPath(".gitattributes"))) {
854+
/* Use Nix 2.19 semantics to generate locks, but if a NAR hash is specified, support Nix >= 2.20 semantics
855+
* as well. */
856+
warn("Using Nix 2.19 semantics to export Git repository '%s'.", input.to_string());
857+
auto accessorModern = accessor;
858+
accessor = getLegacyGitAccessor(*store, repoInfo, repoDir, rev, options);
859+
if (expectedNarHash) {
860+
auto narHashLegacy =
861+
fetchToStore2(settings, *store, {accessor}, FetchMode::DryRun, input.getName()).second;
862+
if (expectedNarHash != narHashLegacy) {
863+
auto narHashModern =
864+
fetchToStore2(settings, *store, {accessorModern}, FetchMode::DryRun, input.getName()).second;
865+
if (expectedNarHash == narHashModern)
866+
accessor = accessorModern;
867+
}
868+
}
869+
} else {
870+
/* Backward compatibility hack for locks produced by Nix < 2.20 that depend on Nix applying Git filters,
871+
* `export-ignore` or `export-subst`. Nix >= 2.20 doesn't do those, so we may get a NAR hash mismatch. If
872+
* that happens, try again using `git archive`. */
873+
auto narHashNew = fetchToStore2(settings, *store, {accessor}, FetchMode::DryRun, input.getName()).second;
874+
if (expectedNarHash && accessor->pathExists(CanonPath(".gitattributes"))) {
875+
if (expectedNarHash != narHashNew) {
876+
auto accessorLegacy = getLegacyGitAccessor(*store, repoInfo, repoDir, rev, options);
877+
auto narHashLegacy =
878+
fetchToStore2(settings, *store, {accessorLegacy}, FetchMode::DryRun, input.getName()).second;
879+
if (expectedNarHash == narHashLegacy) {
880+
warn(
881+
"Git input '%s' specifies a NAR hash '%s' that was created by Nix < 2.20.\n"
882+
"Nix >= 2.20 does not apply Git filters, `export-ignore` and `export-subst` by default, which changes the NAR hash.\n"
883+
"Please update the NAR hash to '%s'.",
884+
input.to_string(),
885+
expectedNarHash->to_string(HashFormat::SRI, true),
886+
narHashNew.to_string(HashFormat::SRI, true));
887+
accessor = accessorLegacy;
888+
}
889+
}
890+
}
891+
}
785892

786893
/* If the repo has submodules, fetch them and return a mounted
787894
input accessor consisting of the accessor for the top-level
788895
repo and the accessors for the submodules. */
789-
if (getSubmodulesAttr(input)) {
896+
if (options.submodules) {
790897
std::map<CanonPath, nix::ref<SourceAccessor>> mounts;
791898

792-
for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) {
899+
for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, options.exportIgnore)) {
793900
auto resolved = repo->resolveSubmoduleUrl(submodule.url);
794901
debug(
795902
"Git submodule %s: %s %s %s -> %s",
@@ -812,9 +919,9 @@ struct GitInputScheme : InputScheme
812919
}
813920
}
814921
attrs.insert_or_assign("rev", submoduleRev.gitRev());
815-
attrs.insert_or_assign("exportIgnore", Explicit<bool>{exportIgnore});
922+
attrs.insert_or_assign("exportIgnore", Explicit<bool>{options.exportIgnore});
816923
attrs.insert_or_assign("submodules", Explicit<bool>{true});
817-
attrs.insert_or_assign("lfs", Explicit<bool>{smudgeLfs});
924+
attrs.insert_or_assign("lfs", Explicit<bool>{options.smudgeLfs});
818925
attrs.insert_or_assign("allRefs", Explicit<bool>{true});
819926
auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs));
820927
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store);
@@ -823,8 +930,10 @@ struct GitInputScheme : InputScheme
823930
}
824931

825932
if (!mounts.empty()) {
933+
auto newFingerprint = accessor->getFingerprint(CanonPath::root).second->append(";s");
826934
mounts.insert_or_assign(CanonPath::root, accessor);
827935
accessor = makeMountedSourceAccessor(std::move(mounts));
936+
accessor->fingerprint = newFingerprint;
828937
}
829938
}
830939

@@ -848,7 +957,7 @@ struct GitInputScheme : InputScheme
848957
auto exportIgnore = getExportIgnoreAttr(input);
849958

850959
ref<SourceAccessor> accessor =
851-
repo->getAccessor(repoInfo.workdirInfo, exportIgnore, makeNotAllowedError(repoPath));
960+
repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath));
852961

853962
/* If the repo has submodules, return a mounted input accessor
854963
consisting of the accessor for the top-level repo and the
@@ -942,13 +1051,12 @@ struct GitInputScheme : InputScheme
9421051

9431052
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
9441053
{
945-
auto makeFingerprint = [&](const Hash & rev) {
946-
return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "")
947-
+ (getLfsAttr(input) ? ";l" : "");
948-
};
1054+
auto options = getGitAccessorOptions(input);
9491055

9501056
if (auto rev = input.getRev())
951-
return makeFingerprint(*rev);
1057+
// FIXME: this can return a wrong fingerprint for the legacy (`git archive`) case, since we don't know here
1058+
// whether to append the `;legacy` suffix or not.
1059+
return options.makeFingerprint(*rev);
9521060
else {
9531061
auto repoInfo = getRepoInfo(input);
9541062
if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) {
@@ -964,7 +1072,7 @@ struct GitInputScheme : InputScheme
9641072
writeString("deleted:", hashSink);
9651073
writeString(file.abs(), hashSink);
9661074
}
967-
return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev))
1075+
return options.makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev))
9681076
+ ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false);
9691077
}
9701078
return std::nullopt;

‎src/libfetchers/github.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ struct GitArchiveInputScheme : InputScheme
326326
input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
327327

328328
auto accessor =
329-
settings.getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string(true) + "»");
329+
settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string(true) + "»");
330330

331331
if (!settings.trustTarballsFromGitForges)
332332
// FIXME: computing the NAR hash here is wasteful if
@@ -350,7 +350,7 @@ struct GitArchiveInputScheme : InputScheme
350350
std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
351351
{
352352
if (auto rev = input.getRev())
353-
return rev->gitRev();
353+
return "github:" + rev->gitRev();
354354
else
355355
return std::nullopt;
356356
}

0 commit comments

Comments
 (0)