feat(selections): Add the ability to read integrity tag from selections

The `dub.selections.json` file can now contains integrity tags matching the
SRI specifications, allowing dub (and other tools, e.g. Nix) to better validate
that the downloaded archive matches the expected version.

However, Dub will not yet write the integrity tag, as it would result in a bad
user experience. Since `dub` tries hard to reuse packages present on the
filesystem, doing a `dub upgrade` could wipe the integrity tag (or not populate it)
if the package is already present on the system, an issue which would manifest itself
quite often for popular packages.

In order to solve this issue, we could store the integrity tag on disk,
however this can be done in another PR as such package metadata would be
useful for other purposes as well.

Author: Geod24

source/dub/dub.d

@@ -20,6 +20,7 @@ import dub.package_;
 import dub.packagemanager;
 import dub.packagesuppliers;
 import dub.project;
+import dub.recipe.selection : IntegrityTag;
 import dub.generators.generator;
 import dub.init;
 
@@ -726,8 +727,14 @@ class Dub {
 
 			FetchOptions fetchOpts;
 			fetchOpts |= (options & UpgradeOptions.preRelease) != 0 ? FetchOptions.usePrerelease : FetchOptions.none;
-			if (!pack) this.fetch(name, ver.version_, fetchOpts, defaultPlacementLocation, "getting selected version");
-			if ((options & UpgradeOptions.select) && name.toString() != m_project.rootPackage.name) {
+			if (!pack) {
+				auto tag = this.m_project.selections.getIntegrityTag(name);
+				const isInitTag = tag is typeof(tag).init;
+				this.fetch(name, ver.version_, fetchOpts, defaultPlacementLocation, tag);
+				if (isInitTag)
+					this.m_project.selections.selectVersion(name, ver.version_, tag);
+			}
+			else if ((options & UpgradeOptions.select) && name.toString() != m_project.rootPackage.name) {
 				if (!ver.repository.empty) {
 					m_project.selections.selectVersion(name, ver.repository);
 				} else if (ver.path.empty) {
@@ -940,6 +947,10 @@ class Dub {
 	 *   options = A set of options used for fetching / matching versions.
 	 *   location = Where to store the retrieved package. Default to the
 	 *              configured `defaultPlacementLocation`.
+	 *   tag = Dual-purpose `IntegrityTag` parameter. If it is specified
+	 *         (in its non-`init` state), then it will be used to verify
+	 *         the content of the download. If it is left in its init state,
+	 *         it will be populated with a sha512 checksum post-download.
 	 *   reason = Optionally, the reason for retriving this package.
 	 *            This is used only for logging.
 	 *
@@ -952,28 +963,48 @@ class Dub {
 	Package fetch(in PackageName name, in Version vers,
 		FetchOptions options = FetchOptions.none, string reason = "")
 	{
+		IntegrityTag empty;
 		return this.fetch(name, VersionRange(vers, vers), options,
-			this.defaultPlacementLocation, reason);
+			this.defaultPlacementLocation, empty, reason);
 	}
 
 	/// Ditto
 	Package fetch(in PackageName name, in Version vers, FetchOptions options,
 		PlacementLocation location, string reason = "")
+	{
+		IntegrityTag empty;
+		return this.fetch(name, VersionRange(vers, vers), options,
+			this.defaultPlacementLocation, empty, reason);
+	}
+
+	/// Ditto
+	Package fetch(in PackageName name, in Version vers, FetchOptions options,
+		PlacementLocation location, ref IntegrityTag integrity)
 	{
 		return this.fetch(name, VersionRange(vers, vers), options,
-			this.defaultPlacementLocation, reason);
+			this.defaultPlacementLocation, integrity, "getting selected version");
 	}
 
 	/// Ditto
 	Package fetch(in PackageName name, in VersionRange range = VersionRange.Any,
 		FetchOptions options = FetchOptions.none, string reason = "")
 	{
-		return this.fetch(name, range, options, this.defaultPlacementLocation, reason);
+		IntegrityTag empty;
+		return this.fetch(name, range, options, this.defaultPlacementLocation,
+			empty, reason);
 	}
 
 	/// Ditto
 	Package fetch(in PackageName name, in VersionRange range, FetchOptions options,
 		PlacementLocation location, string reason = "")
+	{
+		IntegrityTag empty;
+		return this.fetch(name, range, options, location, empty, reason);
+	}
+
+	/// Ditto
+	Package fetch(in PackageName name, in VersionRange range, FetchOptions options,
+		PlacementLocation location, ref IntegrityTag tag, string reason = "")
 	{
 		Json pinfo;
 		PackageSupplier supplier;
@@ -1030,6 +1061,12 @@ class Dub {
 			import std.zip : ZipException;
 
 			auto data = supplier.fetchPackage(name.main, range, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail?
+			if (tag !is IntegrityTag.init)
+				enforce(tag.matches(data), ("Hash of downloaded package does " ~
+					"not match integrity tag for %s@%s - This can happen if " ~
+					"the version has been re-tagged").format(name.main, range));
+			else
+				tag = IntegrityTag.make(data);
 			logDiagnostic("Placing to %s...", location.toString());
 
 			try {

source/dub/packagemanager.d

@@ -1291,7 +1291,7 @@ symlink_exit:
 			serialized["inheritable"] = true;
 		serialized["versions"] = Json.emptyObject;
 		foreach (p, dep; s.versions)
-			serialized["versions"][p] = dep.toJson(true);
+			serialized["versions"][p] = dep.toJsonDep();
 		return serialized;
 	}
 

source/dub/project.d

@@ -1944,14 +1944,19 @@ public class SelectedVersions {
 	void selectVersion(string package_id, Version version_)
 	{
 		const name = PackageName(package_id);
-		return this.selectVersion(name, version_);
+		return this.selectVersionInternal(name, Dependency(version_));
 	}
 
 	/// Ditto
-	void selectVersion(in PackageName name, Version version_)
+	void selectVersion(in PackageName name, Version version_, in IntegrityTag tag = IntegrityTag.init)
 	{
-		const dep = Dependency(version_);
-		this.selectVersionInternal(name, dep);
+		auto dep = SelectedDependency(Dependency(version_), tag);
+		if (auto pdep = name.main.toString() in this.m_selections.versions) {
+			if (*pdep == dep)
+				return;
+		}
+		this.m_selections.versions[name.main.toString()] = dep;
+		this.m_dirty = true;
 	}
 
 	/// Selects a certain path for a specific package.
@@ -2050,6 +2055,14 @@ public class SelectedVersions {
 		return m_selections.versions[name.main.toString()];
 	}
 
+	/// Returns: The `IntegrityTag` associated to the version, or `.init` if none
+	IntegrityTag getIntegrityTag(in PackageName name) const
+	{
+		if (auto ptr = name.main.toString() in this.m_selections.versions)
+			return (*ptr).integrity;
+		return typeof(return).init;
+	}
+
 	/** Stores the selections to disk.
 
 		The target file will be written in JSON format. Usually, `defaultFile`

source/dub/recipe/selection.d

@@ -9,12 +9,18 @@
 module dub.recipe.selection;
 
 import dub.dependency;
+import dub.internal.vibecompat.data.json : Json;
 import dub.internal.vibecompat.inet.path : NativePath;
 
 import dub.internal.configy.attributes;
 import dub.internal.dyaml.stdsumtype;
 
+import std.algorithm.iteration : each;
+import std.algorithm.searching : canFind;
 import std.exception;
+import std.format : format;
+import std.range : enumerate;
+import std.string : indexOf;
 
 deprecated("Use either `Selections!1` or `SelectionsFile` instead")
 public alias Selected = Selections!1;
@@ -125,21 +131,25 @@ public struct Selections (ushort Version)
 
 /// Wrapper around `SelectedDependency` to do deserialization but still provide
 /// a `Dependency` object to client code.
-private struct SelectedDependency
+package(dub) struct SelectedDependency
 {
     public Dependency actual;
     alias actual this;
+    public IntegrityTag integrity;
 
     /// Constructor, used in `fromConfig`
-    public this (inout(Dependency) dep) inout @safe pure nothrow @nogc
+    public this (inout(Dependency) dep, const IntegrityTag tag = IntegrityTag.init)
+         inout @safe pure nothrow @nogc
     {
         this.actual = dep;
+        this.integrity = tag;
     }
 
     /// Allow external code to assign to this object as if it was a `Dependency`
     public ref SelectedDependency opAssign (Dependency dep) return pure nothrow @nogc
     {
         this.actual = dep;
+        this.integrity = IntegrityTag.init;
         return this;
     }
 
@@ -157,16 +167,41 @@ private struct SelectedDependency
             assert(d.version_.length);
             if (d.repository.length)
                 return SelectedDependency(Dependency(Repository(d.repository, d.version_)));
-            return SelectedDependency(Dependency(Version(d.version_)));
+            return SelectedDependency(Dependency(Version(d.version_)), d.integrity);
         }
     }
 
+    /// Serializes a selected version to JSON for `dub.selections.json`
+    public Json toJsonDep () const {
+        version (none) {
+            // The following is not yet enabled, because we're currently only
+            // able to get an integrity tag value when the package is first
+            // downloaded. This is problematic as most of the time, we try
+            // to reuse packages, and most common use of `dub upgrade` would
+            // make the integrity tag flip between empty or not.
+            // However, with this code enabled, one may get an integrity tag
+            // written to their `dub.selections.json` under two conditions:
+            // 1) The package is not present on the file system;
+            // 2) The package is upgraded (e.g. `dub upgrade` would normally trigger);
+            if (this.integrity.value.length && this.actual.isExactVersion()) {
+                const vers = this.actual.version_();
+                Json result = Json.emptyObject;
+                result["version"] = Json(vers.toString());
+                result["integrity"] = Json(
+                    "%s-%s".format(this.integrity.algorithm, this.integrity.value));
+                return result;
+            }
+        }
+        return this.actual.toJson(true);
+    }
+
 	/// In-file representation of a dependency as permitted in `dub.selections.json`
 	private struct YAMLFormat
 	{
 		@Optional @Name("version") string version_;
 		@Optional string path;
 		@Optional string repository;
+		@Optional IntegrityTag integrity;
 
 		public void validate () const scope @safe pure
 		{
@@ -178,10 +213,134 @@ private struct SelectedDependency
 				"Cannot provide a `path` dependency if a `version` dependency is used");
 			enforce(!this.repository.length || this.version_.length,
 				"Cannot provide a `repository` dependency without a `version`");
+			enforce(!this.integrity.algorithm.length || (!this.path.length && !this.repository.length),
+				"`integrity` property is only supported for `version` dependencies");
 		}
 	}
 }
 
+/**
+ * A subresource integrity declaration
+ *
+ * Implement the SRI (Subresource Integrity) standard, used to validate that
+ * a given dependency is of the expected version.
+ *
+ * One may get an integrity tag in base64 using openssl:
+ * ```
+ * $ cat vibe.d-0.10.1.zip |  openssl dgst -binary -sha512 | base64
+ * vwQ9tYTjLb981j41+3GZZUgKXm/5PlKpmY2bplRSUM8ajL03++LGm/TcfFFarJrHex8CTb5ZLWdi
+ * Y1fFAOSkSw==
+ * ```
+ *
+ * See_Also:
+ *   https://w3c.github.io/webappsec-subresource-integrity/#the-integrity-attribute
+ */
+public struct IntegrityTag
+{
+	/// The hash function to use
+	public string algorithm;
+	/// The value of the digest computed with `algorithm`, base64-encoded
+	public string value;
+
+	/// Parses a string representation as an `IntegrityTag`
+	public this (string value)
+	{
+		auto sep = indexOf(value, '-');
+		enforce(sep > 0, `Expected a string in the form 'hash-algorithm "-" base64-value', e.g. 'sha512-...'`);
+		this.algorithm = value[0 .. sep];
+		this.value = value[sep + 1 .. $];
+		switch (this.algorithm) {
+		case "sha512":
+			enforce(this.value.length == 88,
+				"Excepted a base64-encoded sha512 digest of 88 characters, not %s"
+				.format(this.value.length));
+			break;
+		case "sha384":
+			enforce(this.value.length == 64,
+				"Excepted a base64-encoded sha384 digest of 64 characters, not %s"
+				.format(this.value.length));
+			break;
+		case "sha256":
+			enforce(this.value.length == 40,
+				"Excepted a base64-encoded sha256 digest of 40 characters, not %s"
+				.format(this.value.length));
+			break;
+		default:
+			throw new Exception("Algorithm '" ~ this.algorithm ~
+				"' is not supported, expected one of: 'sha512', 'sha384', 'sha256'");
+		}
+		this.value.enumerate.each!((size_t idx, dchar c) {
+			enforce("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".canFind(c),
+				"Expected digest to be base64 encoded, found non-base64 character '%s' at index '%s'"
+				.format(c, idx));
+		});
+	}
+
+    /// Internal constructor for `IntegrityTag.make`
+    public this (string algorithm, string value) inout @safe pure nothrow @nogc {
+        this.algorithm = algorithm;
+        this.value = value;
+    }
+
+    /**
+     * Verify if the data passed in parameter matches this `IntegrityTag`
+     *
+     * Params:
+     *   data = The content of the archive to check for a match
+     *
+     * Returns:
+     *   Whether the hash of `data` (using `this.algorithm)` matches
+     *   the value that is `base64` encoded.
+     */
+    public bool matches (in ubyte[] data) const @safe pure {
+        import std.base64;
+        import std.digest.sha;
+
+        ubyte[64] buffer; // 32, 48, or 64 bytes used
+        auto decoded = Base64.decode(this.value, buffer[]);
+        switch (this.algorithm) {
+            case "sha512":
+                return sha512Of(data) == decoded;
+            case "sha384":
+                return sha384Of(data) == decoded;
+            case "sha256":
+                return sha256Of(data) == decoded;
+            default:
+                assert(0, "An `IntegrityTag` with non-supported algorithm was created: " ~ this.algorithm);
+        }
+    }
+
+    /**
+     * Build and returns an `IntegrityTag`
+     *
+     * This is a convenience function to build an `IntegrityTag` from the
+     * archive data. Use sha512 by default.
+     *
+     * Params:
+     *   data = The content of the archive to check hash into a digest
+     *   algorithm = One of `sha256`, `sha384`, `sha512`. Default to the latter.
+     *
+     * Returns:
+     *   A populated `IntegrityTag`.
+     */
+    public static IntegrityTag make (in ubyte[] data, string algorithm = "sha512")
+	    @safe pure {
+        import std.base64;
+        import std.digest.sha;
+
+        switch (algorithm) {
+            case "sha512":
+                return IntegrityTag(algorithm, Base64.encode(sha512Of(data)));
+            case "sha384":
+                return IntegrityTag(algorithm, Base64.encode(sha384Of(data)));
+            case "sha256":
+                return IntegrityTag(algorithm, Base64.encode(sha256Of(data)));
+            default:
+                assert(0, "`IntegrityTag.make` was called with non-supported algorithm: " ~ algorithm);
+        }
+    }
+}
+
 // Ensure we can read all type of dependencies
 unittest
 {
@@ -191,6 +350,10 @@ unittest
     "fileVersion": 1,
     "versions": {
         "simple": "1.5.6",
+        "complex": { "version": "1.2.3" },
+        "digest": { "version": "1.2.3", "integrity": "sha256-abcdefghijklmnopqrstuvwxyz0123456789+/==" },
+        "digest1": { "version": "1.2.3", "integrity": "sha384-Li9vy3DqF8tnTXuiaAJuML3ky+er10rcgNR/VqsVpcw+ThHmYcwiB1pbOxEbzJr7" },
+        "digest2": { "version": "1.2.3", "integrity": "sha512-Q2bFTOhEALkN8hOms2FKTDLy7eugP2zFZ1T8LCvX42Fp3WoNr3bjZSAHeOsHrbV1Fu9/A0EzCinRE7Af1ofPrw==" },
         "branch": "~master",
         "branch2": "~main",
         "path": { "path": "../some/where" },
@@ -205,8 +368,12 @@ unittest
         (s) { assert(0); return Selections!(1).init; },
     );
     assert(!s.inheritable);
-    assert(s.versions.length == 5);
+    assert(s.versions.length == 9);
     assert(s.versions["simple"]     == Dependency(Version("1.5.6")));
+    assert(s.versions["complex"]    == Dependency(Version("1.2.3")));
+    assert(s.versions["digest"]     == Dependency(Version("1.2.3")));
+    assert(s.versions["digest1"]    == Dependency(Version("1.2.3")));
+    assert(s.versions["digest2"]    == Dependency(Version("1.2.3")));
     assert(s.versions["branch"]     == Dependency(Version("~master")));
     assert(s.versions["branch2"]    == Dependency(Version("~main")));
     assert(s.versions["path"]       == Dependency(NativePath("../some/where")));