nlohmann::json instance and JSON Schema for MemorySourceAccessor

Ericson2314 · Ericson2314 · commit 4d3d4b59a29f · 2025-10-30T16:17:45.000-04:00
As documented, this for file system objects themselves, since
`MemorySourceAccessor` is an implementation detail.
diff --git a/doc/manual/package.nix b/doc/manual/package.nix
@@ -34,6 +34,7 @@ mkMesonDerivation (finalAttrs: {
       (fileset.unions [
         ../../.version
         # For example JSON
+        ../../src/libutil-tests/data/memory-source-accessor
         ../../src/libutil-tests/data/hash
         ../../src/libstore-tests/data/content-address
         ../../src/libstore-tests/data/store-path
diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in
@@ -119,6 +119,7 @@
 - [Architecture and Design](architecture/architecture.md)
 - [Formats and Protocols](protocols/index.md)
   - [JSON Formats](protocols/json/index.md)
+    - [File System Object](protocols/json/file-system-object.md)
     - [Hash](protocols/json/hash.md)
     - [Content Address](protocols/json/content-address.md)
     - [Store Path](protocols/json/store-path.md)
diff --git a/doc/manual/source/protocols/json/file-system-object.md b/doc/manual/source/protocols/json/file-system-object.md
@@ -0,0 +1,21 @@
+{{#include file-system-object-v1-fixed.md}}
+
+## Examples
+
+### Simple
+
+```json
+{{#include schema/file-system-object-v1/simple.json}}
+```
+
+### Complex
+
+```json
+{{#include schema/file-system-object-v1/complex.json}}
+```
+
+<!-- need to convert YAML to JSON first
+## Raw Schema
+
+[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
+-->
diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed
@@ -11,6 +11,7 @@ s/\\`/`/g
 #
 # As we have more such relative links, more replacements of this nature
 # should appear below.
+s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
 s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
 s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
 s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build
@@ -9,6 +9,7 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false)
 json_schema_config = files('json-schema-for-humans-config.yaml')
 
 schemas = [
+  'file-system-object-v1',
   'hash-v1',
   'content-address-v1',
   'store-path-v1',
diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1 b/doc/manual/source/protocols/json/schema/file-system-object-v1
@@ -0,0 +1 @@
+../../../../../../src/libutil-tests/data/memory-source-accessor
diff --git a/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml b/doc/manual/source/protocols/json/schema/file-system-object-v1.yaml
@@ -0,0 +1,65 @@
+"$schema": http://json-schema.org/draft-04/schema#
+"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
+title: File System Object
+description: |
+  This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
+
+  The schema is recursive because file system objects contain other file system objects.
+type: object
+required: ["type"]
+properties:
+  type:
+    type: string
+    enum: ["regular", "symlink", "directory"]
+
+# Enforce conditional structure based on `type`
+anyOf:
+  - $ref: "#/$defs/regular"
+    required: ["type", "contents"]
+
+  - $ref: "#/$defs/symlink"
+    required: ["type", "target"]
+
+  - $ref: "#/$defs/directory"
+    required: ["type", "contents"]
+
+"$defs":
+  regular:
+    title: Regular File
+    required: ["contents"]
+    properties:
+      type:
+        const: "regular"
+      contents:
+        type: string
+        description: Base64-encoded file contents
+      executable:
+        type: boolean
+        description: Whether the file is executable.
+        default: false
+    additionalProperties: false
+
+  symlink:
+    title: Symbolic Link
+    required: ["target"]
+    properties:
+      type:
+        const: "symlink"
+      target:
+        type: string
+        description: Target path of the symlink.
+    additionalProperties: false
+
+  directory:
+    title: Directory
+    required: ["contents"]
+    properties:
+      type:
+        const: "directory"
+      contents:
+        type: object
+        description: |
+          Map of names to nested file system objects (for type=directory)
+        additionalProperties:
+          $ref: "#"
+    additionalProperties: false
diff --git a/src/json-schema-checks/file-system-object b/src/json-schema-checks/file-system-object
@@ -0,0 +1 @@
+../../src/libutil-tests/data/memory-source-accessor
diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build
@@ -20,6 +20,14 @@ schema_dir = meson.current_source_dir() / 'schema'
 
 # Get all example files
 schemas = [
+  {
+    'stem' : 'file-system-object',
+    'schema' : schema_dir / 'file-system-object-v1.yaml',
+    'files' : [
+      'simple.json',
+      'complex.json',
+    ],
+  },
   {
     'stem' : 'hash',
     'schema' : schema_dir / 'hash-v1.yaml',
diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix
@@ -20,6 +20,7 @@ mkMesonDerivation (finalAttrs: {
   fileset = lib.fileset.unions [
     ../../.version
     ../../doc/manual/source/protocols/json/schema
+    ../../src/libutil-tests/data/memory-source-accessor
     ../../src/libutil-tests/data/hash
     ../../src/libstore-tests/data/content-address
     ../../src/libstore-tests/data/store-path
diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc
@@ -1,5 +1,6 @@
 #include "nix/store/references.hh"
 #include "nix/store/path-references.hh"
+#include "nix/util/bytes.hh"
 #include "nix/util/memory-source-accessor.hh"
 
 #include <gtest/gtest.h>
@@ -104,21 +105,21 @@ TEST(references, scanForReferencesDeep)
                 // file1.txt: contains hash1
                 "file1.txt",
                 File::Regular{
-                    .contents = "This file references " + hash1 + " in its content",
+                    .contents = to_owned(as_bytes("This file references " + hash1 + " in its content")),
                 },
             },
             {
                 // file2.txt: contains hash2 and hash3
                 "file2.txt",
                 File::Regular{
-                    .contents = "Multiple refs: " + hash2 + " and also " + hash3,
+                    .contents = to_owned(as_bytes("Multiple refs: " + hash2 + " and also " + hash3)),
                 },
             },
             {
                 // file3.txt: contains no references
                 "file3.txt",
                 File::Regular{
-                    .contents = "This file has no store path references at all",
+                    .contents = to_owned(as_bytes("This file has no store path references at all")),
                 },
             },
             {
@@ -130,7 +131,7 @@ TEST(references, scanForReferencesDeep)
                             // subdir/file4.txt: contains hash1 again
                             "file4.txt",
                             File::Regular{
-                                .contents = "Subdirectory file with " + hash1,
+                                .contents = to_owned(as_bytes("Subdirectory file with " + hash1)),
                             },
                         },
                     },
@@ -140,7 +141,7 @@ TEST(references, scanForReferencesDeep)
                 // link1: a symlink that contains a reference in its target
                 "link1",
                 File::Symlink{
-                    .target = hash2 + "-target",
+                    .target = to_owned(as_bytes(hash2 + "-target")),
                 },
             },
         },
diff --git a/src/libutil-tests/data/memory-source-accessor/complex.json b/src/libutil-tests/data/memory-source-accessor/complex.json
@@ -0,0 +1,24 @@
+{
+  "contents": {
+    "bar": {
+      "contents": {
+        "baz": {
+          "contents": "Z29vZCBkYXksCg==",
+          "executable": true,
+          "type": "regular"
+        },
+        "quux": {
+          "target": "L292ZXIvdGhlcmU=",
+          "type": "symlink"
+        }
+      },
+      "type": "directory"
+    },
+    "foo": {
+      "contents": "aGVsbG8K",
+      "executable": false,
+      "type": "regular"
+    }
+  },
+  "type": "directory"
+}
diff --git a/src/libutil-tests/data/memory-source-accessor/simple.json b/src/libutil-tests/data/memory-source-accessor/simple.json
@@ -0,0 +1,5 @@
+{
+  "contents": "YXNkZg==",
+  "executable": false,
+  "type": "regular"
+}
diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc
@@ -224,42 +224,15 @@ TEST_F(GitTest, tree_sha256_write)
     });
 }
 
+namespace memory_source_accessor {
+
+extern ref<MemorySourceAccessor> exampleComplex();
+
+}
+
 TEST_F(GitTest, both_roundrip)
 {
-    using File = MemorySourceAccessor::File;
-
-    auto files = make_ref<MemorySourceAccessor>();
-    files->root = File::Directory{
-        .contents{
-            {
-                "foo",
-                File::Regular{
-                    .contents = "hello\n\0\n\tworld!",
-                },
-            },
-            {
-                "bar",
-                File::Directory{
-                    .contents =
-                        {
-                            {
-                                "baz",
-                                File::Regular{
-                                    .executable = true,
-                                    .contents = "good day,\n\0\n\tworld!",
-                                },
-                            },
-                            {
-                                "quux",
-                                File::Symlink{
-                                    .target = "/over/there",
-                                },
-                            },
-                        },
-                },
-            },
-        },
-    };
+    auto files = memory_source_accessor::exampleComplex();
 
     for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) {
         std::map<Hash, std::string> cas;
diff --git a/src/libutil-tests/memory-source-accessor.cc b/src/libutil-tests/memory-source-accessor.cc
@@ -0,0 +1,116 @@
+#include <string_view>
+
+#include "nix/util/bytes.hh"
+#include "nix/util/memory-source-accessor.hh"
+#include "nix/util/tests/json-characterization.hh"
+
+namespace nix {
+
+namespace memory_source_accessor {
+
+using File = MemorySourceAccessor::File;
+
+ref<MemorySourceAccessor> exampleSimple()
+{
+    auto sc = make_ref<MemorySourceAccessor>();
+    sc->root = File{File::Regular{
+        .executable = false,
+        .contents = to_owned(as_bytes("asdf")),
+    }};
+    return sc;
+}
+
+ref<MemorySourceAccessor> exampleComplex()
+{
+    auto files = make_ref<MemorySourceAccessor>();
+    files->root = File::Directory{
+        .contents{
+            {
+                "foo",
+                File::Regular{
+                    .contents = to_owned(as_bytes("hello\n\0\n\tworld!")),
+                },
+            },
+            {
+                "bar",
+                File::Directory{
+                    .contents =
+                        {
+                            {
+                                "baz",
+                                File::Regular{
+                                    .executable = true,
+                                    .contents = to_owned(as_bytes("good day,\n\0\n\tworld!")),
+                                },
+                            },
+                            {
+                                "quux",
+                                File::Symlink{
+                                    .target = to_owned(as_bytes("/over/there")),
+                                },
+                            },
+                        },
+                },
+            },
+        },
+    };
+    return files;
+}
+
+} // namespace memory_source_accessor
+
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+class MemorySourceAccessorTest : public virtual CharacterizationTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor";
+
+public:
+
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+};
+
+using nlohmann::json;
+
+struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest,
+                                      JsonCharacterizationTest<MemorySourceAccessor>,
+                                      ::testing::WithParamInterface<std::pair<std::string_view, MemorySourceAccessor>>
+{};
+
+TEST_P(MemorySourceAccessorJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    /* Cannot use `readJsonTest` because need to compare `root` field of
+       the source accessors for equality. */
+    readTest(Path{name} + ".json", [&](const auto & encodedRaw) {
+        auto encoded = json::parse(encodedRaw);
+        auto decoded = static_cast<MemorySourceAccessor>(encoded);
+        ASSERT_EQ(decoded.root, expected.root);
+    });
+}
+
+TEST_P(MemorySourceAccessorJsonTest, to_json)
+{
+    auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    MemorySourceAccessorJSON,
+    MemorySourceAccessorJsonTest,
+    ::testing::Values(
+        std::pair{
+            "simple",
+            *memory_source_accessor::exampleSimple(),
+        },
+        std::pair{
+            "complex",
+            *memory_source_accessor::exampleComplex(),
+        }));
+
+} // namespace nix
diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build
diff --git a/src/libutil/include/nix/util/bytes.hh b/src/libutil/include/nix/util/bytes.hh
diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh
diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build
diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc

Original file line number	Diff line number	Diff line change
@@ -11,6 +11,7 @@ s/\\`/`/g
`11`	`11`	`#`
`12`	`12`	`# As we have more such relative links, more replacements of this nature`
`13`	`13`	`# should appear below.`
	`14`	`+s^#/\$defs/\(regular\\|symlink\\|directory\)^In this schema^g`
`14`	`15`	s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
`15`	`16`	s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
`16`	`17`	s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+../../../../../../src/libutil-tests/data/memory-source-accessor`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+../../src/libutil-tests/data/memory-source-accessor`