feat(tree): staged allowed types

.changeset/curly-bikes-train.md

@@ -0,0 +1,94 @@
+---
+"fluid-framework": minor
+"@fluidframework/tree": minor
+"__section": feature
+---
+Adds staged allowed types to SchemaFactoryAlpha
+
+This adds the `staged` API to [`SchemaFactoryAlpha`](https://fluidframework.com/docs/api/fluid-framework/schemafactoryalpha-class).
+Staged allowed types can be used for schema evolution to add members to an [`AllowedTypes`](https://fluidframework.com/docs/api/fluid-framework/allowedtypes-typealias) while supporting cross version collaboration.
+
+Staged allowed types are [allowed types](https://fluidframework.com/docs/api/fluid-framework/allowedtypes-typealias) that can be upgraded by [schema upgrades}(https://fluidframework.com/docs/api/fluid-framework/treeview-interface#upgradeschema-methodsignature).
+Before being upgraded, any attempt to insert or move a node to a location which requires its type to be upgraded to be valid will throw an error.
+
+To enable this feature, [schema validation](https://fluidframework.com/docs/api/fluid-framework/treeviewconfiguration-class#enableschemavalidation-property) is now performed by default when editing the tree.
+
+To add a new member to an `AllowedTypes`, add the type wrapped by `staged`.
+For example, migrating an array which previously supported only numbers to support both numbers and strings would start by deploying a version of the app using `staged`:
+```typescript
+class TestArray extends schemaFactoryAlpha.arrayAlpha("TestArray", [SchemaFactoryAlpha.number, SchemaFactoryAlpha.staged(SchemaFactoryAlpha.string)]) {}
+```
+
+Once enough clients have this code update, it is safe to allow writing strings to the array.
+To allow writing strings to the array, a code change must be made to remove the staged annotation:
+```typescript
+class TestArray extends schemaFactoryAlpha.arrayAlpha("TestArray", [schemaFactoryAlpha.number, schemaFactoryAlpha.string]) {}
+```
+
+Then when opening old documents [upgradeSchema](https://fluidframework.com/docs/api/fluid-framework/treeview-interface#upgradeschema-methodsignature) is used to upgrade the stored schema:
+```typescript
+view.upgradeSchema()
+```
+
+In the future, SharedTree may add an API that allows staged allowed types to be upgraded via a runtime schema upgrade so that the type can be more easily deployed using a configuration flag change rather than a code change.
+
+Below is a full example of how the schema migration process works. This can also be found in our [tests](https://github.com/jenn-le/FluidFramework/blob/main/packages/dds/tree/src/test/simple-tree/api/stagedSchemaUpgrade.spec.ts).
+
+```typescript
+// schema A: only number allowed
+const schemaA = factory.optional([SchemaFactoryAlpha.number]);
+
+// schema B: number or string (string is staged)
+const schemaB = factory.optional([
+	SchemaFactoryAlpha.number,
+	factory.staged(SchemaFactoryAlpha.string),
+]);
+
+// schema C: number or string, both fully allowed
+const schemaC = factory.optional([SchemaFactoryAlpha.number, SchemaFactoryAlpha.string]);
+
+const provider = new TestTreeProviderLite(3);
+
+// initialize with schema A
+const configA = new TreeViewConfiguration({
+	schema: schemaA,
+});
+const viewA = provider.trees[0].viewWith(configA);
+viewA.initialize(5);
+provider.synchronizeMessages();
+
+assert.deepEqual(viewA.root, 5);
+
+// view second tree with schema B
+const configB = new TreeViewConfiguration({
+	schema: schemaB,
+});
+const viewB = provider.trees[1].viewWith(configB);
+// check that we can read the tree
+assert.deepEqual(viewB.root, 5);
+// upgrade to schema B
+viewB.upgradeSchema();
+provider.synchronizeMessages();
+
+// check view A can read the document
+assert.deepEqual(viewA.root, 5);
+// check view B cannot write strings to the root
+assert.throws(() => {
+	viewB.root = "test";
+});
+
+// view third tree with schema C
+const configC = new TreeViewConfiguration({
+	schema: schemaC,
+});
+const viewC = provider.trees[2].viewWith(configC);
+// upgrade to schema C and change the root to a string
+viewC.upgradeSchema();
+viewC.root = "test";
+provider.synchronizeMessages();
+
+// view A is now incompatible with the stored schema
+assert.throws(viewA.canView, false);
+assert.deepEqual(viewB.root, "test");
+assert.deepEqual(viewC.root, "test");
+```

packages/dds/tree/.vscode/settings.json

@@ -23,7 +23,6 @@
 		"contravariantly",
 		"covariantly",
 		"deprioritized",
-		"enablable",
 		"endregion",
 		"fluidframework",
 		"insertable",

packages/dds/tree/api-report/tree.alpha.api.md

@@ -18,6 +18,7 @@ export function adaptEnum<TScope extends string, const TEnum extends Record<stri
 // @alpha
 export interface AllowedTypeMetadata {
     readonly custom?: unknown;
+    readonly stagedSchemaUpgrade?: SchemaUpgrade;
 }
 
 // @public @system
@@ -849,6 +850,7 @@ export class SchemaFactoryAlpha<out TScope extends string | undefined = string |
     static readonly requiredRecursive: <const T extends System_Unsafe.ImplicitAllowedTypesUnsafe, const TCustomMetadata = unknown>(t: T, props?: Omit<FieldPropsAlpha_2<TCustomMetadata>, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe_2<FieldKind_2.Required, T, TCustomMetadata>;
     readonly requiredRecursive: <const T extends System_Unsafe.ImplicitAllowedTypesUnsafe, const TCustomMetadata = unknown>(t: T, props?: Omit<FieldPropsAlpha_2<TCustomMetadata>, "defaultProvider"> | undefined) => FieldSchemaAlphaUnsafe_2<FieldKind_2.Required, T, TCustomMetadata>;
     scopedFactory<const T extends TName, TNameInner extends number | string = string>(name: T): SchemaFactoryAlpha<ScopedSchemaName<TScope, T>, TNameInner>;
+    static staged<const T extends TreeNodeSchema>(t: T | AnnotatedAllowedType<T>): AnnotatedAllowedType<T>;
 }
 
 // @alpha
@@ -876,6 +878,10 @@ export interface SchemaStatics {
     readonly string: LeafSchema<"string", string>;
 }
 
+// @alpha @sealed
+export class SchemaUpgrade {
+}
+
 // @alpha @input
 export interface SchemaValidationFunction<Schema extends TSchema> {
     check(data: unknown): data is Static<Schema>;

packages/dds/tree/src/core/schema-stored/schema.ts

@@ -125,9 +125,10 @@ export interface SchemaPolicy {
 	readonly fieldKinds: ReadonlyMap<FieldKindIdentifier, FieldKindData>;
 
 	/**
-	 * If true, new content inserted into the tree should be validated against the stored schema.
+	 * This is not used for anything. New content that is inserted into the tree will always be validated against the stored schema except during initialization.
 	 * @remarks
 	 * TODO: AB#43546: This is not information used to interpret the stored schema: this configuration should be moved elsewhere.
+	 * TODO: Evaluate if this should be removed or renamed.
 	 */
 	readonly validateSchema: boolean;
 

packages/dds/tree/src/feature-libraries/object-forest/objectForest.ts

@@ -217,7 +217,7 @@ export class ObjectForest implements IEditableForest, WithBreakable {
 				if (this.forest.additionalAsserts) {
 					// Schema validation:
 					// When doing "additionalAsserts", validate the content against the schema after every batch of edits.
-					this.forest.checkSchema();
+					// this.forest.checkSchema();
 				}
 			}
 			public destroy(detachedField: FieldKey, count: number): void {

packages/dds/tree/src/index.ts

@@ -117,6 +117,7 @@ export {
 	type InternalTreeNode,
 	type WithType,
 	type NodeChangedData,
+	type SchemaUpgrade,
 	// Types not really intended for public use, but used in links.
 	// Can not be moved to internalTypes since doing so causes app code to throw errors like:
 	// Error: src/simple-tree/objectNode.ts:72:1 - (ae-unresolved-link) The @link reference could not be resolved: The package "@fluidframework/tree" does not have an export "TreeNodeApi"

packages/dds/tree/src/shared-tree/schematizingTreeView.ts

@@ -54,6 +54,7 @@ import {
 	type FieldSchema,
 	toStoredSchema,
 	tryDisposeTreeNode,
+	FieldSchemaAlpha,
 } from "../simple-tree/index.js";
 import {
 	type Breakable,
@@ -184,6 +185,7 @@ export class SchematizingSimpleTreeView<
 					policy: this.schemaPolicy,
 				},
 				this,
+				true,
 			);
 
 			initialize(this.checkout, {
@@ -321,10 +323,14 @@ export class SchematizingSimpleTreeView<
 				this.nodeKeyManager,
 			);
 			assert(!slots.has(SimpleContextSlot), 0xa47 /* extra simple tree context */);
+			assert(
+				this.rootFieldSchema instanceof FieldSchemaAlpha,
+				"all field schema should be FieldSchemaAlpha",
+			);
 			slots.set(
 				SimpleContextSlot,
 				new HydratedContext(
-					normalizeFieldSchema(this.rootFieldSchema).annotatedAllowedTypesNormalized,
+					this.rootFieldSchema.annotatedAllowedTypesNormalized,
 					this.flexTreeContext,
 				),
 			);

packages/dds/tree/src/simple-tree/api/schemaCompatibilityTester.ts

@@ -3,11 +3,10 @@
  * Licensed under the MIT License.
  */
 
-import { assert, unreachableCase } from "@fluidframework/core-utils/internal";
+import { unreachableCase } from "@fluidframework/core-utils/internal";
 
 import type { TreeStoredSchema } from "../../core/index.js";
 import {
-	allowsRepoSuperset,
 	FieldKinds,
 	type FullSchemaPolicy,
 	isNeverTree,
@@ -22,7 +21,6 @@ import {
 	PosetComparisonResult,
 	type FieldDiscrepancy,
 } from "../discrepancies.js";
-import { toStoredSchema } from "../toStoredSchema.js";
 
 /**
  * A collection of View information for schema, including policy.
@@ -142,7 +140,7 @@ export class SchemaCompatibilityTester {
 						// View schema has added a node type that the stored schema doesn't know about.
 						// Note that all cases which trigger this should also trigger an AllowedTypeDiscrepancy (where the type is used).
 						// This means this case should be redundant and could be removed in the future if there is a reason to do so
-						// (like simplifying enablable type support).
+						// (like simplifying staged type support).
 						// See the TODO in getAllowedContentDiscrepancies.
 						canView = false;
 					} else if (discrepancy.view === undefined) {
@@ -181,12 +179,15 @@ export class SchemaCompatibilityTester {
 			}
 		}
 
-		if (canUpgrade) {
-			assert(
-				allowsRepoSuperset(this.policy, stored, toStoredSchema(this.viewSchemaRoot)),
-				"View schema must be a superset of the stored schema to allow upgrade",
-			);
-		}
+		// TODO: It is no longer guaranteed that the result of toStoredSchema (which removes any staged allowed types in its conversion)
+		// is a superset of the stored schema, which may have upgraded staged allowed types from another client.
+		// if (canUpgrade) {
+		// 	// This includes staged allowed types in the conversion before their upgrade to ensure compatibility.
+		// 	assert(
+		// 		allowsRepoSuperset(this.policy, stored, toStoredSchema(this.viewSchemaRoot)),
+		// 		"View schema must be a superset of the stored schema to allow upgrade",
+		// 	);
+		// }
 
 		return {
 			canView,

packages/dds/tree/src/simple-tree/api/schemaFactoryAlpha.ts

@@ -26,15 +26,18 @@ import {
 } from "./schemaFactory.js";
 import type { ImplicitAnnotatedFieldSchema, ImplicitFieldSchema } from "../fieldSchema.js";
 import type { RestrictiveStringRecord } from "../../util/index.js";
-import type {
-	NodeKind,
-	TreeNodeSchema,
-	TreeNodeSchemaBoth,
-	TreeNodeSchemaClass,
-	TreeNodeSchemaNonClass,
-	WithType,
-	ImplicitAllowedTypes,
-	ImplicitAnnotatedAllowedTypes,
+import {
+	type NodeKind,
+	type TreeNodeSchema,
+	type TreeNodeSchemaBoth,
+	type TreeNodeSchemaClass,
+	type TreeNodeSchemaNonClass,
+	type WithType,
+	type ImplicitAllowedTypes,
+	type ImplicitAnnotatedAllowedTypes,
+	type AnnotatedAllowedType,
+	normalizeToAnnotatedAllowedType,
+	createSchemaUpgrade,
 } from "../core/index.js";
 import type {
 	ArrayNodeCustomizableSchemaUnsafe,
@@ -63,6 +66,38 @@ export class SchemaFactoryAlpha<
 		) as ScopedSchemaName<TScope, Name>;
 	}
 
+	/**
+	 * Declares a staged type in a set of {@link AllowedTypes}.
+	 *
+	 * @remarks
+	 *
+	 * Staged allowed types add support for reading a type which can be used for schema evolution to add members to
+	 * an {@link AllowedTypes} while supporting cross version collaboration.
+	 *
+	 * Once enough clients support reading the type, support for writing can be added by removing the use of
+	 * `staged` from the schema definition and upgrading the schema.
+	 *
+	 * A future change will allow writing the type using a runtime schema upgrade so that the type can be upgraded
+	 * using a configuration flag change rather than a code change.
+	 *
+	 * @privateRemarks
+	 * TODO:#44317 staged allowed types rely on schema validation of stored schema to output errors, these errors are not very
+	 * user friendly and should be improved, particularly in the case of staged allowed types
+	 *
+	 */
+	public static staged<const T extends TreeNodeSchema>(
+		t: T | AnnotatedAllowedType<T>,
+	): AnnotatedAllowedType<T> {
+		const annotatedType = normalizeToAnnotatedAllowedType(t);
+		return {
+			type: annotatedType.type,
+			metadata: {
+				...annotatedType.metadata,
+				stagedSchemaUpgrade: createSchemaUpgrade(),
+			},
+		};
+	}
+
 	/**
 	 * Define a {@link TreeNodeSchemaClass} for a {@link TreeObjectNode}.
 	 *

packages/dds/tree/src/simple-tree/core/allowedTypes.ts

@@ -131,7 +131,30 @@ export interface AllowedTypeMetadata {
 	 */
 	readonly custom?: unknown;
 
-	// TODO metadata for enablable types will be added here
+	/**
+	 * If defined, indicates that an allowed type is {@link SchemaFactoryAlpha.staged | staged}.
+	 */
+	readonly stagedSchemaUpgrade?: SchemaUpgrade;
+}
+
+/**
+ * Package internal construction API.
+ */
+export let createSchemaUpgrade: () => SchemaUpgrade;
+
+/**
+ * Unique token used to upgrade schemas and determine if a particular upgrade has been completed.
+ *
+ * TODO:#38722 implement runtime schema upgrades until then, the class purely behaves as a placeholder and we disable no-extraneous-class
+ * @sealed @alpha
+ */
+// eslint-disable-next-line @typescript-eslint/no-extraneous-class
+export class SchemaUpgrade {
+	static {
+		createSchemaUpgrade = () => new SchemaUpgrade();
+	}
+
+	private constructor() {}
 }
 
 /**
@@ -237,29 +260,41 @@ export type UnannotateAllowedType<T extends AnnotatedAllowedType> =
  * @internal
  */
 export function normalizeAllowedTypes(
-	types: ImplicitAllowedTypes,
+	types: ImplicitAnnotatedAllowedTypes,
 ): ReadonlySet<TreeNodeSchema> {
+	// remove annotations before normalizing
+	const unannotated = unannotateImplicitAllowedTypes(types);
 	const normalized = new Set<TreeNodeSchema>();
-	if (isReadonlyArray(types)) {
+	if (isReadonlyArray(unannotated)) {
 		// Types array must not be modified after it is normalized since that would result in the user of the normalized data having wrong (out of date) content.
-		Object.freeze(types);
-		for (const lazyType of types) {
+		Object.freeze(unannotated);
+		for (const lazyType of unannotated) {
 			normalized.add(evaluateLazySchema(lazyType));
 		}
 	} else {
-		normalized.add(evaluateLazySchema(types));
+		normalized.add(evaluateLazySchema(unannotated));
 	}
 	return normalized;
 }
 
 /**
- * Normalizes an allowed type to an {@link AnnotatedAllowedType}, by adding empty annotations if they don't already exist.
+ * Normalizes an allowed type to an {@link AnnotatedAllowedType}, by adding empty annotations if they don't already exist
+ * and eagerly evaluating any lazy schema declarations.
+ *
+ * @remarks
+ * Note: this must only be called after all required schemas have been declared, otherwise evaluation of
+ * recursive schemas may fail.
+ * type is frozen and should not be modified after being passed in.
  */
 export function normalizeToAnnotatedAllowedType<T extends TreeNodeSchema>(
 	type: T | AnnotatedAllowedType<T> | AnnotatedAllowedType<LazyItem<T>>,
-): AnnotatedAllowedType<T> | AnnotatedAllowedType<LazyItem<T>> {
+): AnnotatedAllowedType<T> {
+	Object.freeze(type);
 	return isAnnotatedAllowedType(type)
-		? type
+		? {
+				metadata: type.metadata,
+				type: evaluateLazySchema(type.type),
+			}
 		: {
 				metadata: {},
 				type,