Skip to content

Commit ab5334a

Browse files
authored
Merge pull request #130 from platforma-open/feat/project-template-migration
Migrate to structure v2 and declare a block kind
2 parents f104f77 + e85cc03 commit ab5334a

24 files changed

Lines changed: 1188 additions & 658 deletions

.changeset/soft-hoops-search.md

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
'@platforma-open/milaboratories.samples-and-data.parse-h5ad': patch
3+
'@platforma-open/milaboratories.samples-and-data.parse-seurat': patch
4+
'@platforma-open/milaboratories.samples-and-data.model': minor
5+
'@platforma-open/milaboratories.samples-and-data': minor
6+
---
7+
8+
Migrate to structure v2 and declare a block kind
9+
10+
The SDK upgrade moves the block onto the current canonical layout (block-tools
11+
2.14.0, model/ui-vue 1.82.x, workflow-tengo 6.8.2, tengo-builder 4.0.23) and
12+
switches the software packages from `pl-pkg` to `block-tools software build`.
13+
14+
The block now declares a kind, whose init-params contract is the study setup a
15+
project template supplies — the metadata columns the study collects, the name of
16+
the sample column, and how its datasets are configured — plus whichever of its
17+
data can be resolved from where the block lands.
18+
19+
A dataset travels whole when every one of its files is a storage reference
20+
(`index://` names a storage and a path); a dataset holding any local upload is
21+
reduced to its configuration with the file slots unset, because an upload handle
22+
carries a machine-local path signed with the installation's own secret. The
23+
decision is per dataset rather than per file so that a grouped dataset can never
24+
arrive with fewer sample groups than file groups, which is the invariant `args`
25+
enforces. Samples, their labels and their metadata values follow the datasets
26+
that reference them, so a sample never arrives without the files that created
27+
it.
28+
29+
The dataset and metadata types move into the kind package and are re-exported
30+
from the model, so every existing import keeps resolving. The model also exports
31+
`templateParamsFor`, the projection itself.

.github/workflows/build.yaml

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ on:
1111
workflow_dispatch: {}
1212
jobs:
1313
init:
14-
runs-on: ubuntu-latest
14+
runs-on: hz-ubuntu-dind
1515
steps:
1616
- uses: milaboratory/github-ci/actions/context/init@v4
1717
with:
@@ -24,8 +24,10 @@ jobs:
2424
with:
2525
app-name: 'Block: Samples And Data'
2626
app-name-slug: 'block-samples-and-data'
27-
node-version: '20.x'
28-
build-script-name: 'build'
27+
node-version: '22.x'
28+
gha-runner-label: hz-ubuntu-dind
29+
build-script-name: 'build:dev-local'
30+
build-before-publish-script-name: 'build:release'
2931
pnpm-recursive-build: false
3032

3133
test: true
@@ -37,6 +39,11 @@ jobs:
3739
package-path: 'block'
3840
create-tag: 'true'
3941

42+
# Require the published `block` package to be bumped by a changeset on
43+
# PRs (empty changeset or the `skip-changelog` label waives it). Needs
44+
# the input to exist on the pinned `@v4` reusable workflow.
45+
require-package-path-bump: true
46+
4047
npmrc-config: |
4148
{
4249
"registries": {
@@ -56,6 +63,15 @@ jobs:
5663
5764
"AWS_CI_IAM_MONOREPO_SIMPLE_ROLE": ${{ toJSON(secrets.AWS_CI_IAM_MONOREPO_SIMPLE_ROLE) }},
5865
"AWS_CI_TURBOREPO_S3_BUCKET": ${{ toJSON(secrets.AWS_CI_TURBOREPO_S3_BUCKET) }},
66+
67+
"HZ_CI_TURBO_S3_BUCKET": ${{ toJSON(vars.HZ_CI_TURBO_S3_BUCKET) }},
68+
"HZ_CI_TURBO_S3_ENDPOINT": ${{ toJSON(vars.HZ_CI_TURBO_S3_ENDPOINT) }},
69+
"HZ_CI_TURBO_S3_REGION": ${{ toJSON(vars.HZ_CI_TURBO_S3_REGION) }},
70+
"HZ_CI_TURBO_S3_ACCESS_KEY": ${{ toJSON(secrets.HZ_CI_TURBO_S3_ACCESS_KEY) }},
71+
"HZ_CI_TURBO_S3_SECRET_KEY": ${{ toJSON(secrets.HZ_CI_TURBO_S3_SECRET_KEY) }},
72+
"HZ_CI_CACHE_S3_ACCESS_KEY": ${{ toJSON(secrets.HZ_CI_CACHE_S3_ACCESS_KEY) }},
73+
"HZ_CI_CACHE_S3_SECRET_KEY": ${{ toJSON(secrets.HZ_CI_CACHE_S3_SECRET_KEY) }},
74+
5975
"PL_REGISTRY_PLATFORMA_OPEN_UPLOAD_URL": ${{ toJSON(secrets.PL_REGISTRY_PLOPEN_UPLOAD_URL) }},
6076
"QUAY_USERNAME": ${{ toJSON(secrets.QUAY_USERNAME) }},
6177
"QUAY_ROBOT_TOKEN": ${{ toJSON(secrets.QUAY_ROBOT_TOKEN) }} }

.github/workflows/mark-stable.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
uses: milaboratory/github-ci/.github/workflows/block-mark-stable.yaml@v4
1616
with:
1717
app-name: 'Block: Samples And Data - Mark Stable'
18-
node-version: '20.x'
18+
node-version: '22.x'
1919
npmrc-config: |
2020
{
2121
"registries": {

.structure

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"version":1}
1+
{"version":2}

block/package.json

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,24 @@
1111
"types": "./dist/index.d.ts",
1212
"exports": {
1313
".": {
14-
"sources": "./src/index.ts",
1514
"types": "./dist/index.d.ts",
15+
"sources": "./src/index.ts",
1616
"default": "./dist/index.js"
1717
}
1818
},
1919
"scripts": {
2020
"pretty": "prettier --write \"./**/*.{js,jsx,mjs,cjs,ts,tsx,json,vue}\"",
2121
"build": "ts-builder build --target block-facade && block-tools pack",
2222
"prepublishOnly": "block-tools publish -r s3://milab-euce1-prod-pkgs-s3-block-registry/pub/releases/?region=eu-central-1 --registry-serve-url https://blocks.pl-open.science",
23-
"do-pack": "pnpm pack && shx mv *.tgz package.tgz",
23+
"mark-stable": "block-tools mark-stable -r 's3://milab-euce1-prod-pkgs-s3-block-registry/pub/releases/?region=eu-central-1'",
24+
"do-pack": "shx rm -f package.tgz && pnpm pack && shx mv *.tgz package.tgz",
2425
"check": "ts-builder type-check --target block-facade"
2526
},
2627
"dependencies": {},
2728
"devDependencies": {
2829
"@milaboratories/ts-builder": "catalog:",
2930
"@milaboratories/ts-configs": "catalog:",
31+
"@platforma-open/milaboratories.samples-and-data.kind": "workspace:*",
3032
"@platforma-open/milaboratories.samples-and-data.model": "workspace:*",
3133
"@platforma-open/milaboratories.samples-and-data.ui": "workspace:*",
3234
"@platforma-open/milaboratories.samples-and-data.workflow": "workspace:*",

kind/.oxfmtrc.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"extends": ["node_modules/@milaboratories/ts-builder/configs/oxfmt.json"],
3+
"ignorePatterns": ["dist", "coverage", "CHANGELOG.md"]
4+
}

kind/.oxlintrc.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"extends": ["node_modules/@milaboratories/ts-builder/dist/configs/oxlint-node.json"]
3+
}

kind/package.json

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"name": "@platforma-open/milaboratories.samples-and-data.kind",
3+
"version": "1.0.0",
4+
"private": true,
5+
"type": "module",
6+
"main": "./dist/index.cjs",
7+
"module": "./dist/index.js",
8+
"types": "./dist/index.d.ts",
9+
"exports": {
10+
".": {
11+
"types": "./dist/index.d.ts",
12+
"sources": "./src/index.ts",
13+
"import": "./dist/index.js",
14+
"require": "./dist/index.cjs",
15+
"default": "./dist/index.js"
16+
}
17+
},
18+
"scripts": {
19+
"fmt": "ts-builder format",
20+
"watch": "ts-builder build --target block-kind --watch",
21+
"build": "ts-builder build --target block-kind && block-tools build-kind-manifest",
22+
"check": "ts-builder check --target block-kind"
23+
},
24+
"dependencies": {
25+
"@milaboratories/pl-model-common": "catalog:",
26+
"@platforma-sdk/block-kind": "catalog:",
27+
"es-toolkit": "catalog:"
28+
},
29+
"devDependencies": {
30+
"@milaboratories/ts-builder": "catalog:",
31+
"@milaboratories/ts-configs": "catalog:",
32+
"@platforma-sdk/block-tools": "catalog:"
33+
},
34+
"peerDependencies": {
35+
"@types/node": "*",
36+
"typescript": "*"
37+
}
38+
}

kind/src/index.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { defineBlockKind } from "@platforma-sdk/block-kind";
2+
import { name, version } from "../package.json" with { type: "json" };
3+
import { parseInitializationParams } from "./params";
4+
import type { BlockParams } from "./types";
5+
6+
export * from "./types";
7+
8+
// Identity (`name`/`version`) comes from this package's own `package.json`, so
9+
// the on-wire `{name}@{version}` reference can never drift from what npm
10+
// publishes; the bundler inlines the JSON import.
11+
export const kind = defineBlockKind<BlockParams>({
12+
name,
13+
version,
14+
parseInitializationParams,
15+
});

kind/src/params.ts

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
import { assertParamsObject } from "@platforma-sdk/block-kind";
2+
import { isImportFileHandleIndex, isImportFileHandleUpload } from "@milaboratories/pl-model-common";
3+
import { isBoolean, isPlainObject, isString } from "es-toolkit";
4+
import { isArray } from "es-toolkit/compat";
5+
import type {
6+
BlockParams,
7+
DSAny,
8+
DSType,
9+
ImportFileHandle,
10+
MTColumn,
11+
MTValueType,
12+
PlId,
13+
} from "./types";
14+
15+
/**
16+
* The contract at runtime, for params that arrive from a template file rather
17+
* than from typed code.
18+
*
19+
* Each field the contract names is read and checked; a key it does not name is
20+
* dropped by never being read, so it needs no rejection here. Params written
21+
* against a different version of the contract are caught by the version in the
22+
* template entry's `{name}@{selector}` reference, not by a key-set check.
23+
*/
24+
export function parseInitializationParams(value: unknown): BlockParams {
25+
assertParamsObject(value);
26+
27+
const params: Record<string, unknown> = {};
28+
for (const [field, { is, must }] of Object.entries(CONTRACT)) {
29+
const raw = value[field];
30+
if (raw === undefined) continue;
31+
if (!is(raw)) throw new Error(`'${field}' must be ${must}.`);
32+
params[field] = raw;
33+
}
34+
// Every value placed here passed its own field's guard, and `CONTRACT` is
35+
// proven exhaustive over `BlockParams` by the `satisfies` below.
36+
return params as BlockParams;
37+
}
38+
39+
// ---------------------------------------------------------------------------
40+
// Internals
41+
// ---------------------------------------------------------------------------
42+
43+
type Guard<T> = (value: unknown) => value is T;
44+
45+
/** A guard plus how to finish the sentence "'field' must be …". */
46+
type Check<T> = { readonly is: Guard<T>; readonly must: string };
47+
48+
function check<T>(is: Guard<T>, must: string): Check<T> {
49+
return { is, must };
50+
}
51+
52+
function arrayOf<T>(item: Guard<T>): Guard<T[]> {
53+
return (v): v is T[] => isArray(v) && v.every((e) => item(e));
54+
}
55+
56+
function recordOf<T>(item: Guard<T>): Guard<Record<string, T>> {
57+
return (v): v is Record<string, T> => isPlainObject(v) && Object.values(v).every((e) => item(e));
58+
}
59+
60+
/**
61+
* A guard over the keys of one of the tag sets below, so the runtime check and
62+
* the type it guards are both read off the same object.
63+
*/
64+
function keyOf<T extends string>(set: Record<T, true>): Guard<T> {
65+
return (v): v is T => isString(v) && v in set;
66+
}
67+
68+
function keyList(set: Record<string, true>): string {
69+
return Object.keys(set).join(", ");
70+
}
71+
72+
/**
73+
* The value types a metadata column may declare, as a runtime set. `satisfies
74+
* Record<MTValueType, true>` is what keeps it in step: a value type added to the
75+
* union and forgotten here fails to compile, rather than turning into a kind
76+
* that refuses a correct file.
77+
*/
78+
const MT_VALUE_TYPES = {
79+
Long: true,
80+
Double: true,
81+
String: true,
82+
} as const satisfies Record<MTValueType, true>;
83+
84+
/** The dataset kinds this block knows, kept in step with the union the same way. */
85+
const DATASET_TYPES = {
86+
Fastq: true,
87+
MultilaneFastq: true,
88+
TaggedFastq: true,
89+
Fasta: true,
90+
Xsv: true,
91+
TaggedXsv: true,
92+
BulkCountMatrix: true,
93+
CellRangerMTX: true,
94+
MultiplexedFastq: true,
95+
H5AD: true,
96+
H5: true,
97+
Seurat: true,
98+
MultiSampleH5AD: true,
99+
MultiSampleSeurat: true,
100+
} as const satisfies Record<DSType, true>;
101+
102+
/**
103+
* A `PlId` is a branded 24-char base32 string, but the brand is erased at
104+
* runtime and the block treats these ids as opaque keys, so the envelope check
105+
* is the string test.
106+
*/
107+
const isPlId = isString as Guard<PlId>;
108+
109+
/**
110+
* Both handle forms are accepted. The projection only sends `index://` handles,
111+
* but `upload://` is a legitimate value of the type, and the two SDK guards are
112+
* prefix tests — the cast only gets a checked string past a signature that
113+
* expects the union.
114+
*/
115+
const isFileHandle: Guard<ImportFileHandle> = (v): v is ImportFileHandle =>
116+
isString(v) &&
117+
(isImportFileHandleIndex(v as ImportFileHandle) ||
118+
isImportFileHandleUpload(v as ImportFileHandle));
119+
120+
/**
121+
* A metadata column at the envelope: its identity, its declared value type, and
122+
* nothing about the values. What the values mean is settled where they are used.
123+
*/
124+
const isMetadataColumn: Guard<MTColumn> = (v): v is MTColumn =>
125+
isPlainObject(v) &&
126+
isString(v.id) &&
127+
isString(v.label) &&
128+
isBoolean(v.global) &&
129+
keyOf(MT_VALUE_TYPES)(v.valueType);
130+
131+
/**
132+
* A dataset at the envelope: its identity and the discriminator of its content.
133+
*
134+
* The file handles inside are deliberately not walked. Their shape differs per
135+
* dataset kind, and whether a handle still resolves is knowable only where the
136+
* block runs, not here — the same limit every kind has on a handle. Likewise
137+
* the per-sample and per-group maps: the block's own `args` is what holds
138+
* datasets, groups and rules to each other, and rejecting a half-configured
139+
* study here would refuse states the editor can reach.
140+
*/
141+
const isDataset: Guard<DSAny> = (v): v is DSAny =>
142+
isPlainObject(v) &&
143+
isString(v.id) &&
144+
isString(v.label) &&
145+
isPlainObject(v.content) &&
146+
keyOf(DATASET_TYPES)(v.content.type) &&
147+
isBoolean(v.content.gzipped);
148+
149+
/**
150+
* The contract, field by field, at runtime.
151+
*
152+
* The `satisfies` clause is the drift guard: it demands an entry for every key
153+
* `BlockParams` declares, and types each guard against that key's own type. Add
154+
* a field to the contract and this stops compiling until the check exists —
155+
* which matters here because every field is optional, so a parser that simply
156+
* forgot one would otherwise return a valid `BlockParams` and say nothing.
157+
*/
158+
const CONTRACT = {
159+
datasets: check(arrayOf(isDataset), `an array of datasets of: ${keyList(DATASET_TYPES)}`),
160+
metadata: check(
161+
arrayOf(isMetadataColumn),
162+
`an array of metadata columns valued: ${keyList(MT_VALUE_TYPES)}`,
163+
),
164+
sampleIds: check(arrayOf(isPlId), "an array of sample ids"),
165+
sampleLabelColumnLabel: check(isString, "a string"),
166+
sampleLabels: check(recordOf(isString), "an object of sample id to label"),
167+
h5adFilesToPreprocess: check(arrayOf(isFileHandle), "an array of file handles"),
168+
seuratFilesToPreprocess: check(arrayOf(isFileHandle), "an array of file handles"),
169+
} satisfies { [K in keyof BlockParams]-?: Check<NonNullable<BlockParams[K]>> };

0 commit comments

Comments
 (0)