Merge branch 'kats/s3' into beta

harmony7 · harmony7 · commit 2227202e4edf · 2025-11-11T11:35:04.000+09:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,17 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 
-### Fixed
-
-- Use pathToFileURL so that import() works under ESM across platforms [#38](https://github.com/fastly/compute-js-static-publish/issues/38)
-
-## [7.0.2] - 2025-09-16
-
-### Fixed
-
-- Fix script listed in readme instructions
-- Handle symlinks properly in file enumeration [#41](https://github.com/fastly/compute-js-static-publish/issues/41)
-
 ### Breaking
 
 - Rename symbols
@@ -26,12 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- S3-compatible storage
+- S3-compatible storage (BETA)
    - Add support for S3-compatible storage, such as Fastly Object Storage
    - Store items using same keys as KV Store
    - Use S3 object metadata for storing asset metadata
    - Storage factored out to StorageProvider, and S3 is implemented using this architecture
-   - Configure using  
+   - Add AWS-related configurations to fastly.toml in local_server and setup sections
 
 - `static-publish.rc.js`
    - Add `s3` mode configuration
@@ -41,6 +30,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `publish-content.config.js`
    - `kvStoreAssetInclusionTest` renamed to `assetInclusionTest`. Previous name deprecated.
 
+## [7.0.3] - 2025-09-29
+
+### Fixed
+
+- Use pathToFileURL so that import() works under ESM across platforms [#38](https://github.com/fastly/compute-js-static-publish/issues/38)
+
+## [7.0.2] - 2025-09-16
+
+### Fixed
+
+- Fix script listed in readme instructions
+- Handle symlinks properly in file enumeration [#41](https://github.com/fastly/compute-js-static-publish/issues/41)
+
 ## [7.0.1] - 2025-04-24
 
 ### Fixed
diff --git a/README.md b/README.md
@@ -56,7 +56,6 @@ Create a directory for your project, place your static files in `./public`, then
 ```sh
 npx @fastly/compute-js-static-publish@latest \
   --root-dir=./public \
-  --storage-mode=kv-store \
   --kv-store-name=site-content
 ```
 
@@ -236,7 +235,7 @@ const rc = {
 export default rc;
 ```
 
-### Using S3-compatible storage
+### Using S3-compatible storage (BETA)
 
 #### Fields:
 
@@ -525,7 +524,7 @@ If you do need to rebuild and redeploy the Compute app, simply run:
 npm run fastly:deploy
 ```
 
-### Using S3-compatible storage
+### Using S3-compatible storage (BETA)
 
 #### Local development
 
@@ -731,10 +730,9 @@ Run outside an existing Compute app directory:
 # Using KV store storage
 npx @fastly/compute-js-static-publish@latest \
   --root-dir=./public \
-  --storage-mode=kv-store \
   --kv-store-name=<site-content>
 
-# Using S3 storage
+# Using S3 storage (BETA)
 npx @fastly/compute-js-static-publish@latest \
   --root-dir=./public \
   --storage-mode=s3 \
@@ -748,7 +746,7 @@ npx @fastly/compute-js-static-publish@latest \
 ```sh
 npx @fastly/compute-js-static-publish@latest \
   --root-dir=./public \
-  { --storage-mode=kv-store --kv-store-name=<site-content> | \
+  { [--storage-mode=kv-store] --kv-store-name=<site-content> | \
     --storage-mode=s3 --s3-region=<s3 region> --s3-bucket=<bucket name> [--s3-endpoint=<endpoint>] } \
   [--output=./compute-js] \
   [--static-publisher-working-dir=<output>/static-publisher] \
@@ -768,7 +766,7 @@ npx @fastly/compute-js-static-publish@latest \
 #### Options:
 
 **Used to generate the Compute app:**
-- `--storage-mode`: Required. Specifies the storage mode. Must be either `kv-store` or `s3`.
+- `--storage-mode`: Specifies the storage mode. Must be either `kv-store` or `s3` (default: `kv-store`).
 
    If `--storage-mode=kv-store`:
    - `--kv-store-name`: Required. Name of KV Store to use.
diff --git a/README.short.md b/README.short.md
@@ -18,7 +18,7 @@ This CLI tool helps you:
 Create a directory for your project, place your static files in `./public`, then type:
 
 ```sh
-npx @fastly/compute-js-static-publish@latest --root-dir=./public --storage-mode=kv-store --kv-store-name=site-content
+npx @fastly/compute-js-static-publish@latest --root-dir=./public --kv-store-name=site-content
 ```
 
 **New in v8:** S3-compatible storage (such as Fastly Object Storage) is also supported (Beta). To use this mode, type:
diff --git a/src/cli/commands/manage/clean.ts b/src/cli/commands/manage/clean.ts
@@ -39,7 +39,7 @@ KV Store Options:
                                      1. FASTLY_API_TOKEN environment variable
                                      2. Logged-in Fastly CLI profile
 
-S3 Storage Options:
+S3 Storage Options (BETA):
   --aws-access-key-id=<key>        AWS Access Key ID and Secret Access Key used to
   --aws-secret-access-key=<key>    interface with S3.
                                    If not set, the tool will check:
diff --git a/src/cli/commands/manage/collections/delete.ts b/src/cli/commands/manage/collections/delete.ts
@@ -37,7 +37,7 @@ KV Store Options:
                                      1. FASTLY_API_TOKEN environment variable
                                      2. Logged-in Fastly CLI profile
 
-S3 Storage Options:
+S3 Storage Options (BETA):
   --aws-access-key-id=<key>        AWS Access Key ID and Secret Access Key used to
   --aws-secret-access-key=<key>    interface with S3.
                                    If not set, the tool will check:
diff --git a/src/cli/commands/manage/collections/index.ts b/src/cli/commands/manage/collections/index.ts
@@ -35,7 +35,7 @@ KV Store Options:
                                      1. FASTLY_API_TOKEN environment variable
                                      2. Logged-in Fastly CLI profile
 
-S3 Storage Options:
+S3 Storage Options (BETA):
   --aws-access-key-id=<key>        AWS Access Key ID and Secret Access Key used to
   --aws-secret-access-key=<key>    interface with S3.
                                    If not set, the tool will check:
diff --git a/src/cli/commands/manage/collections/list.ts b/src/cli/commands/manage/collections/list.ts
@@ -32,7 +32,7 @@ KV Store Options:
                                      1. FASTLY_API_TOKEN environment variable
                                      2. Logged-in Fastly CLI profile
 
-S3 Storage Options:
+S3 Storage Options (BETA):
   --aws-access-key-id=<key>        AWS Access Key ID and Secret Access Key used to
   --aws-secret-access-key=<key>    interface with S3.
                                    If not set, the tool will check:
diff --git a/src/cli/commands/manage/collections/promote.ts b/src/cli/commands/manage/collections/promote.ts
@@ -54,7 +54,7 @@ KV Store Options:
                                      1. FASTLY_API_TOKEN environment variable
                                      2. Logged-in Fastly CLI profile
 
-S3 Storage Options:
+S3 Storage Options (BETA):
   --aws-access-key-id=<key>        AWS Access Key ID and Secret Access Key used to
   --aws-secret-access-key=<key>    interface with S3.
                                    If not set, the tool will check:
diff --git a/src/cli/commands/manage/collections/update-expiration.ts b/src/cli/commands/manage/collections/update-expiration.ts
@@ -50,7 +50,7 @@ KV Store Options:
                                      1. FASTLY_API_TOKEN environment variable
                                      2. Logged-in Fastly CLI profile
 
-S3 Storage Options:
+S3 Storage Options (BETA):
   --aws-access-key-id=<key>        AWS Access Key ID and Secret Access Key used to
   --aws-secret-access-key=<key>    interface with S3.
                                    If not set, the tool will check:
diff --git a/src/cli/commands/manage/index.ts b/src/cli/commands/manage/index.ts
@@ -37,7 +37,7 @@ KV Store Options:
                                      1. FASTLY_API_TOKEN environment variable
                                      2. Logged-in Fastly CLI profile
 
-S3 Storage Options:
+S3 Storage Options (BETA):
   --aws-access-key-id=<key>        AWS Access Key ID and Secret Access Key used to
   --aws-secret-access-key=<key>    interface with S3.
                                    If not set, the tool will check:
diff --git a/src/cli/commands/manage/publish-content.ts b/src/cli/commands/manage/publish-content.ts
@@ -22,6 +22,7 @@ import { calculateFileSizeAndHash, enumerateFiles, rootRelative } from '../../ut
 import { ensureVariantFileExists, type Variants } from '../../util/variants.js';
 import {
   loadStorageProviderFromStaticPublishRc,
+  StorageProvider,
   StorageProviderBatch,
   type StorageProviderBatchEntry,
 } from '../../storage/storage-provider.js';
@@ -56,6 +57,11 @@ Optional:
   --root-dir=<dir>                 Directory to publish from. Overrides the config file setting.
                                    Default: rootDir from publish-content.config.js
 
+  --fastly-api-token=<token>       Fastly API token for KV Store or cache access.
+                                   If not set, the tool will check:
+                                     1. FASTLY_API_TOKEN environment variable
+                                     2. Logged-in Fastly CLI profile
+
   --overwrite-existing             Always overwrite existing entries in storage, even if unchanged.
 
 Expiration:
@@ -75,14 +81,9 @@ KV Store Options:
                                    local files that will be used to simulate the KV Store
                                    with the local development environment.
 
-  --fastly-api-token=<token>       Fastly API token for KV Store access.
-                                   If not set, the tool will check:
-                                     1. FASTLY_API_TOKEN environment variable
-                                     2. Logged-in Fastly CLI profile
-
   --kv-overwrite                   Alias for --overwrite-existing.
 
-S3 Storage Options:
+S3 Storage Options (BETA):
   --aws-access-key-id=<key>        AWS Access Key ID and Secret Access Key used to
   --aws-secret-access-key=<key>    interface with S3.
                                    If not set, the tool will check:
@@ -229,7 +230,7 @@ export async function action(actionArgs: string[]) {
   console.log(`  | Static publisher working directory: ${staticPublisherWorkingDir}`);
 
   // Storage Provider
-  let storageProvider: any;
+  let storageProvider: StorageProvider;
   try {
     storageProvider = await loadStorageProviderFromStaticPublishRc(staticPublisherRc, {
       computeAppDir,
@@ -594,6 +595,8 @@ export async function action(actionArgs: string[]) {
 
   console.log(`✅  Settings have been saved.`);
 
+  await storageProvider.purgeSurrogateKey(`${publishId}-${collectionName}`);
+
   console.log(`🎉 Completed.`);
 
 }
diff --git a/src/cli/commands/scaffold/index.ts b/src/cli/commands/scaffold/index.ts
@@ -32,8 +32,9 @@ Description:
   management mode.
 
 Options:
-  --storage-mode <mode>                 (required) Storage mode for content storage. 
-                                        Can be "kv-store" or "s3".
+  --storage-mode <mode>                 Storage mode for content storage. 
+                                        Can be "kv-store" or "s3" (BETA).
+                                        (default: kv-store)
   If --storage-mode=kv-store, then:
     --kv-store-name <name>              (required) Name of the KV Store.
 
@@ -45,8 +46,8 @@ Options:
   --root-dir <path>                     (required) Path to static content (e.g., ./public)
   -o, --output <dir>                    Output directory for Compute app (default: ./compute-js)
   --static-publisher-working-dir <dir>  Working directory for build artifacts (default: <output>/static-publisher)
-  --publish-id <id>                     Advanced. Prefix for KV keys (default: "default")
                                         (default: ./compute-js/static-publisher)
+  --publish-id <id>                     Advanced. Prefix for KV keys (default: "default")
 
 Compute Service Metadata:
   --name <name>                         App name (for fastly.toml)
@@ -407,8 +408,8 @@ export async function action(actionArgs: string[]) {
   const optionDefinitions: OptionDefinition[] = [
     { name: 'verbose', type: Boolean },
 
-    // Required. Storage mode for content storage. Can be "kv-store" or "s3".
-    { name: 'storage-mode', type: String, },
+    // Storage mode for content storage. Can be "kv-store" or "s3".
+    { name: 'storage-mode', type: String, defaultValue: 'kv-store', },
 
     // If storage-mode=kv-store, then:
 
@@ -651,7 +652,7 @@ export async function action(actionArgs: string[]) {
   const fastlyServiceId = options.serviceId;
   const storageMode = options.storageMode;
   if (!(storageMode === 'kv-store' || storageMode === 's3')) {
-    console.error(`❌ required parameter --storage-mode must be set to 'kv-store' or 's3'.`);
+    console.error(`❌ parameter --storage-mode must be set to 'kv-store' or 's3'.`);
     process.exitCode = 1;
     return;
   }
@@ -725,7 +726,7 @@ export async function action(actionArgs: string[]) {
     console.log('Storage Mode                   : Fastly KV Store');
     console.log('KV Store Name                  :', kvStoreName);
   } else if (storageMode === 's3') {
-    console.log('Storage Name                   : S3-compatible storage');
+    console.log('Storage Name                   : S3-compatible storage (BETA)');
     console.log('S3 Region                      :', s3Region);
     console.log('S3 Bucket                      :', s3Bucket);
     console.log('S3 Endpoint                    :', String(s3EndpointUrl));
@@ -808,17 +809,24 @@ export async function action(actionArgs: string[]) {
   if (storageMode === 'kv-store') {
     const localServerKvStorePath = dotRelative(computeJsDir, path.resolve(staticPublisherWorkingDir, 'kvstore.json'));
     fastlyTomlLocalServer = /* language=text */ `\
+[local_server]
+
 [local_server.kv_stores]
 ${kvStoreName} = { file = "${localServerKvStorePath}", format = "json" }
 `;
     fastlyTomlSetup = /* language=text */ `\
+[setup]
+
+[setup.kv_stores]
 [setup.kv_stores.${kvStoreName}]
 `;
 
     // kvstore.json
     resourceFiles[localServerKvStorePath] = '{}';
   } else if (storageMode === 's3') {
     fastlyTomlLocalServer = /* language=text */ `\
+[local_server]
+
 [local_server.secret_stores]
 [[local_server.secret_stores.AWS_CREDENTIALS]]
 key = "AWS_ACCESS_KEY_ID"
@@ -833,10 +841,21 @@ url = "${String(s3EndpointUrl)}"
 override_host = "${s3EndpointUrl!.hostname}"
 `;
     fastlyTomlSetup = /* language=text */ `\
+[setup]
+
+[setup.backends]
 [setup.backends.aws]
 address = "${s3EndpointUrl!.hostname}"
 description = "S3 API endpoint"
 port = 443
+[setup.secret_stores]
+[setup.secret_stores.AWS_CREDENTIALS]
+description = "Credentials for S3 storage"
+[setup.secret_stores.AWS_CREDENTIALS.items]
+[setup.secret_stores.AWS_CREDENTIALS.items.AWS_ACCESS_KEY_ID]
+description = "AWS Access Key ID"
+[setup.secret_stores.AWS_CREDENTIALS.items.AWS_SECRET_ACCESS_KEY]
+description = "AWS Secret Access Key"
 `;
   }
 
diff --git a/src/cli/storage/kv-store-local-provider.ts b/src/cli/storage/kv-store-local-provider.ts
@@ -186,4 +186,7 @@ export class KvStoreLocalProvider implements StorageProvider {
     // to save time by checking for an existing item. This is not applicable for local.
     return null;
   }
+
+  async purgeSurrogateKey(_surrogateKey: string): Promise<void> {
+  }
 }
diff --git a/src/cli/storage/kv-store-provider.ts b/src/cli/storage/kv-store-provider.ts
@@ -267,6 +267,9 @@ export class KvStoreProvider implements StorageProvider {
     return kvStoreItemMetadata;
 
   }
+
+  async purgeSurrogateKey(_surrogateKey: string): Promise<void> {
+  }
 }
 
 export function kvStoreEntryToStorageEntry(
diff --git a/src/cli/storage/s3-storage-provider.ts b/src/cli/storage/s3-storage-provider.ts
diff --git a/src/cli/storage/storage-provider.ts b/src/cli/storage/storage-provider.ts
diff --git a/src/cli/util/purge.ts b/src/cli/util/purge.ts
diff --git a/src/models/config/static-publish-rc.ts b/src/models/config/static-publish-rc.ts
diff --git a/src/server/publisher-server/index.ts b/src/server/publisher-server/index.ts
diff --git a/src/server/storage/s3-storage-provider.ts b/src/server/storage/s3-storage-provider.ts
diff --git a/src/server/storage/storage-provider.ts b/src/server/storage/storage-provider.ts
