Expires cached files after configured seconds

docs/config.md

@@ -27,6 +27,16 @@ The path to the source root; defaults to `src`. (Prior to <a href="https://githu
 
 The path to the output root; defaults to `dist`.
 
+## cacheExpiration
+
+The duration in seconds after which the cache is considered expired causing the data loader to reload the data.
+
+To set a cache expiration of 3600 seconds (one hour):
+
+```js run=false
+cacheExpiration: 3600
+```
+
 ## theme
 
 The theme name or names, if any; defaults to `default`. [Themes](./themes) affect visual appearance by specifying colors and fonts, or by augmenting default styles. The theme option is a shorthand alternative to specifying a [custom stylesheet](#style).

docs/data-loaders.md

@@ -297,6 +297,8 @@ During preview, Framework considers the cache “fresh” if the modification ti
 
 During build, Framework ignores modification times and only runs a data loader if its output is not cached. Continuous integration caches typically don’t preserve modification times, so this design makes it easier to control which data loaders to run by selectively populating the cache.
 
+A custom [cache expiration](./config#cache-expiration) can be set, causing the cache to expire if the cached output's modification time exceeds the specified value. The data loader will then reload the data.
+
 To purge the data loader cache and force all data loaders to run on the next build, delete the entire cache. For example:
 
 ```sh

src/config.ts

@@ -97,6 +97,7 @@ interface DuckDBExtensionConfigSpec {
 export interface Config {
   root: string; // defaults to src
   output: string; // defaults to dist
+  cacheExpiration?: number; // cache expiration in seconds
   base: string; // defaults to "/"
   home: string; // defaults to the (escaped) title, or "Home"
   title?: string;
@@ -122,6 +123,7 @@ export interface Config {
 export interface ConfigSpec {
   root?: unknown;
   output?: unknown;
+  cacheExpiration?: unknown;
   base?: unknown;
   sidebar?: unknown;
   style?: unknown;
@@ -251,6 +253,7 @@ export function normalizeConfig(spec: ConfigSpec = {}, defaultRoot?: string, wat
   if (cachedConfig) return cachedConfig;
   const root = spec.root === undefined ? findDefaultRoot(defaultRoot) : String(spec.root);
   const output = spec.output === undefined ? "dist" : String(spec.output);
+  const cacheExpiration = spec.cacheExpiration === undefined ? undefined : Number(spec.cacheExpiration);
   const base = spec.base === undefined ? "/" : normalizeBase(spec.base);
   const style =
     spec.style === null
@@ -298,6 +301,7 @@ export function normalizeConfig(spec: ConfigSpec = {}, defaultRoot?: string, wat
   const config: Config = {
     root,
     output,
+    cacheExpiration,
     base,
     home,
     title,

src/loader.ts

@@ -51,6 +51,9 @@ const defaultEffects: LoadEffects = {
 export interface LoadOptions {
   /** Whether to use a stale cache; true when building. */
   useStale?: boolean;
+
+  /** Cache expiration time in seconds */
+  cacheExpiration?: number;
 }
 
 export interface LoaderOptions {
@@ -398,7 +401,7 @@ abstract class AbstractLoader implements Loader {
     this.targetPath = targetPath;
   }
 
-  async load({useStale = false}: LoadOptions = {}, effects = defaultEffects): Promise<string> {
+  async load({useStale = false, cacheExpiration}: LoadOptions = {}, effects = defaultEffects): Promise<string> {
     const loaderPath = join(this.root, this.path);
     const key = join(this.root, this.targetPath);
     let command = runningCommands.get(key);
@@ -409,7 +412,9 @@ abstract class AbstractLoader implements Loader {
         const loaderStat = await maybeStat(loaderPath);
         const cacheStat = await maybeStat(cachePath);
         if (!cacheStat) effects.output.write(faint("[missing] "));
-        else if (cacheStat.mtimeMs < loaderStat!.mtimeMs) {
+        else if (cacheExpiration !== undefined && (Date.now() - cacheStat.mtimeMs > cacheExpiration * 1000)) {
+          effects.output.write(faint("[expired] "));
+        } else if (cacheStat.mtimeMs < loaderStat!.mtimeMs) {
           if (useStale) return effects.output.write(faint("[using stale] ")), outputPath;
           else effects.output.write(faint("[stale] "));
         } else return effects.output.write(faint("[fresh] ")), outputPath;

src/preview.ts

@@ -119,7 +119,7 @@ export class PreviewServer {
 
   _handleRequest: RequestListener = async (req, res) => {
     const config = await this._readConfig();
-    const {root, loaders, duckdb} = config;
+    const {root, loaders, duckdb, cacheExpiration} = config;
     if (this._verbose) console.log(faint(req.method!), req.url);
     const url = new URL(req.url!, "http://localhost");
     const {origin} = req.headers;
@@ -180,7 +180,7 @@ export class PreviewServer {
         const path = pathname.slice("/_file".length);
         const loader = loaders.find(path);
         if (!loader) throw enoent(path);
-        send(req, await loader.load(), {root}).pipe(res);
+        send(req, await loader.load({cacheExpiration}), {root}).pipe(res);
       } else {
         if ((pathname = normalize(pathname)).startsWith("..")) throw new Error("Invalid path: " + pathname);
 

test/loader-test.ts

@@ -114,6 +114,43 @@ describe("LoaderResolver.find(path)", () => {
       ]
     );
   });
+  it("data loaders reload when cacheExpiration is set and expired", async () => {
+    const out = [] as string[];
+    const outputEffects: LoadEffects = {
+      logger: {log() {}, warn() {}, error() {}},
+      output: {
+        write(a) {
+          out.push(a);
+        }
+      }
+    };
+    const loader = loaders.find("/dataloaders/data1.txt")!;
+    // remove the cache set by another test (unless we it.only this test).
+    try {
+      await unlink("test/.observablehq/cache/dataloaders/data1.txt");
+    } catch {
+      // ignore;
+    }
+    // populate the cache (missing)
+    await loader.load(undefined, outputEffects);
+    // run again (fresh)
+    await loader.load(undefined, outputEffects);
+    // run again (stale due to cacheExpiration)
+    await loader.load({cacheExpiration: 0}, outputEffects);
+
+    assert.deepStrictEqual(
+      // eslint-disable-next-line no-control-regex
+      out.map((l) => l.replaceAll(/\x1b\[[0-9]+m/g, "")),
+      [
+        "load /dataloaders/data1.txt → ",
+        "[missing] ",
+        "load /dataloaders/data1.txt → ",
+        "[fresh] ",
+        "load /dataloaders/data1.txt → ",
+        "[expired] "
+      ]
+    );
+  });
 });
 
 describe("LoaderResolver.getSourceFileHash(path)", () => {