cache blogData to make site deployable to Cloudflare

Author: dario-piotrowicz

apps/site/.cloudflare/node/fs.mjs

@@ -1,5 +1,6 @@
 import fsPromises from 'fs/promises';
 
+import nodejsorgCacheManifest from '../.asset-manifests/.nodejsorg-cache.mjs';
 import pagesManifest from '../.asset-manifests/pages.mjs';
 import snippetsManifest from '../.asset-manifests/snippets.mjs';
 
@@ -28,18 +29,33 @@ export function readdir(path, options, cb) {
 }
 
 function findInDirentLikes(path) {
-  if (!path.startsWith('/pages') && !path.startsWith('/snippets')) {
+  if (
+    allowedPaths.every(
+      allowedPath =>
+        !path.startsWith(allowedPath) && !path.startsWith(`.${allowedPath}`)
+    )
+  ) {
     return null;
   }
 
+  if (path.startsWith('./.next')) {
+    // remove the leading `.`
+    path = path.slice('./.next'.length);
+  }
+
   // remove the leading `/`
   path = path.slice(1);
 
   const paths = path.split('/');
 
   const manifestType = paths.shift();
 
-  const manifest = manifestType === 'pages' ? pagesManifest : snippetsManifest;
+  const manifest =
+    manifestType === '.nodejsorg-cache'
+      ? nodejsorgCacheManifest
+      : manifestType === 'pages'
+        ? pagesManifest
+        : snippetsManifest;
 
   return recursivelyFindInDirentLikes(paths, manifest);
   function recursivelyFindInDirentLikes(paths, direntLikes) {
@@ -63,7 +79,12 @@ export function existsSync(path) {
 }
 
 function existsImpl(path) {
-  if (!path.startsWith('/pages') && !path.startsWith('/snippets')) {
+  if (
+    allowedPaths.every(
+      allowedPath =>
+        !path.startsWith(allowedPath) && !path.startsWith(`.${allowedPath}`)
+    )
+  ) {
     return false;
   }
   return !!findInDirentLikes(path);
@@ -82,6 +103,8 @@ export function createReadStream(path) {
   return env.ASSETS.fetch(url);
 }
 
+const allowedPaths = ['/pages', '/snippets', '/.next/.nodejsorg-cache'];
+
 export default {
   readdir,
   exists,

apps/site/.cloudflare/prepare-build.mjs

@@ -5,6 +5,7 @@ import nodePath from 'node:path';
 
 await collectAndCopyDirToAssets('./pages');
 await collectAndCopyDirToAssets('./snippets');
+await collectAndCopyDirToAssets('./.next/.nodejsorg-cache');
 
 /**
  * @param {string} path

apps/site/next-data/generators/blogData.mjs

@@ -1,7 +1,13 @@
 'use strict';
 
-import { createReadStream } from 'node:fs';
-import { basename, extname, join } from 'node:path';
+import {
+  createReadStream,
+  existsSync,
+  mkdirSync,
+  writeFileSync,
+} from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { basename, dirname, extname, join } from 'node:path';
 import readline from 'node:readline';
 
 import graymatter from 'gray-matter';
@@ -58,6 +64,18 @@ const getFrontMatter = (filename, source) => {
  * @return {Promise<import('../../types').BlogData>}
  */
 const generateBlogData = async () => {
+  // Note: we cache the result of this function in a json file, we do this so that the cached
+  //       version can be read instead of re processing all the files, this is necessary for the
+  //       site to be deployed on a Cloudflare worker (which can't cope with reading hundred
+  //       of files)
+  const blogDataCacheFile = './.next/.nodejsorg-cache/blogData.json';
+  if (existsSync(blogDataCacheFile)) {
+    const rawContent = await readFile(blogDataCacheFile, 'utf8');
+    const content = JSON.parse(rawContent);
+    content.posts.forEach(p => (p.date = new Date(p.date)));
+    return content;
+  }
+
   // We retrieve the full pathnames of all Blog Posts to read each file individually
   const filenames = await getMarkdownFiles(process.cwd(), 'pages/en/blog', [
     '**/index.md',
@@ -102,7 +120,10 @@ const generateBlogData = async () => {
     )
   );
 
-  return { categories: [...blogCategories], posts };
+  const result = { categories: [...blogCategories], posts };
+  mkdirSync(dirname(blogDataCacheFile), { recursive: true });
+  writeFileSync(blogDataCacheFile, JSON.stringify(result));
+  return result;
 };
 
 export default generateBlogData;