Merge branch 'main' into release

Author: andrew-polk

package-lock.json

@@ -26,7 +26,6 @@
   "//note: ts-node": "really is a runtime dependency",
   "dependencies": {
     "@notionhq/client": "2.2.3",
-    "axios": "^1.6.8",
     "chalk": "^4.1.2",
     "commander": "^9.2.0",
     "cosmiconfig": "^8.0.0",

package.json

@@ -0,0 +1,23 @@
+import * as fs from "fs-extra";
+import * as Path from "path";
+import { verbose } from "./log";
+
+export enum AssetType {
+  Image = "image",
+  Video = "video",
+}
+
+export function writeAsset(path: string, buffer: Buffer): void {
+  // Note: it's tempting to not spend time writing this out if we already have
+  // it from a previous run. But we don't really know it's the same. A) it
+  // could just have the same name, B) it could have been previously
+  // unlocalized and thus filled with a copy of the primary language image
+  // while and now is localized.
+  if (fs.pathExistsSync(path)) {
+    verbose("Replacing asset " + path);
+  } else {
+    verbose("Adding asset " + path);
+    fs.mkdirsSync(Path.dirname(path));
+  }
+  fs.createWriteStream(path).write(buffer); // async but we're not waiting
+}

src/assets.ts

@@ -1,15 +1,14 @@
 import * as fs from "fs-extra";
 import FileType, { FileTypeResult } from "file-type";
-import axios from "axios";
-import * as Path from "path";
 import { makeImagePersistencePlan } from "./MakeImagePersistencePlan";
-import { warning, logDebug, verbose, info } from "./log";
+import { logDebug, verbose } from "./log";
 import { ListBlockChildrenResponseResult } from "notion-to-md/build/types";
 import {
   IDocuNotionContext,
   IDocuNotionContextPageInfo,
   IPlugin,
 } from "./plugins/pluginTypes";
+import { writeAsset } from "./assets";
 
 // We handle several things here:
 // 1) copy images locally instead of leaving them in Notion
@@ -150,19 +149,26 @@ async function processImageBlock(
 async function readPrimaryImage(imageSet: ImageSet) {
   // In Mar 2024, we started having a problem getting a particular gif from imgur using
   // node-fetch. Switching to axios resolved it. I don't know why.
-  const response = await axios.get(imageSet.primaryUrl, {
-    responseType: "arraybuffer",
-  });
-  imageSet.primaryBuffer = Buffer.from(response.data, "utf-8");
+  // Then, in Apr 2025, we started getting 429 responses from imgur through axios,
+  // so we switched to node's built-in fetch (different than the node-fetch package).
+  // Just a guess, but probably imgur keeps locking down what it suspects as code running
+  // to scrape images.
+  // Apparently, imgur is getting to be more and more of a liability,
+  // so we should probably stop using it.
+  const response = await fetch(imageSet.primaryUrl);
+  const arrayBuffer = await response.arrayBuffer();
+  imageSet.primaryBuffer = Buffer.from(arrayBuffer);
   imageSet.fileType = await FileType.fromBuffer(imageSet.primaryBuffer);
 }
 
 async function saveImage(imageSet: ImageSet): Promise<void> {
-  writeImageIfNew(imageSet.primaryFileOutputPath!, imageSet.primaryBuffer!);
+  const path = imageSet.primaryFileOutputPath!;
+  imageWasSeen(path);
+  writeAsset(path, imageSet.primaryBuffer!);
 
   for (const localizedImage of imageSet.localizedUrls) {
     let buffer = imageSet.primaryBuffer!;
-    // if we have a urls for the localized screenshot, download it
+    // if we have a url for the localized screenshot, download it
     if (localizedImage?.url.length > 0) {
       verbose(`Retrieving ${localizedImage.iso632Code} version...`);
       const response = await fetch(localizedImage.url);
@@ -180,30 +186,15 @@ async function saveImage(imageSet: ImageSet): Promise<void> {
       imageSet.pageInfo!.relativeFilePathToFolderContainingPage
     }`;
 
-    writeImageIfNew(
-      (directory + "/" + imageSet.outputFileName!).replaceAll("//", "/"),
-      buffer
+    const newPath = (directory + "/" + imageSet.outputFileName!).replaceAll(
+      "//",
+      "/"
     );
+    imageWasSeen(newPath);
+    writeAsset(newPath, buffer);
   }
 }
 
-function writeImageIfNew(path: string, buffer: Buffer) {
-  imageWasSeen(path);
-
-  // Note: it's tempting to not spend time writing this out if we already have
-  // it from a previous run. But we don't really know it's the same. A) it
-  // could just have the same name, B) it could have been previously
-  // unlocalized and thus filled with a copy of the primary language image
-  // while and now is localized.
-  if (fs.pathExistsSync(path)) {
-    verbose("Replacing image " + path);
-  } else {
-    verbose("Adding image " + path);
-    fs.mkdirsSync(Path.dirname(path));
-  }
-  fs.createWriteStream(path).write(buffer); // async but we're not waiting
-}
-
 export function parseImageBlock(image: any): ImageSet {
   if (!locales) throw Error("Did you call initImageHandling()?");
   const imageSet: ImageSet = {

src/images.ts

@@ -1,7 +1,28 @@
+import * as fs from "fs-extra";
 import { setLogLevel } from "../log";
 import { NotionBlock } from "../types";
 import { standardVideoTransformer } from "./VideoTransformer";
-import { blocksToMarkdown } from "./pluginTestRun";
+import { blocksToMarkdown, kTemporaryTestDirectory } from "./pluginTestRun";
+
+beforeAll(async () => {
+  try {
+    if (await fs.pathExists(kTemporaryTestDirectory)) {
+      await fs.emptyDir(kTemporaryTestDirectory);
+    } else {
+      await fs.mkdirp(kTemporaryTestDirectory);
+    }
+  } catch (err) {
+    console.error("Error in beforeAll:", err);
+  }
+});
+
+afterAll(async () => {
+  try {
+    await fs.remove(kTemporaryTestDirectory);
+  } catch (err) {
+    console.error("Error in afterAll:", err);
+  }
+});
 
 test("youtube embedded", async () => {
   const config = { plugins: [standardVideoTransformer] };
@@ -89,6 +110,9 @@ test("video link, not embedded", async () => {
 test("direct upload to to Notion (embedded)", async () => {
   setLogLevel("verbose");
   const config = { plugins: [standardVideoTransformer] };
+
+  const fileName1 = "first_video.mp4";
+  const fileName2 = "second_video.mp4";
   const result = await blocksToMarkdown(config, [
     {
       object: "block",
@@ -103,13 +127,39 @@ test("direct upload to to Notion (embedded)", async () => {
         caption: [],
         type: "file",
         file: {
-          url: "https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/people_fre_motionAsset_p3.mp4?X-Blah-blah",
+          url: `https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/${fileName1}?X-Blah-blah`,
+        },
+      },
+    } as unknown as NotionBlock,
+    {
+      object: "block",
+      id: "12f7db3b-4412-4be9-a3f7-6ac423fee94b",
+      parent: {
+        type: "page_id",
+        page_id: "edaffeb2-ece8-4d44-976f-351e6b5757bb",
+      },
+
+      type: "video",
+      video: {
+        caption: [],
+        type: "file",
+        file: {
+          url: `https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/${fileName2}?X-Blah-blah`,
         },
       },
     } as unknown as NotionBlock,
   ]);
+
   expect(result).toContain(`import ReactPlayer from "react-player";`);
-  expect(result).toContain(
-    `<ReactPlayer controls url="https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/people_fre_motionAsset_p3.mp4?X-Blah-blah" />`
-  );
+  expect(result).toContain(`import video1 from "./${fileName1}";`);
+  expect(result).toContain(`import video2 from "./${fileName2}";`);
+  expect(result).toContain(`<ReactPlayer controls url={video1} />`);
+  expect(result).toContain(`<ReactPlayer controls url={video2} />`);
+
+  // Wait half a second for the files to be written
+  await new Promise(resolve => setTimeout(resolve, 500));
+
+  // We should have actually created files in "tempTestFileDir/"
+  expect(await fs.pathExists("tempTestFileDir/" + fileName1)).toBe(true);
+  expect(await fs.pathExists("tempTestFileDir/" + fileName2)).toBe(true);
 });