-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathupdatenotebooks.js
122 lines (117 loc) · 4.01 KB
/
updatenotebooks.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
// go through alll mdx files in the notebooks folder and get their url from the frontmatter
// If the url points to the latest version of a notebook, we use utils/githubApi module to get the latest sha.
import { readdirSync, readFileSync, writeFileSync } from "fs"
import { join } from "path"
import { parse, stringify } from "yaml"
import {
getLatestCommitFromUrl,
getIpynbContentsFromUrl,
} from "./utils/githubApi.js"
import { extractMdFromIpynbCells, getTitleFromIpynb } from "./utils/ipynb.js"
const NotebooksDir = "./src/content/notebooks"
const Notebooks = readdirSync(NotebooksDir)
console.log("NotebooksDir", NotebooksDir)
// if there is a argument to the script, we only update the notebook with the same name
const notebookToUpdate = process.argv[2]
if (notebookToUpdate) {
console.log("update only", notebookToUpdate)
}
// get all files in the dir
;(async () => {
for (const notebook of Notebooks) {
if (notebookToUpdate && notebook !== notebookToUpdate) {
console.log("skipping", notebook, "looking for", notebookToUpdate)
continue
}
const notebookPath = join(NotebooksDir, notebook)
console.log("\n- read notebook", notebookPath)
if (!notebook.endsWith(".mdx")) {
console.log("\x1b[33m⚠ skip\x1b[0m, not a mdx file ")
continue
}
console.log(" get `githubUrl` value from yaml frontmatter...")
const notebookMdx = readFileSync(notebookPath, "utf8")
const frontmatterMdx = notebookMdx.match(/---\n([\s\S]*?)\n---/)
if (!frontmatterMdx) {
console.log("\x1b[33m⚠ skip\x1b[0m, no frontmatter found")
continue
}
const frontmatter = parse(frontmatterMdx[1])
console.log("✓ frontmatter found", frontmatter)
const match = notebookMdx.match(/githubUrl:\s(https:\/\/.*)/)
if (!match) {
console.log("\x1b[33m⚠ skip\x1b[0m, no `githubUrl` found")
continue
}
const url = match[1].replace(/"/g, "")
console.log(" githubUrl:", url)
const commit = await getLatestCommitFromUrl(url)
if (!commit) {
console.log(
"\x1b[33m⚠ skip\x1b[0m, no commit found? Please double check tuhe url.",
)
continue
}
console.log("✓ sha:", commit.sha)
console.log("✓ date:", commit.commit.author.date)
// if the sha is the same as the one in the frontmatter, we skip
if (!process.env.FORCE_UPDATE && frontmatter.sha === commit.sha) {
console.log(
"\x1b[33m⚠ skip\x1b[0m, sha is the same as the one in the frontmatter",
)
continue
}
// get the ipynb content
const ipynbUrl = url
.replace("github.com", "raw.githubusercontent.com")
.replace("/blob", "")
console.log("✓ ipynbUrl:", ipynbUrl)
// get the content of the remote file
const ipynb = await getIpynbContentsFromUrl(ipynbUrl)
// save the content to the notebook file
if (!ipynb) {
console.log(
"\x1b[33m⚠ skip\x1b[0m, no ipynb found? Please double check the url.",
)
continue
}
console.log("✓ ipynb", ipynb.cells.length, "cells")
const {
title,
source: sourceWithoutTitle,
cellIdx,
} = getTitleFromIpynb(ipynb.cells)
if (!title) {
console.log("⚠ title? no heading found in ipynb")
continue
}
console.log("✓ title:", `\x1b[33m${title}\x1b[0m`)
const { content: contentMdx, links } = extractMdFromIpynbCells(
ipynb.metadata.kernelspec,
ipynb.cells.map((_d, i) => {
if (i !== cellIdx) {
return _d
}
return {
..._d,
source: sourceWithoutTitle,
}
}),
)
const googleColabUrl = `https://colab.research.google.com/${url.replace(
/https:\/\/.*?github.com/,
"github",
)}`
// get current yaml from frontùatter
const newFrontmatter = {
...frontmatter,
title,
sha: commit.sha,
date: commit.commit.author.date,
links: links,
googleColabUrl,
}
const newMdx = `---\n${stringify(newFrontmatter)}---\n\n${contentMdx}`
writeFileSync(notebookPath, newMdx)
}
})()