Sync collection

apps/worker/src/index.ts

@@ -5,4 +5,5 @@ import { Tasks } from "@playfulprogramming/common";
 createWorker(Tasks.POST_IMAGES, "./tasks/post-images/processor.ts");
 createWorker(Tasks.URL_METADATA, "./tasks/url-metadata/processor.ts");
 createWorker(Tasks.SYNC_AUTHOR, "./tasks/sync-author/processor.ts");
+createWorker(Tasks.SYNC_COLLECTION, "./tasks/sync-collection/processor.ts");
 createHealthcheck();

apps/worker/src/tasks/sync-collection/processor.ts

@@ -0,0 +1,175 @@
+import { Tasks, env } from "@playfulprogramming/common";
+import { collectionData, db } from "@playfulprogramming/db";
+import * as github from "@playfulprogramming/github-api";
+import { createProcessor } from "../../createProcessor.ts";
+import { eq } from "drizzle-orm";
+import matter from "gray-matter";
+import { CollectionMetaSchema } from "./types.ts";
+import { Value } from "@sinclair/typebox/value";
+import sharp from "sharp";
+import { Readable } from "node:stream";
+import { s3 } from "@playfulprogramming/s3";
+
+function extractLocale(name: string) {
+	const match = name.match(/\.([a-z]+)\.md$/);
+	return match ? match[1] : "en";
+}
+
+
+const IMAGE_SIZE_MAX = 2048;
+
+async function processImg(
+	stream: ReadableStream<Uint8Array>,
+	uploadKey: string,
+) {
+	const pipeline = sharp()
+		.resize({
+			width: IMAGE_SIZE_MAX,
+			height: IMAGE_SIZE_MAX,
+			fit: "inside",
+		})
+		.jpeg({ mozjpeg: true });
+
+	Readable.fromWeb(stream as never).pipe(pipeline);
+
+	const bucket = await s3.createBucket(env.S3_BUCKET);
+	await s3.upload(bucket, uploadKey, undefined, pipeline, "image/jpeg");
+}
+
+
+export default createProcessor(Tasks.SYNC_COLLECTION, async (job, { signal }) => {
+	const authorId = job.data.author;
+	const collectionId = job.data.collection;
+
+	const collectionMetaUrl = new URL(
+		`content/${encodeURIComponent(authorId)}/collections/${encodeURIComponent(collectionId)}`,
+		"http://localhost",
+	);
+
+	const collectionMetaResponse = await github.getContents({
+		ref: job.data.ref,
+		path: collectionMetaUrl.pathname,
+		repoOwner: env.GITHUB_REPO_OWNER,
+		repoName: env.GITHUB_REPO_NAME,
+		signal,
+	});
+
+	if (collectionMetaResponse.data === undefined) {
+		if (collectionMetaResponse.response.status == 404) {
+			console.log(
+				`Metadata for ${collectionId} (${collectionMetaUrl.pathname}) returned 404 - removing collection entry.`,
+			);
+			await db.delete(collectionData).where(eq(collectionData.slug, collectionId));
+			return;
+		}
+
+		throw new Error(`Unable to fetch collection data for ${collectionId}`);
+	}
+
+	if (!collectionMetaResponse.data.entries || !Array.isArray(collectionMetaResponse.data.entries)) {
+		throw new Error(`Unable to fetch collection data for ${collectionId}`);
+	}
+
+	type Entry = typeof collectionMetaResponse.data.entries[number]
+
+	const collectionEntries = collectionMetaResponse.data.entries.reduce(
+		(prev,
+			// entry.name is `index.md` and path is `content/{authorId}/collections/{collectionId}/index.md`
+			// We may have many locales in the future, so we need to check the path as well.
+			entry) => {
+			if (!(entry.name.startsWith("index") && entry.name.endsWith(".md"))) {
+				return prev;
+			}
+
+			prev.push({ entry, locale: extractLocale(entry.name) })
+			return prev;
+		},
+		[] as Array<{ entry: Entry, locale: string }>
+	)
+
+	// TODO: Should Promise.all()?
+	// Check if coverImg or socialImg have changed since last edit, if so upload to S3
+	for (let { entry, locale } of collectionEntries) {
+		const contentUrl = new URL(
+			entry.path,
+			"http://localhost",
+		);
+
+		const contentResponse = await github.getContentsRaw({
+			ref: job.data.ref,
+			path: contentUrl.pathname,
+			repoOwner: env.GITHUB_REPO_OWNER,
+			repoName: env.GITHUB_REPO_NAME,
+			signal,
+		});
+
+		if (contentResponse.data === undefined) {
+			throw new Error(`Unable to fetch collection content for ${collectionId} locale ${locale}`);
+		}
+
+		const { data } = matter(contentResponse.data);
+		const collectionParsedData = Value.Parse(CollectionMetaSchema, data);
+
+		let coverImgKey: string | null = null;
+		let socialImgKey: string | null = null;
+		if (collectionParsedData.coverImg) {
+			const coverImgUrl = new URL(collectionParsedData.coverImg, collectionMetaUrl);
+			const { data: coverImgStream } = await github.getContentsRawStream({
+				ref: job.data.ref,
+				path: coverImgUrl.pathname,
+				repoOwner: env.GITHUB_REPO_OWNER,
+				repoName: env.GITHUB_REPO_NAME,
+				signal,
+			});
+
+			if (coverImgStream === null || typeof coverImgStream === "undefined") {
+				throw new Error(
+					`Unable to fetch cover image for ${collectionId} (${coverImgUrl.pathname})`,
+				);
+			}
+
+			coverImgKey = `collections/${collectionId}/${locale}/cover.jpg`;
+			await processImg(coverImgStream, coverImgKey);
+		}
+
+		if (collectionParsedData.socialImg) {
+			const socialImgUrl = new URL(collectionParsedData.socialImg, collectionMetaUrl);
+			const { data: socialImgStream } = await github.getContentsRawStream({
+				ref: job.data.ref,
+				path: socialImgUrl.pathname,
+				repoOwner: env.GITHUB_REPO_OWNER,
+				repoName: env.GITHUB_REPO_NAME,
+				signal,
+			});
+
+			if (socialImgStream === null || typeof socialImgStream === "undefined") {
+				throw new Error(
+					`Unable to fetch social image for ${collectionId} (${socialImgUrl.pathname})`,
+				);
+			}
+
+			socialImgKey = `collections/${collectionId}/${locale}/social.jpg`;
+			await processImg(socialImgStream, socialImgKey);
+		}
+
+		const result = {
+			slug: collectionId,
+			locale: locale,
+			title: collectionParsedData.title,
+			description: collectionParsedData.description,
+			coverImage: coverImgKey,
+			socialImage: socialImgKey,
+			meta: {
+				buttons: collectionParsedData.buttons,
+				tags: collectionParsedData.tags,
+				chapterList: collectionParsedData.chapterList,
+			}
+		};
+
+		await db.insert(collectionData)
+			.values(result)
+			.onConflictDoUpdate({ target: [collectionData.slug, collectionData.locale], set: result });
+
+		// TODO: How to handle authors?
+	}
+});

apps/worker/src/tasks/sync-collection/types.ts

@@ -0,0 +1,90 @@
+import { Type } from "@sinclair/typebox";
+
+const CollectionButtonSchema = Type.Object(
+	{
+		text: Type.String(),
+		url: Type.String(),
+	},
+	{
+		additionalProperties: false,
+		examples: [
+			{
+				text: "Learn More",
+				url: "https://example.com/learn-more",
+			},
+		],
+	},
+);
+
+const CollectionCurrentPost = Type.Object(
+	{
+		post: Type.String(),
+	},
+	{
+		additionalProperties: false,
+		examples: [
+			{
+				post: "abc123",
+			},
+		],
+	},
+);
+
+const CollectionFuturePost = Type.Object(
+	{
+		order: Type.Number(),
+		title: Type.String(),
+		description: Type.String({ default: "" }),
+	},
+	{
+		additionalProperties: false,
+		examples: [
+			{
+				order: 1,
+				title: "Chapter One",
+				description: "An introduction to chapter one.",
+			},
+		],
+	},
+);
+
+export const CollectionMetaSchema = Type.Object(
+	{
+		title: Type.String(),
+		description: Type.String({ default: "" }),
+		authors: Type.Optional(Type.Array(Type.String())),
+		coverImg: Type.String(),
+		socialImg: Type.Optional(Type.String()),
+		type: Type.Optional(Type.Literal("book")),
+		pageLayout: Type.Optional(Type.Literal("none")),
+		customChaptersText: Type.Optional(Type.String()),
+		tags: Type.Optional(Type.Array(Type.String())),
+		published: Type.String(),
+		noindex: Type.Optional(Type.Boolean({ default: false })),
+		version: Type.Optional(Type.String()),
+		upToDateSlug: Type.Optional(Type.String()),
+		buttons: Type.Optional(Type.Array(CollectionButtonSchema)),
+		chapterList: Type.Optional(Type.Array(Type.Union([CollectionCurrentPost, CollectionFuturePost]))),
+	},
+	{
+		additionalProperties: false,
+		examples: [
+			{
+				title: "My Collection",
+				description: "A collection of my favorite posts.",
+				coverImg: "./cover.jpg",
+				published: "2023-01-01T00:00:00Z",
+				tags: ["tag1", "tag2"],
+				buttons: [
+					{
+						text: "Learn More",
+						url: "/learn-more",
+					},
+					{
+						post: "abc123",
+					},
+				],
+			},
+		],
+	},
+);

packages/db/src/schema/collections.ts

@@ -14,6 +14,7 @@ export const collections = pgTable("collections", {
 	slug: text("slug").primaryKey(),
 });
 
+// TODO: This is missing a ton of fields that are in the Collections task type, what do?
 export const collectionData = pgTable(
 	"collection_data",
 	{
@@ -25,6 +26,8 @@ export const collectionData = pgTable(
 		description: text("description").notNull().default(""),
 		publishedAt: timestamp("published_at", { withTimezone: true }),
 		meta: jsonb("meta").notNull(),
+		coverImage: text("cover_image"),
+		socialImage: text("social_image"),
 	},
 	(table) => [primaryKey({ columns: [table.slug, table.locale] })],
 );
@@ -46,6 +49,7 @@ export const collectionAuthors = pgTable(
 	],
 );
 
+// TODO: Do we need this in the DB or can it just be in the meta JSON?
 export const collectionChapters = pgTable("collection_chapters", {
 	id: integer("id").primaryKey().generatedAlwaysAsIdentity(),
 	locale: text("locale").notNull(),

packages/github-api/src/getContents.ts

@@ -9,7 +9,7 @@ export interface GetContentsParams {
 }
 
 export async function getContents(params: GetContentsParams) {
-	const response = await clientWithType("application/vnd.github.object").GET(
+	return await clientWithType("application/vnd.github.object").GET(
 		"/repos/{owner}/{repo}/contents/{path}",
 		{
 			params: {
@@ -28,14 +28,4 @@ export async function getContents(params: GetContentsParams) {
 			signal: params.signal,
 		},
 	);
-
-	const data = response.data;
-
-	if (typeof data === "undefined" || response.error) {
-		throw new Error(
-			`GitHub API (getContents) returned ${response.response.status} ${response.error}`,
-		);
-	}
-
-	return data;
 }