feat: add support for external uploads (#1578)

This brings Google Drive and Dropbox support.  It will be exposed in the
next UI update on 2anki.net.

Closes: #1559
Closes: #1560

Documentation/README.md

@@ -0,0 +1,12 @@
+# Development
+
+This documentation is intended for developers, for users see https://docs.2anki.net/.
+
+
+## Database reminders
+
+To create a new migration file (from the source root directory):
+
+```bash
+ npx knex migrate:make descriptive-migration-name --knexfile ./src/KnexConfig.ts  --migrations-directory ../migrations -x js
+```

migrations/20240816074001_external-storage-dropbox.js

@@ -0,0 +1,16 @@
+module.exports.up = (knex) => {
+  return knex.schema.createTable('dropbox_uploads', (table) => {
+    table.increments('id').primary();
+    table.integer('bytes').notNullable();
+    table.string('icon').notNullable();
+    table.string('dropbox_id').notNullable();
+    table.boolean('isDir').notNullable();
+    table.string('link').notNullable();
+    table.string('linkType').notNullable();
+    table.string('name').notNullable();
+  });
+};
+
+module.exports.down = (knex) => {
+  return knex.schema.dropTable('dropbox_uploads');
+};

migrations/20240816074735_dropbox_uploads-add_owner.js

@@ -0,0 +1,19 @@
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = function(knex) {
+  return knex.schema.table('dropbox_uploads', (table) => {
+    table.integer('owner').notNullable();
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function(knex) {
+  return knex.schema.table('dropbox_uploads', (table) => {
+    table.dropColumn('owner');
+  });
+};

migrations/20240816122405_google_drive-uploads.js

@@ -0,0 +1,32 @@
+// migrations/20240816122405_google_drive_uploads.js
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.up = function(knex) {
+  return knex.schema.createTable('google_drive_uploads', function(table) {
+    table.string('id').primary();
+    table.string('description').notNullable();
+    table.string('embedUrl').notNullable();
+    table.string('iconUrl').notNullable();
+    table.bigInteger('lastEditedUtc').notNullable();
+    table.string('mimeType').notNullable();
+    table.string('name').notNullable().notNullable();
+    table.string('organizationDisplayName').notNullable();
+    table.string('parentId').notNullable();
+    table.string('serviceId').notNullable();
+    table.bigInteger('sizeBytes').notNullable();
+    table.string('type').notNullable();
+    table.string('url').notNullable().notNullable();
+    table.integer('owner').notNullable();
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = function(knex) {
+  return knex.schema.dropTable('google_drive_uploads');
+};

src/controllers/Upload/UploadController.ts

@@ -7,6 +7,8 @@ import UploadService from '../../services/UploadService';
 import { getUploadHandler } from '../../lib/misc/GetUploadHandler';
 import { isLimitError } from '../../lib/misc/isLimitError';
 import { handleUploadLimitError } from './helpers/handleUploadLimitError';
+import { handleDropbox } from './helpers/handleDropbox';
+import { handleGoogleDrive } from './helpers/handleGoogleDrive';
 
 class UploadController {
   constructor(
@@ -60,6 +62,18 @@ class UploadController {
       res.status(400);
     }
   }
+
+  async dropbox(req: express.Request, res: express.Response): Promise<void> {
+    await handleDropbox(req, res, this.service.handleUpload).then(() => {
+      console.debug('dropbox upload success');
+    });
+  }
+
+  async googleDrive(req: express.Request, res: express.Response) {
+    await handleGoogleDrive(req, res, this.service.handleUpload).then(() => {
+      console.debug('google drive upload success');
+    });
+  }
 }
 
 export default UploadController;

src/controllers/Upload/helpers/createGoogleDriveDownloadLink.ts

@@ -0,0 +1,10 @@
+import { GoogleDriveFile } from '../../../data_layer/GoogleDriveRepository';
+
+/**
+ * Create a download link for a Google Drive file. The default URL is just a preview link and request user interaction
+ * to download the file. This link will directly download the file.
+ * @param file
+ */
+export function createGoogleDriveDownloadLink(file: GoogleDriveFile) {
+  return 'https://www.googleapis.com/drive/v3/files/' + file.id + '?alt=media';
+}

src/controllers/Upload/helpers/getFilesOrEmpty.ts

@@ -0,0 +1,8 @@
+import { ParamsDictionary } from 'express-serve-static-core';
+
+export function getFilesOrEmpty<T>(body: ParamsDictionary): T[] {
+  if (body === undefined || body === null) {
+    return [];
+  }
+  return body.files ? JSON.parse(body.files) : [];
+}

src/controllers/Upload/helpers/handleDropbox.ts

@@ -0,0 +1,63 @@
+import axios from 'axios';
+import express from 'express';
+
+import {
+  DropboxFile,
+  DropboxRepository,
+} from '../../../data_layer/DropboxRepository';
+import { isPaying } from '../../../lib/isPaying';
+import { getUploadLimits } from '../../../lib/misc/getUploadLimits';
+import { handleUploadLimitError } from './handleUploadLimitError';
+import { getDatabase } from '../../../data_layer';
+import { getOwner } from '../../../lib/User/getOwner';
+import { isEmptyUpload } from './isEmptyUpload';
+import { getFilesOrEmpty } from './getFilesOrEmpty';
+
+export async function handleDropbox(
+  req: express.Request,
+  res: express.Response,
+  handleUpload: (req: express.Request, res: express.Response) => void
+) {
+  try {
+    const files = getFilesOrEmpty<DropboxFile>(req.body);
+    if (isEmptyUpload(files)) {
+      console.debug('No dropbox files selected.');
+      res
+        .status(400)
+        .json({ error: 'No dropbox files selected, one is required.' });
+      return;
+    }
+
+    const paying = isPaying(res.locals);
+    const limits = getUploadLimits(paying);
+    const totalSize = files.reduce((acc, file) => acc + file.bytes, 0);
+    if (!paying && totalSize > limits.fileSize) {
+      handleUploadLimitError(req, res);
+      return;
+    }
+    const repo = new DropboxRepository(getDatabase());
+    const owner = getOwner(res);
+    if (owner) {
+      await repo.saveFiles(files, owner);
+    } else {
+      console.log('Not storing anon users dropbox files');
+    }
+    // @ts-ignore
+    req.files = await Promise.all(
+      files.map(async (file) => {
+        const contents = await axios.get(file.link, {
+          responseType: 'arraybuffer',
+        });
+        return {
+          originalname: file.name,
+          size: file.bytes,
+          buffer: contents.data,
+        };
+      })
+    );
+    handleUpload(req, res);
+  } catch (error) {
+    console.debug('Error handling dropbox files', error);
+    res.status(400).json({ error: 'Error handling dropbox files' });
+  }
+}

src/controllers/Upload/helpers/handleGoogleDrive.ts

@@ -0,0 +1,78 @@
+import axios from 'axios';
+import express from 'express';
+
+import { isPaying } from '../../../lib/isPaying';
+import { getUploadLimits } from '../../../lib/misc/getUploadLimits';
+import { handleUploadLimitError } from './handleUploadLimitError';
+import { getDatabase } from '../../../data_layer';
+import { getOwner } from '../../../lib/User/getOwner';
+import {
+  GoogleDriveFile,
+  GoogleDriveRepository,
+} from '../../../data_layer/GoogleDriveRepository';
+import { isEmptyUpload } from './isEmptyUpload';
+import { getFilesOrEmpty } from './getFilesOrEmpty';
+import { createGoogleDriveDownloadLink } from './createGoogleDriveDownloadLink';
+
+export async function handleGoogleDrive(
+  req: express.Request,
+  res: express.Response,
+  handleUpload: (req: express.Request, res: express.Response) => void
+) {
+  try {
+    console.log('handling Google Drive files', req.body);
+    const files = getFilesOrEmpty<GoogleDriveFile>(req.body);
+    if (isEmptyUpload(files)) {
+      console.debug('No Google Drive files selected.');
+      res.status(400).send('No Google Drive files selected, one is required.');
+      return;
+    }
+
+    const googleDriveAuth = req.body.googleDriveAuth;
+    if (
+      googleDriveAuth === undefined ||
+      googleDriveAuth === null ||
+      googleDriveAuth === 'undefined' ||
+      googleDriveAuth === 'null'
+    ) {
+      res.status(400).send('Google Drive authentication is missing.');
+      return;
+    }
+
+    const paying = isPaying(res.locals);
+    const limits = getUploadLimits(paying);
+    const totalSize = files.reduce((acc, file) => acc + file.sizeBytes, 0);
+    if (!paying && totalSize > limits.fileSize) {
+      handleUploadLimitError(req, res);
+      return;
+    }
+    const repo = new GoogleDriveRepository(getDatabase());
+    const owner = getOwner(res);
+    if (owner) {
+      await repo.saveFiles(files, owner);
+    } else {
+      console.log('Not storing anon users Google Drive files');
+    }
+
+    // @ts-ignore
+    req.files = await Promise.all(
+      files.map(async (file) => {
+        const contents = await axios.get(createGoogleDriveDownloadLink(file), {
+          headers: {
+            Authorization: `Bearer ${googleDriveAuth}`,
+          },
+          responseType: 'blob',
+        });
+        return {
+          originalname: file.name,
+          size: file.sizeBytes,
+          buffer: contents.data,
+        };
+      })
+    );
+    handleUpload(req, res);
+  } catch (error) {
+    console.debug('Error handling Google files', error);
+    res.status(400).send('Error handling Google Drive files');
+  }
+}

src/controllers/Upload/helpers/isEmptyUpload.ts

@@ -0,0 +1,3 @@
+export function isEmptyUpload(files: unknown) {
+  return !files || !Array.isArray(files) || files.length === 0;
+}

src/data_layer/DropboxRepository.ts

@@ -0,0 +1,30 @@
+import { Knex } from 'knex';
+
+export type DropboxFile = {
+  bytes: number;
+  icon: string;
+  id: string;
+  isDir: boolean;
+  link: string;
+  linkType: string;
+  name: string;
+};
+
+export class DropboxRepository {
+  constructor(private readonly database: Knex) {}
+
+  async saveFiles(files: DropboxFile[], owner: number | string) {
+    await this.database('dropbox_uploads').insert(
+      files.map((file) => ({
+        owner,
+        bytes: file.bytes,
+        icon: file.icon,
+        dropbox_id: file.id,
+        isDir: file.isDir,
+        link: file.link,
+        linkType: file.linkType,
+        name: file.name,
+      }))
+    );
+  }
+}

src/data_layer/GoogleDriveRepository.ts

@@ -0,0 +1,66 @@
+import { Knex } from 'knex';
+
+export type GoogleDriveFile = {
+  downloadUrl?: string;
+  uploadState?: string;
+  description: string;
+  driveSuccess: boolean;
+  embedUrl: string;
+  iconUrl: string;
+  id: string;
+  isShared: boolean;
+  lastEditedUtc: number;
+  mimeType: string;
+  name: string;
+  rotation: number;
+  rotationDegree: number;
+  serviceId: string;
+  sizeBytes: number;
+  type: string;
+  url: string;
+};
+
+export class GoogleDriveRepository {
+  constructor(private readonly database: Knex) {}
+
+  private generateFileData(file: GoogleDriveFile, owner: number | string) {
+    return {
+      id: file.id,
+      description: file.description,
+      embedUrl: file.embedUrl,
+      iconUrl: file.iconUrl,
+      lastEditedUtc: file.lastEditedUtc,
+      mimeType: file.mimeType,
+      name: file.name,
+      organizationDisplayName: '', // Assuming default value
+      parentId: '', // Assuming default value
+      serviceId: file.serviceId,
+      sizeBytes: file.sizeBytes,
+      type: file.type,
+      url: file.url,
+      owner: owner,
+    };
+  }
+
+  async saveFiles(files: GoogleDriveFile[], owner: number | string) {
+    for (const file of files) {
+      const fileData = this.generateFileData(file, owner);
+      try {
+        await this.database('google_drive_uploads').insert(fileData);
+      } catch (error) {
+        if (!(error instanceof Error) || (error as any).code !== '23505')
+          throw error;
+
+        const existingFile = await this.database('google_drive_uploads')
+          .where({ id: file.id, owner: owner })
+          .first();
+
+        if (!existingFile) throw error;
+
+        await this.database('google_drive_uploads')
+          .where({ id: file.id, owner: owner })
+          .update(fileData);
+      }
+    }
+  }
+}

src/data_layer/UsersRepository.ts

@@ -60,6 +60,9 @@ class UsersRepository {
       'templates',
       'uploads',
       'blocks',
+      'dropbox_uploads',
+      'subscriptions',
+      'google_drive_uploads',
     ];
     return Promise.all([
       ...ownerTables.map((tableName) =>

src/lib/error/sendError.ts

@@ -1,4 +1,8 @@
 export const sendError = (error: unknown) => {
+  console.warn(
+    'sendError is deprecated, instead handle the error on the callsite'
+  );
+  console.error(error);
   if (error instanceof Error) {
     console.error(error);
   } else {