clean/big-clean-init-repo

appwrite/functions/sync-with-meilisearch/src/main.js

@@ -0,0 +1,106 @@
+import { Client, Databases, Query } from 'node-appwrite';
+import { getStaticFile, interpolate, throwIfMissing } from './utils.js';
+import { MeiliSearch } from 'meilisearch';
+
+export default async ({ req, res, log }) => {
+  throwIfMissing(process.env, [
+    'APPWRITE_DATABASE_ID',
+    'APPWRITE_COLLECTION_ID',
+    'MEILISEARCH_ENDPOINT',
+    'MEILISEARCH_INDEX_NAME',
+    'MEILISEARCH_ADMIN_API_KEY',
+    'MEILISEARCH_SEARCH_API_KEY',
+  ]);
+
+  if (req.method === 'GET') {
+    const html = interpolate(getStaticFile('index.html'), {
+      MEILISEARCH_ENDPOINT: process.env.MEILISEARCH_ENDPOINT,
+      MEILISEARCH_INDEX_NAME: process.env.MEILISEARCH_INDEX_NAME,
+      MEILISEARCH_SEARCH_API_KEY: process.env.MEILISEARCH_SEARCH_API_KEY,
+    });
+
+    return res.text(html, 200, { 'Content-Type': 'text/html; charset=utf-8' });
+  }
+
+  const client = new Client()
+    .setEndpoint(process.env.APPWRITE_FUNCTION_API_ENDPOINT)
+    .setProject(process.env.APPWRITE_FUNCTION_PROJECT_ID)
+    .setKey(req.headers['x-appwrite-key']);
+
+  const databases = new Databases(client);
+
+  const meilisearch = new MeiliSearch({
+    host: process.env.MEILISEARCH_ENDPOINT,
+    apiKey: process.env.MEILISEARCH_ADMIN_API_KEY,
+  });
+
+  const index = meilisearch.index(process.env.MEILISEARCH_INDEX_NAME);
+
+  // 🟢 Étape 1 : Récupérer tous les IDs actuels dans Meilisearch avec pagination
+  const meiliIds = [];
+  let offset = 0;
+  const limit = 1000;
+
+  while (true) {
+    const meilisearchDocuments = await index.getDocuments({
+      fields: ["$id"],
+      limit,
+      offset
+    });
+
+    if (meilisearchDocuments.results.length === 0) break;
+
+    meiliIds.push(...meilisearchDocuments.results.map((doc) => doc.$id));
+    log(`Fetched ${meiliIds.length} documents from Meilisearch...`);
+
+    offset += limit;
+  }
+
+  // 🟢 Étape 2 : Récupérer tous les documents depuis Appwrite avec pagination
+  let appwriteIds = [];
+  let documents = [];
+  let cursor = null;
+
+  do {
+    const queries = [Query.limit(100)];
+
+    if (cursor) {
+      queries.push(Query.cursorAfter(cursor));
+    }
+
+    const response = await databases.listDocuments(
+      process.env.APPWRITE_DATABASE_ID,
+      process.env.APPWRITE_COLLECTION_ID,
+      queries
+    );
+
+    if (response.documents.length > 0) {
+      cursor = response.documents[response.documents.length - 1].$id;
+      appwriteIds.push(...response.documents.map((doc) => doc.$id));
+      documents.push(...response.documents);
+    } else {
+      log(`No more documents found.`);
+      cursor = null;
+      break;
+    }
+
+    log(`Syncing chunk of ${response.documents.length} documents ...`);
+    await index.addDocuments(response.documents, { primaryKey: '$id' });
+  } while (cursor !== null);
+
+  // 🟢 Étape 3 : Identifier les documents obsolètes à supprimer
+  const idsToDelete = meiliIds.filter((id) => !appwriteIds.includes(id));
+  log(`Found ${idsToDelete.length} obsolete documents to delete.`);
+
+  // 🟢 Étape 4 : Supprimer les documents obsolètes par lots de 1000
+  const deleteBatchSize = 1000;
+  for (let i = 0; i < idsToDelete.length; i += deleteBatchSize) {
+    const batch = idsToDelete.slice(i, i + deleteBatchSize);
+    await index.deleteDocuments(batch);
+    log(`Deleted ${batch.length} obsolete documents.`);
+  }
+
+  log('Sync finished.');
+
+  return res.text('Sync finished.', 200);
+};

appwrite/functions/sync-with-meilisearch/src/utils.js

@@ -0,0 +1,43 @@
+import path from 'path';
+import { fileURLToPath } from 'url';
+import fs from 'fs';
+
+/**
+ * Throws an error if any of the keys are missing from the object
+ * @param {*} obj
+ * @param {string[]} keys
+ * @throws {Error}
+ */
+export function throwIfMissing(obj, keys) {
+  const missing = [];
+  for (let key of keys) {
+    if (!(key in obj) || !obj[key]) {
+      missing.push(key);
+    }
+  }
+  if (missing.length > 0) {
+    throw new Error(`Missing required fields: ${missing.join(', ')}`);
+  }
+}
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const staticFolder = path.join(__dirname, '../static');
+
+/**
+ * Returns the contents of a file in the static folder
+ * @param {string} fileName
+ * @returns {string} Contents of static/{fileName}
+ */
+export function getStaticFile(fileName) {
+  return fs.readFileSync(path.join(staticFolder, fileName)).toString();
+}
+
+/**
+ * @param {string} template
+ * @param {Record<string, string | undefined>} values
+ * @returns {string}
+ */
+export function interpolate(template, values) {
+  return template.replace(/{{([^}]+)}}/g, (_, key) => values[key] || '');
+}

appwrite/functions/sync-with-meilisearch/static/index.html

@@ -0,0 +1,72 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Meilisearch demo</title>
+
+    <script src="https://unpkg.com/[email protected]"></script>
+    <script src="https://unpkg.com/alpinejs" defer></script>
+
+    <link rel="stylesheet" href="https://unpkg.com/@appwrite.io/pink@0" />
+    <link rel="stylesheet" href="https://unpkg.com/@appwrite.io/pink-icons@0" />
+  </head>
+  <body class="theme-dark">
+    <main class="main-content">
+      <div class="top-cover u-padding-block-end-56">
+        <div class="container">
+          <div
+            class="u-flex u-gap-16 u-flex-justify-center u-margin-block-start-16"
+          >
+            <h1 class="heading-level-1">Meilisearch demo</h1>
+            <code class="u-un-break-text"></code>
+          </div>
+          <p
+            class="body-text-1 u-normal u-margin-block-start-8"
+            style="max-width: 50rem"
+          >
+            Use this demo to verify that the sync between Appwrite Databases and
+            Meilisearch was successful. Search your Meilisearch index using the
+            input below.
+          </p>
+        </div>
+      </div>
+      <div
+        class="container u-margin-block-start-negative-56"
+        x-data="{ search: '', results: [ ] }"
+        x-init="$watch('search', async (value) => { results = await onSearch(value) })"
+      >
+        <div class="card u-flex u-gap-24 u-flex-vertical">
+          <div id="searchbox">
+            <div
+              class="input-text-wrapper is-with-end-button u-width-full-line"
+            >
+              <input x-model="search" type="search" placeholder="Search" />
+              <div class="icon-search" aria-hidden="true"></div>
+            </div>
+          </div>
+          <div id="hits" class="u-flex u-flex-vertical u-gap-12">
+            <template x-for="result in results">
+              <div class="card">
+                <pre x-text="JSON.stringify(result, null, '\t')"></pre>
+              </div>
+            </template>
+          </div>
+        </div>
+      </div>
+    </main>
+    <script>
+      const meilisearch = new MeiliSearch({
+        host: '{{MEILISEARCH_ENDPOINT}}',
+        apiKey: '{{MEILISEARCH_SEARCH_API_KEY}}',
+      });
+
+      const index = meilisearch.index('{{MEILISEARCH_INDEX_NAME}}');
+
+      window.onSearch = async function (prompt) {
+        return (await index.search(prompt)).hits;
+      };
+    </script>
+  </body>
+</html>