Add Algolia index script to CI

Author: pruett

.circleci/config.yml

@@ -22,6 +22,21 @@ jobs:
                 docker login -u $WEBSITE_DOCKER_USER -p $WEBSITE_DOCKER_PASS
                 docker push hashicorp/vagrant-website
             fi
+  algolia-index:
+    docker:
+      - image: node:12
+    steps:
+      - checkout
+      - run:
+          name: Push content to Algolia Index
+          command: |
+            if  [ "$CIRCLE_REPOSITORY_URL" != "[email protected]:hashicorp/vagrant.git" ]; then
+              echo "Not Vagrant OSS Repo, not indexing Algolia"
+              exit 0
+            fi
+            cd website/
+            npm install
+            node scripts/index_search_content.js
 workflows:
   version: 2
   build_website_docker_image:
@@ -31,3 +46,10 @@ workflows:
             branches:
               only:
                 - master
+  algolia_index:
+    jobs:
+      - algolia-index:
+          filters:
+            branches:
+              only:
+                - stable-website

website/.env

@@ -0,0 +1,3 @@
+NEXT_PUBLIC_ALGOLIA_APP_ID=YY0FFNI7MF
+NEXT_PUBLIC_ALGOLIA_INDEX=product_VAGRANT
+NEXT_PUBLIC_ALGOLIA_SEARCH_ONLY_API_KEY=5037da4824714676226913c65e961ca0

website/.gitignore

@@ -3,3 +3,7 @@ node_modules
 .next
 out
 .mdx-data
+
+# As per Next.js conventions (https://nextjs.org/docs/basic-features/environment-variables#default-environment-variables)
+!.env
+.env*.local

website/package-lock.json

@@ -18,7 +18,10 @@
     "@hashicorp/react-section-header": "^2.0.0",
     "@hashicorp/react-subnav": "^3.2.2",
     "@hashicorp/react-vertical-text-block-list": "^2.0.1",
+    "algoliasearch": "^4.3.0",
     "babel-plugin-import-glob-array": "^0.2.0",
+    "dotenv": "^8.2.0",
+    "gray-matter": "^4.0.2",
     "imagemin-mozjpeg": "^9.0.0",
     "imagemin-optipng": "^8.0.0",
     "imagemin-svgo": "^8.0.0",

website/package.json

@@ -0,0 +1,101 @@
+require('dotenv').config()
+
+const algoliasearch = require('algoliasearch')
+const glob = require('glob')
+const matter = require('gray-matter')
+const path = require('path')
+
+// In addition to the content of the page,
+// define additional front matter attributes that will be search-indexable
+const SEARCH_DIMENSIONS = ['page_title', 'description']
+
+main()
+
+async function main() {
+  const pagesFolder = path.join(__dirname, '../pages')
+
+  // Grab all search-indexable content and format for Algolia
+  const searchObjects = glob
+    .sync(path.join(pagesFolder, '**/*.mdx'))
+    .map((fullPath) => {
+      const { content, data } = matter.read(fullPath)
+
+      // Get path relative to `pages`
+      const __resourcePath = fullPath.replace(`${pagesFolder}/`, '')
+
+      // Use clean URL for Algolia id
+      const objectID = __resourcePath.replace('.mdx', '')
+
+      const searchableDimensions = Object.keys(data)
+        .filter((key) => SEARCH_DIMENSIONS.includes(key))
+        .map((dimension) => ({
+          [dimension]: data[dimension],
+        }))
+
+      return {
+        ...searchableDimensions,
+        content,
+        __resourcePath,
+        objectID,
+      }
+    })
+
+  try {
+    await indexSearchContent(searchObjects)
+  } catch (e) {
+    console.error(e)
+    process.exit(1)
+  }
+}
+
+async function indexSearchContent(objects) {
+  const {
+    NEXT_PUBLIC_ALGOLIA_APP_ID: appId,
+    NEXT_PUBLIC_ALGOLIA_INDEX: index,
+    ALGOLIA_API_KEY: apiKey,
+  } = process.env
+
+  if (!apiKey || !appId || !index) {
+    throw new Error(
+      `[*** Algolia Search Indexing Error ***] Received: ALGOLIA_API_KEY=${apiKey} ALGOLIA_APP_ID=${appId} ALGOLIA_INDEX=${index} \n Please ensure all Algolia Search-related environment vars are set in CI settings.`
+    )
+  }
+
+  console.log(`updating ${objects.length} indices...`)
+
+  try {
+    const searchClient = algoliasearch(appId, apiKey)
+    const searchIndex = searchClient.initIndex(index)
+
+    await searchIndex.partialUpdateObjects(objects, {
+      createIfNotExists: true,
+    })
+
+    // Remove indices for items that aren't included in the new batch
+    const newObjectIds = objects.map(({ objectID }) => objectID)
+    let staleObjects = []
+
+    await searchIndex.browseObjects({
+      query: '',
+      batch: (batch) => {
+        staleObjects = staleObjects.concat(
+          batch.filter(({ objectID }) => !newObjectIds.includes(objectID))
+        )
+      },
+    })
+
+    const staleIds = staleObjects.map(({ objectID }) => objectID)
+
+    if (staleIds.length > 0) {
+      console.log(`deleting ${staleIds.length} stale indices:`)
+      console.log(staleIds)
+
+      await searchIndex.deleteObjects(staleIds)
+    }
+
+    console.log('done')
+    process.exit(0)
+  } catch (error) {
+    throw new Error(error)
+  }
+}