From c97bc4d1320a1a1e705cf622d4138de9e51bfbb0 Mon Sep 17 00:00:00 2001
From: Kristian Freeman <kristian@kristianfreeman.com>
Date: Thu, 13 Apr 2023 11:11:55 -0500
Subject: [PATCH 1/2] Update action
---
.github/workflows/crawl.yml | 10 +++++-----
.gitignore | 2 ++
.github/workflows/crawl.js => crawl.js | 6 +++++-
package.json | 16 ++++++++++++++++
4 files changed, 28 insertions(+), 6 deletions(-)
create mode 100644 .gitignore
rename .github/workflows/crawl.js => crawl.js (94%)
create mode 100644 package.json
diff --git a/.github/workflows/crawl.yml b/.github/workflows/crawl.yml
index 57ef8b9d1..c44ce3dbd 100644
--- a/.github/workflows/crawl.yml
+++ b/.github/workflows/crawl.yml
@@ -1,6 +1,9 @@
name: Get all URLs and begin crawling
on:
+ pull_request:
+ branches:
+ - main
push:
branches:
- main
@@ -12,12 +15,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
- name: Run script
- uses: tj-actions/puppeteer@v4
- id: puppeteer
- with:
- files: |
- .github/workflows/crawl.js
+ run: npm run crawl
env:
ALGOLIA_CRAWLER_API_KEY: ${{secrets.ALGOLIA_CRAWLER_API_KEY}}
ALGOLIA_CRAWLER_ID: ${{secrets.ALGOLIA_CRAWLER_ID}}
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..25c8fdbab
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+node_modules
+package-lock.json
\ No newline at end of file
diff --git a/.github/workflows/crawl.js b/crawl.js
similarity index 94%
rename from .github/workflows/crawl.js
rename to crawl.js
index e7f6ec963..abbf0866b 100644
--- a/.github/workflows/crawl.js
+++ b/crawl.js
@@ -1,4 +1,3 @@
-const fetch = require('node-fetch');
const puppeteer = require('puppeteer');
(async () => {
@@ -9,6 +8,11 @@ const puppeteer = require('puppeteer');
apiKey: process.env.ALGOLIA_CRAWLER_API_KEY,
}
+ if (!crawler.url || !crawler.id || !crawler.userId || !crawler.apiKey) {
+ console.log("Missing required env vars")
+ return
+ }
+
const updateUrls = async (urls) => {
const updateResponse = await fetch(`${crawler.url}/${crawler.id}/config`, {
method: "PATCH",
diff --git a/package.json b/package.json
new file mode 100644
index 000000000..6937873a3
--- /dev/null
+++ b/package.json
@@ -0,0 +1,16 @@
+{
+ "name": "api-schemas",
+ "version": "1.0.0",
+ "description": "This repository contains OpenAPI schemas for the Cloudflare API.",
+ "main": "index.js",
+ "scripts": {
+ "crawl": "node crawl.js",
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "dependencies": {
+ "puppeteer": "^19.9.0"
+ }
+}
From 3c8c38efad712c50aec1767b9b7208d77e55d621 Mon Sep 17 00:00:00 2001
From: Kristian Freeman <kristian@kristianfreeman.com>
Date: Thu, 13 Apr 2023 11:13:47 -0500
Subject: [PATCH 2/2] npm install
---
.github/workflows/crawl.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/workflows/crawl.yml b/.github/workflows/crawl.yml
index c44ce3dbd..5a8faa926 100644
--- a/.github/workflows/crawl.yml
+++ b/.github/workflows/crawl.yml
@@ -16,6 +16,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
+ - run: npm install
- name: Run script
run: npm run crawl
env: