cache + speedup windows e2e (#5654)

* cache + speedup windows e2e

* fix README

* Merge prerelease jobs

* Run tests on PRs

* Fix turborepo

* Cache changesets

* Use turbo for create-cloudflare publishing

Author: penalosa

.github/workflows/README.md

@@ -6,7 +6,7 @@ See below for a summary of this repo's Actions
 
 ## PR related actions
 
-### Pull Request (pullrequests.yml)
+### Tests + Checks (test-and-check.yml)
 
 - Triggers
   - Updates to PRs.
@@ -28,7 +28,7 @@ See below for a summary of this repo's Actions
 ## Test old Node.js version (test-old-node-error.yml)
 
 - Triggers
-  - Updates to PRs.
+  - Commits merged to the `changeset-release/main` branch.
 - Actions
   - Makes sure that Wrangler's warning for old Node.js versions works.
 
@@ -75,15 +75,6 @@ See below for a summary of this repo's Actions
 
 ## Main branch actions
 
-### Main branch (main.yml)
-
-- Triggers
-  - Commits merged to the `main` branch, on the Cloudflare fork.
-- Actions
-  - Builds all the packages.
-  - Runs formatting, linting and type checks.
-  - Runs fixture tests, Wrangler unit tests, C3 unit tests, Miniflare unit tests, and ESLint + Prettier checks.
-
 ### Handle Changesets (changesets.yml)
 
 - Triggers
@@ -94,12 +85,13 @@ See below for a summary of this repo's Actions
     - Public packages are deployed to npm
     - Private packages will run their `deploy` script, if they have one.
 
-### Prerelease (prereleases.yml)
+### Publish @beta pre-releases (prereleases.yml)
 
 - Triggers
   - Commits merged to the `main` branch, on the Cloudflare fork.
 - Actions
   - Publishes the `wrangler` package to npm under the `beta` dist-tag.
+  - Publishes the `create-cloudflare` package to npm under the `beta` dist-tag.
 
 ## Product-specific branch actions
 
@@ -141,10 +133,3 @@ See below for a summary of this repo's Actions
   - Scheduled to run at 3am each day.
 - Actions
   - Deletes any Workers and Pages projects that were not properly cleaned up by the C3 E2E tests.
-
-### Prerelease create-cloudflare (prerelease-create-cloudflare.yml)
-
-- Triggers
-  - Commits merged to the `main` branch, on the Cloudflare fork.
-- Actions
-  - Publishes the `create-cloudflare` package to npm under the `beta` dist-tag.

.github/workflows/changesets.yml

@@ -39,16 +39,20 @@ jobs:
       - name: Check the changesets
         run: node -r esbuild-register tools/deployments/validate-changesets.ts
 
-      - name: Check the build
-        run: pnpm run build
-        env:
-          CI_OS: ${{ runner.os }}
+      - name: "Configure turbo repo cache environment variables"
+        run: |
+          touch .env
+          echo TURBO_API=${{ secrets.TURBO_API }} >> .env
+          echo TURBO_TEAM=${{ secrets.TURBO_TEAM }} >> .env
+          echo TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} >> .env
+          echo TURBO_REMOTE_CACHE_SIGNATURE_KEY=${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} >> .env
 
       - name: Create Version PR or Publish to NPM
         id: changesets
         uses: changesets/action@v1
         with:
           version: node .github/changeset-version.js
+          # This runs wrangler#prepublishOnly, which runs a turbo build at the root of the repo
           publish: pnpm exec changeset publish
         env:
           GITHUB_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/deploy-pages-previews.yml

@@ -25,7 +25,7 @@ jobs:
     # Only run this on PRs that are for the "cloudflare" org and not "from" `main`
     # - non-Cloudflare PRs will not have the secrets needed
     # - PRs "from" main would accidentally do a production deployment
-    if: github.repository_owner == 'cloudflare' && github.head_ref != 'main'
+    if: github.repository_owner == 'cloudflare' && github.head_ref != 'main' && (contains(github.event.*.labels.*.name, 'preview:wrangler-devtools') || contains(github.event.*.labels.*.name, 'preview:quick-edit') || contains(github.event.*.labels.*.name, 'preview:workers-playground'))
     timeout-minutes: 60
     concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/e2e.yml

@@ -39,6 +39,14 @@ jobs:
       - name: Install NPM Dependencies
         run: pnpm install --frozen-lockfile
 
+      - name: "Configure turbo repo cache environment variables"
+        run: |
+          touch .env
+          echo TURBO_API=${{ secrets.TURBO_API }} >> .env
+          echo TURBO_TEAM=${{ secrets.TURBO_TEAM }} >> .env
+          echo TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} >> .env
+          echo TURBO_REMOTE_CACHE_SIGNATURE_KEY=${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} >> .env
+
       - name: Run builds
         run: pnpm run build
         env:
@@ -53,14 +61,15 @@ jobs:
           NODE_ENV: "production"
 
       - name: Run tests
-        run: pnpm run --filter wrangler test:e2e
+        run: pnpm run test:e2e
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.TEST_CLOUDFLARE_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.TEST_CLOUDFLARE_ACCOUNT_ID }}
           WRANGLER: node ${{ github.workspace}}/temp/wrangler/bin/wrangler.js
           WRANGLER_IMPORT: ${{ github.workspace}}/temp/wrangler/wrangler-dist/cli.js
           NODE_OPTIONS: "--max_old_space_size=8192"
           WRANGLER_LOG_PATH: ${{ runner.temp }}/wrangler-debug-logs/
+          CI_OS: ${{ matrix.os }}
 
       - name: Upload debug logs
         if: always()

.github/workflows/prerelease-create-cloudflare.yml

@@ -1,4 +1,4 @@
-name: Prerelease
+name: Publish @beta pre-release
 
 on:
   push:
@@ -53,7 +53,7 @@ jobs:
         env:
           NODE_OPTIONS: "--max_old_space_size=8192"
 
-      - name: Publish Beta to NPM
+      - name: Publish wrangler@beta to NPM
         run: pnpm publish --tag beta
         env:
           NPM_PUBLISH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
@@ -64,6 +64,12 @@ jobs:
           ALGOLIA_PUBLIC_KEY: ${{ secrets.ALGOLIA_PUBLIC_KEY }}
         working-directory: packages/wrangler
 
+      - name: Publish create-cloudflare@beta to NPM
+        run: pnpm publish --tag beta
+        env:
+          NPM_PUBLISH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
+        working-directory: packages/create-cloudflare
+
       - name: Get Package Version
         run: echo "WRANGLER_VERSION=$(npm view wrangler@beta version)" >> $GITHUB_ENV
         working-directory: packages/wrangler

.github/workflows/prereleases.yml

@@ -1,11 +1,12 @@
-name: Pull Request
+name: Tests + Checks
 
 on: pull_request
 
 env:
   node-version: 16.18
 jobs:
   add-to-project:
+    if: github.ref != 'refs/heads/changeset-release/main'
     concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-add-pr
       cancel-in-progress: true
@@ -42,7 +43,7 @@ jobs:
           node-version: ${{ env.node-version }}
           cache: "pnpm"
 
-      - name: "Create .env file"
+      - name: "Configure turbo repo cache environment variables"
         run: |
           touch .env
           echo TURBO_API=${{ secrets.TURBO_API }} >> .env
@@ -91,7 +92,7 @@ jobs:
           node-version: ${{ env.node-version }}
           cache: "pnpm"
 
-      - name: "Create .env file"
+      - name: "Configure turbo repo cache environment variables"
         run: |
           touch .env
           echo TURBO_API=${{ secrets.TURBO_API }} >> .env
@@ -102,19 +103,14 @@ jobs:
       - name: Install NPM Dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Run builds
-        run: pnpm run build
-        env:
-          NODE_ENV: "production"
-          CI_OS: ${{ runner.os }}
-
       - name: Run tests
         run: pnpm run test:ci
         env:
           TMP_CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           TMP_CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           NODE_OPTIONS: "--max_old_space_size=8192"
           WRANGLER_LOG_PATH: ${{ runner.temp }}/wrangler-debug-logs/
+          CI_OS: ${{ runner.os }}
 
       - name: Upload debug logs
         if: always()

.github/workflows/pullrequests.yml .github/workflows/test-and-check.yml

@@ -1,6 +1,9 @@
 name: Test old Node.js version
 
-on: pull_request
+on:
+  push:
+    branches:
+      - changeset-release/main
 
 jobs:
   check:

.github/workflows/test-old-node-error.yml

@@ -94,7 +94,7 @@ async function runLongLivedWrangler(
 			separator,
 		].join("\n");
 		rejectReadyPromise(new Error(message));
-	}, 20_000);
+	}, 50_000);
 
 	async function stop() {
 		const closePromise = events.once(wranglerProcess, "close");

fixtures/shared/src/run-wrangler-long-lived.ts

@@ -21,6 +21,7 @@
 		"prettify": "prettier . --write --ignore-unknown",
 		"test": "vitest run --no-file-parallelism && dotenv -- turbo test --filter=wrangler --filter=miniflare --filter=kv-asset-handler --filter=@cloudflare/vitest-pool-workers --filter=@cloudflare/vitest-pool-workers-examples",
 		"test:ci": "dotenv -- turbo test:ci",
+		"test:e2e": "dotenv -- turbo test:e2e --filter=wrangler",
 		"test:watch": "turbo test:watch",
 		"type:tests": "dotenv -- turbo type:tests",
 		"gen:package": "turbo gen package"
@@ -30,7 +31,6 @@
 		"@types/node": "^20.0.0",
 		"cross-env": "^7.0.3",
 		"esbuild-register": "^3.5.0",
-		"ioredis": "^4.28.2",
 		"jsonc-parser": "^3.2.0",
 		"prettier": "^2.6.2",
 		"prettier-plugin-packagejson": "^2.2.18",

package.json

@@ -31,7 +31,7 @@
 		"check:lint": "eslint .",
 		"check:type": "tsc",
 		"lint": "eslint",
-		"prepublishOnly": "pnpm run build",
+		"prepublishOnly": "pnpm -w run build",
 		"test:e2e:cleanup": "node -r esbuild-register scripts/e2eCleanup.ts",
 		"test:e2e:npm": "pnpm run build && cross-env TEST_PM=npm vitest run --config ./vitest-e2e.config.ts",
 		"test:e2e:pnpm": "pnpm run build && cross-env TEST_PM=pnpm vitest run --config ./vitest-e2e.config.ts",

packages/create-cloudflare/package.json

@@ -60,7 +60,7 @@
 		"dev": "pnpm run clean && concurrently -c black,blue --kill-others-on-fail false \"pnpm run bundle --watch\" \"pnpm run check:type --watch --preserveWatchOutput\"",
 		"emit-types": "tsc -p tsconfig.emit.json && node -r esbuild-register scripts/emit-types.ts",
 		"generate-json-schema": "pnpm exec ts-json-schema-generator --no-type-check --path src/config/config.ts --type RawConfig --out config-schema.json",
-		"prepublishOnly": "SOURCEMAPS=false pnpm run build",
+		"prepublishOnly": "SOURCEMAPS=false pnpm run -w build",
 		"start": "pnpm run bundle && cross-env NODE_OPTIONS=--enable-source-maps ./bin/wrangler.js",
 		"test": "pnpm run assert-git-version && jest",
 		"test:ci": "pnpm run test",
@@ -113,7 +113,6 @@
 		"xxhash-wasm": "^1.0.1"
 	},
 	"devDependencies": {
-		"@cloudflare/ai": "^1.0.35",
 		"@cloudflare/cli": "workspace:*",
 		"@cloudflare/eslint-config-worker": "*",
 		"@cloudflare/pages-shared": "workspace:^",
@@ -127,7 +126,6 @@
 		"@sentry/types": "^7.86.0",
 		"@sentry/utils": "^7.86.0",
 		"@types/body-parser": "^1.19.2",
-		"@types/busboy": "^1.5.0",
 		"@types/command-exists": "^1.2.0",
 		"@types/express": "^4.17.13",
 		"@types/glob-to-regexp": "0.4.1",
@@ -142,7 +140,6 @@
 		"@types/serve-static": "^1.13.10",
 		"@types/shell-quote": "^1.7.2",
 		"@types/signal-exit": "^3.0.1",
-		"@types/source-map-support": "^0.5.7",
 		"@types/supports-color": "^8.1.1",
 		"@types/ws": "^8.5.3",
 		"@types/yargs": "^17.0.10",
@@ -160,7 +157,6 @@
 		"esbuild-jest": "0.5.0",
 		"execa": "^6.1.0",
 		"express": "^4.18.1",
-		"finalhandler": "^1.2.0",
 		"find-up": "^6.3.0",
 		"get-port": "^6.1.2",
 		"glob-to-regexp": "0.4.1",
@@ -171,8 +167,6 @@
 		"ink-select-input": "^4.2.1",
 		"ink-spinner": "^4.0.3",
 		"ink-table": "^3.0.0",
-		"ink-testing-library": "^2.1.0",
-		"ink-text-input": "^4.0.3",
 		"is-ci": "^3.0.1",
 		"javascript-time-ago": "^2.5.4",
 		"jest": "^29.7.0",
@@ -188,7 +182,6 @@
 		"prompts": "^2.4.2",
 		"react": "^17.0.2",
 		"react-error-boundary": "^3.1.4",
-		"remove-accents-esm": "^0.0.1",
 		"semiver": "^1.1.0",
 		"serve-static": "^1.15.0",
 		"shell-quote": "^1.8.1",