tweak continuous integration pipeline

* Fix "C1083: Cannot open include file: 'v8.h'" error. The error has recently started to occur on Windows system during node modules installing which involves native modules compiling process. The fix lies in "./scripts/ci/github/install-node-gyp.ps1". * Extend "build-proton-clients" job to please aaf0235 change/commit (caching "./app/assets/db-search-monaco-editor" dir). * Cache "yarn cache" dir instead of "./node_modules".
vladimiry · Mar 21, 2021 · d8d8ec4 · d8d8ec4
1 parent 5e26b6a
commit d8d8ec4
Show file tree

Hide file tree

Showing 13 changed files with 83 additions and 86 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -12,32 +12,50 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       # <common> TODO use YAML anchors feature when it gets supported
-      - { uses: actions/checkout@v2 }
-      - { uses: actions/cache@v2, id: cache-node-modules, with: { path: ./node_modules, key: "node-modules-cache-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.NODE_MODULES_CACHE_VERSION }}" } }
-      - { name: system setup, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh }
       - { uses: actions/setup-node@v1, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}" } }
-      - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os != 'Windows', run: "yarn --pure-lockfile --network-timeout 60000" }
-      - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Windows', shell: pwsh, run: ./scripts/ci/github/install-node-modules.ps1 }
+      - { uses: actions/checkout@v2 }
+      - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh }
+      - name: get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: cache yarn cache directory
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: "yarn-cache-dir-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.YARN_CACHE_DIR_CACHE_VERSION }}"
+      - { name: install node-gyp, if: runner.os == 'Windows', run: ./scripts/ci/github/install-node-gyp.ps1 }
+      - { name: install node modules, run: "yarn --pure-lockfile --network-timeout 60000" }
       # </common>
       - name: cache artifact
         uses: actions/cache@v2
-        with: { path: ./output/git/backup, key: "proton-clients-cache-${{ hashFiles('./src/shared/proton-apps-constants.ts') }}-${{ secrets.PROTON_CLIENTS_ARTIFACT_CACHE_VERSION }}" }
+        with:
+          path: |
+            ./output/git/backup
+            ./app/assets/db-search-monaco-editor
+          key: "proton-clients-cache-${{ hashFiles('./src/shared/proton-apps-constants.ts') }}-${{ secrets.PROTON_CLIENTS_ARTIFACT_CACHE_VERSION }}"
       - { name: print ./output/git/backup files, shell: bash, run: ./scripts/ci/list-fs-content.sh ./output/git/backup }
       - { name: build, run: ./scripts/ci/prepare-webclients.sh }
       - { name: print ./output/git/backup files, shell: bash, run: ./scripts/ci/list-fs-content.sh ./output/git/backup }
-      - { name: tar artifact, run: tar -cvf proton-clients-artifact.tar ./output/git/backup }
+      - { name: tar artifact, run: tar -cvf proton-clients-artifact.tar ./output/git/backup ./app/assets/db-search-monaco-editor }
       - { name: upload artifact, uses: actions/upload-artifact@v2, with: { name: proton-clients-artifact, path: ./proton-clients-artifact.tar, if-no-files-found: error } }
   build-app:
     strategy: { matrix: { os: [ ubuntu-20.04, windows-2019, macos-10.15 ] }, fail-fast: false }
     runs-on: ${{ matrix.os }}
     steps:
       # <common> TODO use YAML anchors feature when it gets supported
-      - { uses: actions/checkout@v2 }
-      - { uses: actions/cache@v2, id: cache-node-modules, with: { path: ./node_modules, key: "node-modules-cache-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.NODE_MODULES_CACHE_VERSION }}" } }
-      - { name: system setup, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh }
       - { uses: actions/setup-node@v1, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}" } }
-      - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os != 'Windows', run: "yarn --pure-lockfile --network-timeout 60000" }
-      - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Windows', shell: pwsh, run: ./scripts/ci/github/install-node-modules.ps1 }
+      - { uses: actions/checkout@v2 }
+      - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh }
+      - name: get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: cache yarn cache directory
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: "yarn-cache-dir-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.YARN_CACHE_DIR_CACHE_VERSION }}"
+      - { name: install node-gyp, if: runner.os == 'Windows', run: ./scripts/ci/github/install-node-gyp.ps1 }
+      - { name: install node modules, run: "yarn --pure-lockfile --network-timeout 60000" }
       # </common>
       - { name: build, run: 'npm exec --package=npm-run-all -- npm-run-all lint test:electron-main build' }
       - { name: print ./app files, shell: bash, run: ./scripts/ci/list-fs-content.sh ./app }
@@ -49,19 +67,26 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       # <common> TODO use YAML anchors feature when it gets supported
-      - { uses: actions/checkout@v2 }
-      - { uses: actions/cache@v2, id: cache-node-modules, with: { path: ./node_modules, key: "node-modules-cache-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.NODE_MODULES_CACHE_VERSION }}" } }
-      - { name: system setup, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh }
       - { uses: actions/setup-node@v1, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}" } }
-      - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os != 'Windows', run: "yarn --pure-lockfile --network-timeout 60000" }
-      - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Windows', shell: pwsh, run: ./scripts/ci/github/install-node-modules.ps1 }
+      - { uses: actions/checkout@v2 }
+      - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh }
+      - name: get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: cache yarn cache directory
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: "yarn-cache-dir-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.YARN_CACHE_DIR_CACHE_VERSION }}"
+      - { name: install node-gyp, if: runner.os == 'Windows', run: ./scripts/ci/github/install-node-gyp.ps1 }
+      - { name: install node modules, run: "yarn --pure-lockfile --network-timeout 60000" }
       # </common>
       - { name: download proton clients artifact, uses: actions/download-artifact@v2, with: { name: proton-clients-artifact } }
       - { name: unpack proton clients artifact, shell: bash, run: tar -xvf ./proton-clients-artifact.tar }
       - { name: download app artifact, uses: actions/download-artifact@v2, with: { name: 'app-artifact-${{ runner.os }}', path: ./app } }
       - { name: assets, run: "yarn assets", env: { ELECTRON_MAIL_SHOULD_FAIL_ON_BUILD: "true" } }
       - { name: print ./app files, shell: bash, run: ./scripts/ci/list-fs-content.sh ./app }
-      - { name: build, if: runner.os == 'Windows', shell: pwsh, run: ./scripts/ci/github/package-app.ps1 }
+      - { name: build, if: runner.os == 'Windows', run: ./scripts/ci/github/package-app.ps1 }
       - { name: build, if: runner.os == 'Linux', run: ./scripts/ci/github/package-app-linux.sh }
       - { name: build, if: matrix.os == 'macos-10.15', run: ./scripts/ci/github/package-app-osx.sh, env: { ARTIFACT_NAME_POSTFIX: "-catalina" } }
       - name: upload e2e/spectron output

diff --git a/.travis.yml b/.travis.yml
@@ -24,9 +24,9 @@ jobs:
   include:
     - stage: "build proton clients"
       <<: *macos-common
-      cache: { yarn: false, npm: false, directories: [ "./output/git/backup" ] }
+      cache: { yarn: false, npm: false, directories: [ "./output/git/backup", "./app/assets/db-search-monaco-editor" ] }
       script: ./scripts/ci/prepare-webclients.sh
-      workspaces: { create: { name: "macos-webclients-artifact", paths: [ "./output/git/backup" ] } }
+      workspaces: { create: { name: "macos-webclients-artifact", paths: [ "./output/git/backup", "./app/assets/db-search-monaco-editor" ] } }
     - stage: "build app package"
       <<: *macos-common
       <<: *macos-build-app

diff --git a/package.json b/package.json
@@ -94,7 +94,6 @@
     "test:electron-main:compiled:compile": "cross-env TS_NODE_FILES=true npx --no --node-options=\"--require tsconfig-paths/register\" ttsc --project ./src/electron-main/__test__/tsconfig.json",
     "test:web": "cross-env NODE_ENV=test TS_NODE_FILES=true npx --no --node-options=\"--require tsconfig-paths/register\" karma start ./src/web/browser-window/test/karma.conf.ts --single-run",
     "scripts/ci/appveyor/download-webclients-artifact": "yarn ts-node:shortcut ./scripts/ci/appveyor/download-webclients-artifact.ts",
-    "scripts/ci/print-webclients-dist-paths-pattern": "yarn --silent ts-node:shortcut ./scripts/ci/print-webclients-dist-paths-pattern.ts",
     "scripts/code-generation/electron-main": "yarn ts-node:shortcut ./scripts/code-generation/electron-main.ts",
     "scripts/dist-packages/print-hashes": "yarn ts-node:shortcut ./scripts/dist-packages/print-hashes.ts ./dist",
     "scripts/dist-packages/upload": "yarn ts-node:shortcut ./scripts/dist-packages/upload.ts ./dist ./output",

diff --git a/scripts/ci/archive-webclients-dist-only.sh b/scripts/ci/archive-webclients-dist-only.sh
@@ -5,7 +5,7 @@ set -ev
 ARCHIVE_FILE=$1
 
 # paths string separated with ";"
-WEBCLIENT_DIST_PATHS_STRING=$(yarn --silent scripts/ci/print-webclients-dist-paths-pattern)
+WEBCLIENT_DIST_PATHS_STRING=$(yarn --silent ts-node:shortcut ./scripts/ci/print-webclients-backup-paths-pattern.ts)
 # split paths string to array
 WEBCLIENT_DIST_PATHS=$(IFS=';';echo $WEBCLIENT_DIST_PATHS_STRING);
 

diff --git a/scripts/ci/github/install-node-gyp.ps1 b/scripts/ci/github/install-node-gyp.ps1
@@ -0,0 +1,6 @@
+npm install -g node-gyp
+npx node-gyp --verbose list
+npx node-gyp --verbose install $(node -v)
+npx node-gyp --verbose list
+# TODO resolve "node-gyp" cache path or set it for "node-gyp" explicitly rather than hardcoding the value
+ls "C:\Users\runneradmin\AppData\Local\node-gyp\Cache\$($(node -v).TrimStart('v'))\include\node"
diff --git a/scripts/ci/github/install-node-modules.ps1 b/scripts/ci/github/install-node-modules.ps1
diff --git a/scripts/ci/github/system-setup.ps1 → scripts/ci/github/install-vs-build-tools.ps1 b/scripts/ci/github/system-setup.ps1 → scripts/ci/github/install-vs-build-tools.ps1
diff --git a/scripts/ci/github/package-app.ps1 b/scripts/ci/github/package-app.ps1
@@ -1,21 +1,14 @@
-# enable debug output printing for "electron-builder" / "node-gyp" like modules
-$env:DEBUG = "true"
-
-echo "::group::system setup"
-./scripts/ci/github/system-setup.ps1 -IncludeWin81Sdk $true
+echo "::group::vs build tools setup"
+./scripts/ci/github/install-vs-build-tools.ps1 -IncludeWin81Sdk $true
 npm config set msvs_version 2017
 echo "::endgroup::"
 
-echo "::group::build native modules (electron-rebuild)"
-npm run postinstall:remove:prebuild-install
-npm run clean:prebuilds
-npm exec --package=electron-rebuild -- electron-rebuild --version $((Get-Content ./package.json | ConvertFrom-Json).devDependencies.electron)
-echo "::endgroup::"
-
 echo "::group::build native modules"
 npm run postinstall:remove:prebuild-install
 npm run clean:prebuilds
+# $env:DEBUG = "*"
 npm exec --package=electron-builder -- electron-builder install-app-deps --arch=x64
+# $env:DEBUG = $null
 echo "::endgroup::"
 
 echo "::group::test:e2e"

diff --git a/...ci/print-webclients-dist-paths-pattern.ts → .../print-webclients-backup-paths-pattern.ts b/...ci/print-webclients-dist-paths-pattern.ts → .../print-webclients-backup-paths-pattern.ts
diff --git a/scripts/prepare-webclient/index.ts b/scripts/prepare-webclient/index.ts
@@ -4,7 +4,7 @@ import pathIsInside from "path-is-inside";
 import {CWD_ABSOLUTE_DIR} from "scripts/const";
 import {buildProtonClients} from "scripts/prepare-webclient/protonmail";
 import {catchTopLeventAsync} from "scripts/lib";
-import {resolveProtonMetadata} from "scripts/prepare-webclient/monaco-editor-dts";
+import {generateDtsForMonacoEditor} from "scripts/prepare-webclient/monaco-editor-dts";
 
 const [, , appDestDir] = process.argv;
 
@@ -18,5 +18,5 @@ if (!pathIsInside(path.resolve(CWD_ABSOLUTE_DIR, appDestDir), CWD_ABSOLUTE_DIR))
 
 catchTopLeventAsync(async () => {
     await buildProtonClients({destDir: path.join(appDestDir, "./webclient")});
-    await resolveProtonMetadata({destDir: path.join(appDestDir)});
+    await generateDtsForMonacoEditor({destDir: appDestDir});
 });
diff --git a/scripts/prepare-webclient/protonmail-lib.ts → scripts/prepare-webclient/lib.ts b/scripts/prepare-webclient/protonmail-lib.ts → scripts/prepare-webclient/lib.ts
@@ -148,29 +148,20 @@ export async function executeBuildFlow<T extends FolderAsDomainEntry[], O = Unpa
 
         const repoDistBackupDir = resolveGitOutputBackupDir({repoType, suffix: `dist-${folderAsDomainEntry.folderNameAsDomain}`});
 
-        const installModules = async (): Promise<void> => {
-            if (fsExtra.pathExistsSync(path.resolve(repoDir, "node_modules"))) {
-                CONSOLE_LOG("Skip dependencies installing");
-            } else if (flow.install) {
-                await flow.install(flowOptions);
-            } else {
-                await execShell(["npm", ["ci"], {cwd: repoDir}]);
-            }
-        };
-
         if (fsExtra.pathExistsSync(repoDistBackupDir)) { // taking dist from the backup
             const src = repoDistBackupDir;
             const dest = repoDistDir;
 
             CONSOLE_LOG(`Copying backup ${src} to ${dest}`);
             await fsExtra.copy(src, dest);
-
-            if (repoType === "proton-mail") {
-                // installing modules since required by "./scripts/prepare-webclient/monaco-editor-dts.ts"
-                await installModules();
-            }
         } else { // executing the build
-            await installModules();
+            if (fsExtra.pathExistsSync(path.resolve(repoDir, "node_modules"))) {
+                CONSOLE_LOG("Skip dependencies installing");
+            } else if (flow.install) {
+                await flow.install(flowOptions);
+            } else {
+                await execShell(["npm", ["ci"], {cwd: repoDir}]);
+            }
 
             if (shouldFailOnBuild) {
                 throw new Error(`Halting since "${shouldFailOnBuildEnvVarName}" env var has been enabled`);

diff --git a/scripts/prepare-webclient/monaco-editor-dts.ts b/scripts/prepare-webclient/monaco-editor-dts.ts
@@ -3,7 +3,6 @@ import fsExtra from "fs-extra";
 import path from "path";
 
 import {CONSOLE_LOG} from "scripts/lib";
-import {IpcMainServiceScan} from "src/shared/api/main";
 import {PROTON_MONACO_EDITOR_DTS_ASSETS_LOCATION} from "src/shared/constants";
 import {PROTON_SHARED_MESSAGE_INTERFACE} from "src/shared/proton-apps-constants";
 
@@ -12,9 +11,9 @@ export const dtsGenerator: { // eslint-disable-line @typescript-eslint/no-unsafe
     default: (options: { baseDir: string, files: string[], out: string }) => Promise<string>
 } = require("dts-generator"); // eslint-disable-line @typescript-eslint/no-var-requires
 
-export const resolveProtonMetadata = async (
+export const generateDtsForMonacoEditor = async (
     {destDir}: { destDir: string },
-): Promise<IpcMainServiceScan["ApiImplReturns"]["staticInit"]["monacoEditorExtraLibArgs"]> => {
+): Promise<void> => {
     const options = {
         system: {
             base: "./node_modules/typescript/lib",
@@ -30,16 +29,25 @@ export const resolveProtonMetadata = async (
 
     // TODO replace "dts-generator" dependency with something capable to combine "./node_modules/typescript/lib/lib.esnext.d.ts"
     for (const key of [/* "system", */ "protonMessage"] as const) {
-        const sourceFile = options[key].in;
+        const {in: sourceFile, out: destFile, base: baseDir} = options[key];
+        if (fsExtra.pathExistsSync(destFile)) {
+            CONSOLE_LOG(`The "${destFile}" file already exists.`);
+            continue;
+        }
         if (!fsExtra.pathExistsSync(sourceFile)) {
             throw new Error(`The source "${sourceFile}" file doesn't exits.`);
         }
-        await dtsGenerator.default({baseDir: options[key].base, files: [sourceFile], out: options[key].out});
-        CONSOLE_LOG(`Merged "${options[key].in}" to "${options[key].out}"`);
+        await dtsGenerator.default({baseDir, files: [sourceFile], out: destFile});
+        CONSOLE_LOG(`Merged "${sourceFile}" to "${destFile}"`);
     }
 
     // TODO drop custom "./node_modules/typescript/lib/lib.esnext.d.ts" combining when "dts-generator" gets replaced
     {
+        const {in: sourceFile, out: destFile} = options.system;
+        if (fsExtra.pathExistsSync(destFile)) {
+            CONSOLE_LOG(`The "${destFile}" file already exists.`);
+            return;
+        }
         const referenceTagRe = /\/\/\/[\s\t]+<reference[\s\t]+lib=["']+(.*)["']+[\s\t]+\/>/;
         const mergedFiles: string[] = [];
         const extractContent = (file: string): string => {
@@ -59,23 +67,8 @@ export const resolveProtonMetadata = async (
             mergedFiles.push(file);
             return resultLines.join("\n");
         };
-        fsExtra.ensureDirSync(path.dirname(options.system.out));
-        fs.writeFileSync(options.system.out, extractContent(options.system.in));
-        CONSOLE_LOG(`Merged to "${options.system.out}" files:`, mergedFiles);
+        fsExtra.ensureDirSync(path.dirname(destFile));
+        fs.writeFileSync(destFile, extractContent(sourceFile));
+        CONSOLE_LOG(`Merged to "${destFile}" files:`, mergedFiles);
     }
-
-    return {
-        system: [
-            fs.readFileSync(options.system.out).toString(),
-            `in-memory:${options.system.in}`,
-        ],
-        protonMessage: [
-            (
-                fs.readFileSync(options.protonMessage.out).toString()
-                +
-                `declare const mail: Omit<import("lib/interfaces/mail/Message").Message, "Body"> & {Body: string};`
-            ),
-            `in-memory:${PROTON_SHARED_MESSAGE_INTERFACE.url}`,
-        ],
-    };
 };
diff --git a/scripts/prepare-webclient/protonmail.ts b/scripts/prepare-webclient/protonmail.ts
@@ -3,7 +3,7 @@ import path from "path";
 
 import {BINARY_NAME} from "src/shared/constants";
 import {CWD_ABSOLUTE_DIR} from "scripts/const";
-import {FolderAsDomainEntry, executeBuildFlow, printAndWriteFile} from "./protonmail-lib";
+import {FolderAsDomainEntry, executeBuildFlow, printAndWriteFile} from "./lib";
 import {PROVIDER_REPO_MAP, PROVIDER_REPO_NAMES} from "src/shared/proton-apps-constants";
 import {applyPatch, execShell} from "scripts/lib";