build: run tests and build golang.go-nightly

Building and testing the extension requires a custom docker
image that includes Go, Node.js, jq, gopls, dlv, staticcheck, etc.

build-ci-image.yaml is a workflow that builds and stores the image
in Artifact Registry. (vscode-go-docker-repo/ci-image).

release-nightly.yaml workflow uses the container.

- The workflow clones the vscode-go repo (master branch).
- Before packaging, the "prepare nightly release" step
  adjusts the package.json for golang.go-nightly use.
- Then run tests. It's possible that the lint test and the
  tools/generate.go test may be unhappy about the spacing and formatting
  caused by the modification made during the previous step. However,
  the errors are not critical for packaging/publishing. Refactoring of
  all.bash was to skip the lint, generate.go tests during nightly
  release.

The workflow uploads the built extension to the project's GCS bucket but
does not publish the built extension to the marketplace yet.

Change-Id: Ib48fdc45f1fd1c86fcfc2d293e7f60690b2cad11
Reviewed-on: https://go-review.googlesource.com/c/vscode-go/+/510515
Reviewed-by: Robert Findley <[email protected]>
TryBot-Result: kokoro <[email protected]>
Commit-Queue: Hyang-Ah Hana Kim <[email protected]>

Author: hyangah

build/README.md

@@ -0,0 +1,9 @@
+### Release process
+
+The golang.go-nightly extension is released daily during weekdays, based on the latest commit to the master branch.
+
+(Note: the release process is currently in GH workflow. We are in process of moving it to a GCB workflow.)
+
+* Dockerfile: defines the image containing tools and environments needed to build and test the extension. (e.g. Go, Node.js, jq, etc.)
+* build-ci-image.yaml: defines the workflow to build the container image used for extension testing and releasing.
+* release-nightly.yaml: defines the workflow that builds the nightly extension and runs tests. The built extension is pushed only after all tests pass.

build/all.bash

@@ -1,4 +1,5 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+set -e
 
 # Copyright (C) Microsoft Corporation. All rights reserved.
 # Modification copyright 2020 The Go Authors. All rights reserved.
@@ -38,27 +39,31 @@ go_binaries_info() {
   go version
 }
 
-run_test() {
+run_doc_test() {
   df -h | grep shm
 
   echo "**** Run settings generator ****"
   go run ./tools/generate.go -w=false -gopls=true
 
-  echo "**** Run Go tests ****"
-  go test ./...
-
   echo "**** Test build ****"
   npm ci
   npm run compile
+}
+
+run_test() {
+  echo "**** Run Go tests ****"
+  go test ./...
 
   echo "**** Run test ****"
   npm run unit-test
   npm test --silent
+}
 
+run_lint() {
+  echo "**** Run lint ****"
   npm run lint
 }
 
-
 run_test_in_docker() {
   which npm && npm version || echo "no npm"
   which go && go version || echo "no go"
@@ -75,8 +80,7 @@ prepare_nightly() {
   #      on 2020/01/05 10:00
   local VER=`git log -1 --format=%cd --date="format:%Y.%-m.%-d%H"`
   local COMMIT=`git log -1 --format=%H`
-  echo "**** Preparing nightly release : $VER ***"
-
+  echo "**** Preparing nightly release : ${VER} (${COMMIT}) ***"
   # Update package.json
   (cat package.json | jq --arg VER "${VER}" '
 .version=$VER |
@@ -86,7 +90,7 @@ prepare_nightly() {
 .publisher="golang" |
 .description="Rich Go language support for Visual Studio Code (Nightly)" |
 .contributes.configuration.properties."go.delveConfig".properties.hideSystemGoroutines.default=true
-') > /tmp/package.json && mv /tmp/package.json package.json
+') > /tmp/package.json && cp /tmp/package.json package.json
 
   # Replace CHANGELOG.md with CHANGELOG.md + Release commit info.
   printf "**Release ${VER} @ ${COMMIT}** \n\n" | cat - CHANGELOG.md > /tmp/CHANGELOG.md.new && mv /tmp/CHANGELOG.md.new CHANGELOG.md
@@ -113,11 +117,17 @@ main() {
     "ci")
       go_binaries_info
       setup_virtual_display
+	  run_doc_test
       run_test
+	  run_lint
       ;;
     "prepare_nightly")
       prepare_nightly
       ;;
+	"test_nightly")
+	  setup_virtual_display
+	  run_test
+	  ;;
     *)
       usage
       exit 2

build/build-ci-image.yaml

@@ -0,0 +1,14 @@
+# Usage: gcloud builds submit --config build/build-ci-image.yaml .
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'bash'
+  args: ['-c', 'docker pull us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image:latest || exit 0']
+- name: 'gcr.io/cloud-builders/docker'
+  # https://cloud.google.com/build/docs/optimize-builds/speeding-up-builds
+  args: [
+        'build',
+        '-t', 'us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image:latest',
+        '--cache-from', 'us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image:latest',
+        '-f', 'build/Dockerfile',
+        '.']
+images: ['us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image']

build/release-nightly.yaml

@@ -1,15 +1,44 @@
+# This workflow will be triggered daily.
+# For local testing, run:
+#   gcloud builds submit --config release-nightly.yaml
+# This will check out the vscode-go repo master branch and run the build from it.
 steps:
-# TODO: check build/test status
-#
-# Install dependencies
-- name: node
+  # TODO: check build/test status
+- id: clone vscode-go repo
+  name: "gcr.io/cloud-builders/git"
+  args: ['clone', '--branch=master', "--depth=1", 'https://go.googlesource.com/vscode-go', 'vscode-go']
+- id: adjust file permissions
+  name: "gcr.io/cloud-builders/docker"
+  entrypoint: "chown"
+  args: ["-R", "1000:1000", "/workspace", "/builder/home"] # ci-image sets USER to node whose uid/gid are 1000.
+  dir: "/"
+- id: install npm dependencies
+  name: "us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image"
   entrypoint: npm
-  args: ['ci']
-# Build .vsix
-- name: node
+  args: ["ci"]
+  dir: "vscode-go"
+- id: prepare nightly release
+  name: "us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image"
+  entrypoint: "bash"
+  args: ["build/all.bash", 'prepare_nightly']
+  dir: "vscode-go"
+- id: build .vsix
+  name: "us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image"
   entrypoint: npm
-  args: ['run', 'package']
+  args: ["run", "package"] # we build vsix before running tests to avoid including unintentional changes.
+  dir: "vscode-go"
+- id: run tests
+  name: "us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image"
+  entrypoint: "bash"
+  args: ["build/all.bash", "test_nightly"]
+  dir: "vscode-go"
+  env:
+    - "IN_RELEASE_WORKFLOW=true"
+options:
+  substitution_option: "ALLOW_LOOSE"
+  machineType: 'E2_HIGHCPU_8'  # tests need powerful cpus to avoid timeout.
+images: ['us-docker.pkg.dev/$PROJECT_ID/vscode-go-docker-repo/ci-image:latest']
 artifacts:
   objects:
-    location: 'gs://$PROJECT_ID/nightly/$BUILD_ID'
-    paths: ['*.vsix']
+    location: "gs://$PROJECT_ID/nightly"
+    paths: ["vscode-go/*.vsix"] # vsix file base name includes the extension version.