Merge pull request #57 from triggermesh/add-ci

implement ci

Author: Sameer Naik

.circleci/config.yml

@@ -0,0 +1,70 @@
+version: 2.1
+
+orbs:
+  go: circleci/go@1
+  gcp-cli: circleci/gcp-cli@1
+
+jobs:
+  build:
+    executor:
+      name: gcp-cli/default
+    steps:
+      - checkout
+      - gcp-cli/install
+      - gcp-cli/initialize
+      - run:
+          name: Building docker image
+          command: make -j6 cloudbuild-test
+
+  publish-image:
+    executor:
+      name: gcp-cli/google
+    steps:
+      - checkout
+      - gcp-cli/initialize
+      - run:
+          name: Publishing docker image
+          command: IMAGE_SHA=${CIRCLE_SHA1} IMAGE_TAG=${CIRCLE_TAG:-latest} make -j6 cloudbuild
+
+  release:
+    executor:
+      name: go/default
+      tag: '1.15'
+    steps:
+      - checkout
+      - run:
+          name: Installing github-release tool
+          command: go get github.com/meterup/github-release
+      - run:
+          name: Creating github release
+          command: |
+            PRE_RELEASE=${CIRCLE_TAG/${CIRCLE_TAG%-rc[0-9]*}/}
+            github-release delete -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -t ${CIRCLE_TAG} 2>/dev/null ||:
+            ./scripts/release-notes.sh ${CIRCLE_TAG} | github-release release ${PRE_RELEASE:+-p} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -t ${CIRCLE_TAG} -d -
+
+workflows:
+  build-test-and-publish:
+    jobs:
+      - build:
+          context: production
+          filters:
+            tags:
+              only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
+      - publish-image:
+          context: production
+          requires:
+            - build
+          filters:
+            tags:
+              only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
+            branches:
+              only: master
+      - release:
+          context: production
+          requires:
+            - publish-image
+          filters:
+            tags:
+              only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
+            branches:
+              ignore: /.*/

Makefile

@@ -0,0 +1,32 @@
+REPO               = knative-lambda-runtime
+REPO_DESC          = TriggerMesh Knative Lambda Runtime
+RUNTIMES           = java8 node10 node4 python27 python37 ruby25
+
+BASE_DIR          ?= $(CURDIR)
+
+DOCKER            ?= docker
+IMAGE_REPO        ?= gcr.io/triggermesh
+IMAGE_TAG         ?= latest
+IMAGE_SHA         ?= $(shell git rev-parse HEAD)
+
+.PHONY: help images cloudbuild-test cloudbuild
+
+all: images
+
+help: ## Display this help
+	@awk 'BEGIN {FS = ":.*?## "; printf "\n$(REPO_DESC)\nUsage:\n  make \033[36m<source>\033[0m\n"} /^[a-zA-Z0-9._-]+:.*?## / {printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+IMAGES = $(foreach r,$(RUNTIMES),$(r).image)
+images: $(IMAGES) ## Build the Docker images
+$(IMAGES): %.image:
+	docker build -t $(IMAGE_REPO)/knative-lambda-$* $*
+
+CLOUDBUILD_TEST = $(foreach r,$(RUNTIMES),$(r).cloudbuild-test)
+cloudbuild-test: $(CLOUDBUILD_TEST) ## Test container image build with Google Cloud Build
+$(CLOUDBUILD_TEST): %.cloudbuild-test:
+	gcloud builds submit $* --config cloudbuild.yaml --substitutions _RUNTIME=knative-lambda-$*,COMMIT_SHA=${IMAGE_SHA},_KANIKO_IMAGE_TAG=_
+
+CLOUDBUILD = $(foreach r,$(RUNTIMES),$(r).cloudbuild)
+cloudbuild: $(CLOUDBUILD) ## Build and publish image to GCR
+$(CLOUDBUILD): %.cloudbuild:
+	gcloud builds submit $* --config cloudbuild.yaml --substitutions _RUNTIME=knative-lambda-$*,COMMIT_SHA=${IMAGE_SHA},_KANIKO_IMAGE_TAG=${IMAGE_TAG}

README.md

@@ -18,7 +18,7 @@ equivalent to adding `--registry-host knative.registry.svc.cluster.local` to the
 so that builds can run without registry authentication.
 To override, set `--registry-secret` according to [tm docs](https://github.com/triggermesh/tm#docker-registry).
 
-### Concurrency 
+### Concurrency
 
 Concurrency in KLR represented by two components: parallel running [bootstrap](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-custom.html) processes per container and Knative [container concurrency](https://github.com/knative/serving/blob/master/docs/spec/spec.md#revision) model. By default [AWS runtime interface](https://github.com/triggermesh/aws-custom-runtime) fires up 4 bootstrap processes (functions, in other words) and allows multiple concurrent requests (`containerConcurrency: 0`) to be handled by each container. Default concurrency configuration can be changed on function deployment or update using `tm deploy service` command parameters:
 
@@ -38,7 +38,7 @@ NOTE: all examples below work with [Local Registry](https://github.com/triggerme
 1. Install runtime
 
 ```
-tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/python-3.7/runtime.yaml
+tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/python37/runtime.yaml
 ```
 
 2. Deploy [function](https://github.com/serverless/examples/tree/master/aws-python-simple-http-endpoint)
@@ -60,15 +60,15 @@ curl python-test.default.dev.triggermesh.io
 ```
 
 
-To use Python 2.7 runtime simply replace version tag in step 1 and 2 with `python-2.7` and `knative-python27-runtime` accordingly.
+To use Python 2.7 runtime simply replace version tag in step 1 and 2 with `python27` and `knative-python27-runtime` accordingly.
 
 
 #### Nodejs
 
 1. Install node 4.3 runtime
 
 ```
-tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/node-4.x/runtime.yaml
+tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/node4/runtime.yaml
 ```
 
 2. Deploy example function
@@ -110,10 +110,10 @@ EOF
 node -e "require('./handler').sayHelloAsync({}).then(h => console.log(h))"
 ```
 
-2. Install node-10.x runtime
+2. Install node10 runtime
 
 ```
-tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/node-10.x/runtime.yaml
+tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/node10/runtime.yaml
 ```
 
 3. Deploy function
@@ -171,7 +171,7 @@ func main() {
 2. Install Go runtime
 
 ```
-tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/go-1.x/runtime.yaml
+tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/go/runtime.yaml
 ```
 
 3. Deploy function
@@ -202,7 +202,7 @@ where `~/.ssh/id_rsa` is a path to SSH private key associated with your git acco
 1. Install Ruby 2.5 runtime
 
 ```
-tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/ruby-2.5/runtime.yaml
+tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/ruby25/runtime.yaml
 ```
 
 2. Deploy example function
@@ -258,18 +258,17 @@ curl http://aws-java-sample-java-function.default.dev.triggermesh.io -d '{"event
 {"message":"Hello, the current time is Tue Apr 07 13:59:17 GMT 2020"}
 ```
 
-### Run in Docker 
+### Run in Docker
 
 For cases in which the use of additional components (tm CLI, Tekton, Knative, k8s) is undesirable, it is possible to build a KLR function as a standalone Docker container and run it in any environment. To do this, you should extract the Dockerfile from the runtime you are interested in, put it in the directory with your function, update the handler variable, and build the container. Here are Dockerfile definitions for all runtimes:
 
-- [go](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/go-1.x/runtime.yaml#L44-L68)
-- [java](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/java8/runtime.yaml#L43-L53)
-- [node-10](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/node-10.x/runtime.yaml#L43-L48)
-- [node-4](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/node-4.x/runtime.yaml#L43-L48)
-- [python-2](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/python-2.7/runtime.yaml#L43-L50)
-- [python-3](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/python-3.7/runtime.yaml#L43-L50)
-- [ruby-2](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/ruby-2.5/runtime.yaml#L43-L47)
-
+- [go](https://github.com/triggermesh/knative-lambda-runtime/blob/master/go/runtime.yaml#L44-L68)
+- [java](https://github.com/triggermesh/knative-lambda-runtime/blob/master/java8/runtime.yaml#L43-L53)
+- [node-10](https://github.com/triggermesh/knative-lambda-runtime/blob/master/node10/runtime.yaml#L43-L48)
+- [node-4](https://github.com/triggermesh/knative-lambda-runtime/blob/master/node4/runtime.yaml#L43-L48)
+- [python-2](https://github.com/triggermesh/knative-lambda-runtime/blob/master/python27/runtime.yaml#L43-L50)
+- [python-3](https://github.com/triggermesh/knative-lambda-runtime/blob/master/python37/runtime.yaml#L43-L50)
+- [ruby-2](https://github.com/triggermesh/knative-lambda-runtime/blob/master/ruby25/runtime.yaml#L43-L47)
 
 Let's build a Python 3.7 function as an example:
 
@@ -294,7 +293,7 @@ def endpoint(event, context):
 EOF
 ```
 
-2. Extract the runtime's [Dockerfile](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/python-3.7/runtime.yaml#L43-L50), store it in the same directory, and update the `_HANDLER` variable:
+2. Extract the runtime's [Dockerfile](https://github.com/triggermesh/knative-lambda-runtime/blob/master/python37/runtime.yaml#L43-L50), store it in the same directory, and update the `_HANDLER` variable:
 
 ```
   cat > Dockerfile <<EOF
@@ -313,11 +312,11 @@ The `_HANDLER` variable in most cases consists of the filename without the file
 ```
   docker build -t python-klr-image .
   docker run -d --rm --name python-klr-container python-klr-image
-  # following command will work if you use Docker bridge network and you have jq tool 
-  # otherwise, you should get the container address manually  
+  # following command will work if you use Docker bridge network and you have jq tool
+  # otherwise, you should get the container address manually
   curl $(docker inspect python-klr-container | jq .[].NetworkSettings.Networks.bridge.IPAddress -r):8080
 ```
-  
+
   The response will contain a JSON document with the current time.
 
 4. Cleanup:

cloudbuild.yaml

@@ -1,32 +1,28 @@
 steps:
-## Build python37 runtime
-- name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-python37:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-python37:latest', '-f' ,'./python-3.7/Dockerfile', '.' ]
 
-## Build python27 runtime
-- name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-python27:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-python27:latest', '-f' ,'./python-2.7/Dockerfile', '.' ]
+- name: gcr.io/kaniko-project/executor:latest
+  args:
+  - --dockerfile=Dockerfile
+  - --build-arg=VERSION=${_KANIKO_IMAGE_TAG}
+  - --destination=gcr.io/$PROJECT_ID/${_RUNTIME}:${COMMIT_SHA}
+  - --destination=gcr.io/$PROJECT_ID/${_RUNTIME}:${_KANIKO_IMAGE_TAG}
+  - --cache-repo=gcr.io/$PROJECT_ID/${_RUNTIME}/cache
+  - --cache=${_KANIKO_USE_BUILD_CACHE}
+  - --no-push=${_KANIKO_NO_PUSH}
+  - ${_KANIKO_EXTRA_ARGS}
+  waitFor: ['-']
 
-## Build node4 runtime
-- name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-node4:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-node4:latest', '-f' ,'./node-4.x/Dockerfile', '.' ]
+timeout: 600s
 
-## Build node10 runtime
-- name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-node10:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-node10:latest', '-f' ,'./node-10.x/Dockerfile', '.' ]
+substitutions:
+  _RUNTIME:
+  _KANIKO_IMAGE_TAG: latest
+  _KANIKO_NO_PUSH: 'false'
+  _KANIKO_USE_BUILD_CACHE: 'true'
+  _KANIKO_EXTRA_ARGS:
 
-## Build ruby25 runtime
-- name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-ruby25:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-ruby25:latest', '-f' ,'./ruby-2.5/Dockerfile', '.' ]
+options:
+  substitution_option: ALLOW_LOOSE
 
-## Build java8 runtime
-- name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-java8:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-java8:latest', 'java8' ]
-
-images: 
-  - 'gcr.io/$PROJECT_ID/knative-lambda-python37'
-  - 'gcr.io/$PROJECT_ID/knative-lambda-python27'
-  - 'gcr.io/$PROJECT_ID/knative-lambda-node4'
-  - 'gcr.io/$PROJECT_ID/knative-lambda-node10'
-  - 'gcr.io/$PROJECT_ID/knative-lambda-ruby25'
-  - 'gcr.io/$PROJECT_ID/knative-lambda-java8'
+tags:
+  - knative-lambda-runtime

go-1.x/bootstrap.go renamed to go/bootstrap.go

@@ -21,7 +21,7 @@ spec:
   - name: IMAGE
     description: Where to store resulting image
   - name: SSH_KEY
-    description: SSH key 
+    description: SSH key
     default: "placeholder"
   - name: DIRECTORY
     description: The subdirectory of the workspace/repo
@@ -42,11 +42,11 @@ spec:
       cd /workspace/workspace/$(inputs.params.DIRECTORY)
       cat <<EOF > Dockerfile
         FROM golang:alpine
-        # Skip known public key check to be able to pull from private repositories 
+        # Skip known public key check to be able to pull from private repositories
         ENV GIT_SSH_COMMAND "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
         RUN apk --no-cache add git ca-certificates openssh \
         && go get github.com/triggermesh/aws-custom-runtime \
-        && go get github.com/triggermesh/knative-lambda-runtime/go-1.x \
+        && go get github.com/triggermesh/knative-lambda-runtime/go \
         && go get github.com/golang/dep/...
         WORKDIR /go/src/handler
         COPY . .
@@ -63,7 +63,7 @@ spec:
         ENV LAMBDA_TASK_ROOT "/opt"
         ENV _HANDLER "handler"
         COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-        COPY --from=0 /go/bin/go-1.x /opt/bootstrap
+        COPY --from=0 /go/bin/go /opt/bootstrap
         COPY --from=0 /go/bin/ /opt
         ENTRYPOINT ["/opt/aws-custom-runtime"]
       EOF
@@ -73,8 +73,8 @@ spec:
     - --context=/workspace/workspace/$(inputs.params.DIRECTORY)
     - --dockerfile=Dockerfile
     - --destination=$(inputs.params.IMAGE)
-    # Workaround not to use default config which requires gcloud credentials 
-    # to pull base image from public gcr registry 
+    # Workaround not to use default config which requires gcloud credentials
+    # to pull base image from public gcr registry
     # https://groups.google.com/d/msg/kaniko-users/r5yoP_Ejm_c/ExoEXksDBAAJ
     env:
     - name: DOCKER_CONFIG

go-1.x/runtime.yaml renamed to go/runtime.yaml

@@ -0,0 +1,5 @@
+Dockerfile
+.dockerignore
+
+**/*.md
+**/.gitignore

java8/.dockerignore

@@ -1,11 +1,16 @@
+FROM alpine:3 as downloader
+
+RUN apk --no-cache add curl \
+ && DOWNLOAD_URL=$(curl -sSf https://api.github.com/repos/triggermesh/aws-custom-runtime/releases/latest | grep "browser_download_url.*-linux-amd64" | cut -d: -f 2,3 | tr -d \") \
+ && curl -sSfL ${DOWNLOAD_URL} -o /opt/aws-custom-runtime \
+ && chmod +x /opt/aws-custom-runtime
+
 FROM openjdk:8-jre-alpine
 
-COPY runtime /var/runtime
+RUN apk add --no-cache libc6-compat
 
-RUN apk update && apk add --no-cache libc6-compat curl \
-    && API_VERSION=$(curl -sI https://github.com/triggermesh/aws-custom-runtime/releases/latest | grep -i "Location:" | awk -F "/" '{print $NF}' | tr -d "\r") \
-    && curl -sL https://github.com/triggermesh/aws-custom-runtime/releases/download/${API_VERSION}/aws-custom-runtime > /var/runtime/mockserver \
-    && chmod +x /var/runtime/mockserver
+COPY runtime /var/runtime
+COPY --from=downloader /opt/aws-custom-runtime /var/runtime/mockserver
 
 ENV LD_LIBRARY_PATH /lib64:/usr/lib64:/var/runtime:/var/runtime/lib:/var/task:/var/task/lib:/opt/lib
 ENV LAMBDA_RUNTIME_DIR /var/runtime
@@ -23,4 +28,4 @@ ENTRYPOINT ["/usr/bin/java", \
         "-Xshare:auto", \
         "-XX:-TieredCompilation", \
         "-jar", \
-        "/var/runtime/lib/LambdaJavaRTEntry-1.0.jar"]
+        "/var/runtime/lib/LambdaJavaRTEntry-1.0.jar"]