Skip to content
This repository was archived by the owner on Dec 11, 2023. It is now read-only.

Commit

Permalink
Merge pull request #57 from triggermesh/add-ci
Browse files Browse the repository at this point in the history
implement ci
  • Loading branch information
Sameer Naik authored Nov 24, 2020
2 parents aa3b33b + 0ef778e commit bda5601
Show file tree
Hide file tree
Showing 40 changed files with 324 additions and 125 deletions.
70 changes: 70 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
version: 2.1

orbs:
go: circleci/go@1
gcp-cli: circleci/gcp-cli@1

jobs:
build:
executor:
name: gcp-cli/default
steps:
- checkout
- gcp-cli/install
- gcp-cli/initialize
- run:
name: Building docker image
command: make -j6 cloudbuild-test

publish-image:
executor:
name: gcp-cli/google
steps:
- checkout
- gcp-cli/initialize
- run:
name: Publishing docker image
command: IMAGE_SHA=${CIRCLE_SHA1} IMAGE_TAG=${CIRCLE_TAG:-latest} make -j6 cloudbuild

release:
executor:
name: go/default
tag: '1.15'
steps:
- checkout
- run:
name: Installing github-release tool
command: go get github.com/meterup/github-release
- run:
name: Creating github release
command: |
PRE_RELEASE=${CIRCLE_TAG/${CIRCLE_TAG%-rc[0-9]*}/}
github-release delete -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -t ${CIRCLE_TAG} 2>/dev/null ||:
./scripts/release-notes.sh ${CIRCLE_TAG} | github-release release ${PRE_RELEASE:+-p} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -t ${CIRCLE_TAG} -d -
workflows:
build-test-and-publish:
jobs:
- build:
context: production
filters:
tags:
only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
- publish-image:
context: production
requires:
- build
filters:
tags:
only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
branches:
only: master
- release:
context: production
requires:
- publish-image
filters:
tags:
only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
branches:
ignore: /.*/
32 changes: 32 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
REPO = knative-lambda-runtime
REPO_DESC = TriggerMesh Knative Lambda Runtime
RUNTIMES = java8 node10 node4 python27 python37 ruby25

BASE_DIR ?= $(CURDIR)

DOCKER ?= docker
IMAGE_REPO ?= gcr.io/triggermesh
IMAGE_TAG ?= latest
IMAGE_SHA ?= $(shell git rev-parse HEAD)

.PHONY: help images cloudbuild-test cloudbuild

all: images

help: ## Display this help
@awk 'BEGIN {FS = ":.*?## "; printf "\n$(REPO_DESC)\nUsage:\n make \033[36m<source>\033[0m\n"} /^[a-zA-Z0-9._-]+:.*?## / {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)

IMAGES = $(foreach r,$(RUNTIMES),$(r).image)
images: $(IMAGES) ## Build the Docker images
$(IMAGES): %.image:
docker build -t $(IMAGE_REPO)/knative-lambda-$* $*

CLOUDBUILD_TEST = $(foreach r,$(RUNTIMES),$(r).cloudbuild-test)
cloudbuild-test: $(CLOUDBUILD_TEST) ## Test container image build with Google Cloud Build
$(CLOUDBUILD_TEST): %.cloudbuild-test:
gcloud builds submit $* --config cloudbuild.yaml --substitutions _RUNTIME=knative-lambda-$*,COMMIT_SHA=${IMAGE_SHA},_KANIKO_IMAGE_TAG=_

CLOUDBUILD = $(foreach r,$(RUNTIMES),$(r).cloudbuild)
cloudbuild: $(CLOUDBUILD) ## Build and publish image to GCR
$(CLOUDBUILD): %.cloudbuild:
gcloud builds submit $* --config cloudbuild.yaml --substitutions _RUNTIME=knative-lambda-$*,COMMIT_SHA=${IMAGE_SHA},_KANIKO_IMAGE_TAG=${IMAGE_TAG}
41 changes: 20 additions & 21 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ equivalent to adding `--registry-host knative.registry.svc.cluster.local` to the
so that builds can run without registry authentication.
To override, set `--registry-secret` according to [tm docs](https://github.com/triggermesh/tm#docker-registry).

### Concurrency
### Concurrency

Concurrency in KLR represented by two components: parallel running [bootstrap](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-custom.html) processes per container and Knative [container concurrency](https://github.com/knative/serving/blob/master/docs/spec/spec.md#revision) model. By default [AWS runtime interface](https://github.com/triggermesh/aws-custom-runtime) fires up 4 bootstrap processes (functions, in other words) and allows multiple concurrent requests (`containerConcurrency: 0`) to be handled by each container. Default concurrency configuration can be changed on function deployment or update using `tm deploy service` command parameters:

Expand All @@ -38,7 +38,7 @@ NOTE: all examples below work with [Local Registry](https://github.com/triggerme
1. Install runtime

```
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/python-3.7/runtime.yaml
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/python37/runtime.yaml
```

2. Deploy [function](https://github.com/serverless/examples/tree/master/aws-python-simple-http-endpoint)
Expand All @@ -60,15 +60,15 @@ curl python-test.default.dev.triggermesh.io
```


To use Python 2.7 runtime simply replace version tag in step 1 and 2 with `python-2.7` and `knative-python27-runtime` accordingly.
To use Python 2.7 runtime simply replace version tag in step 1 and 2 with `python27` and `knative-python27-runtime` accordingly.


#### Nodejs

1. Install node 4.3 runtime

```
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/node-4.x/runtime.yaml
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/node4/runtime.yaml
```

2. Deploy example function
Expand Down Expand Up @@ -110,10 +110,10 @@ EOF
node -e "require('./handler').sayHelloAsync({}).then(h => console.log(h))"
```

2. Install node-10.x runtime
2. Install node10 runtime

```
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/node-10.x/runtime.yaml
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/node10/runtime.yaml
```

3. Deploy function
Expand Down Expand Up @@ -171,7 +171,7 @@ func main() {
2. Install Go runtime

```
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/go-1.x/runtime.yaml
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/go/runtime.yaml
```

3. Deploy function
Expand Down Expand Up @@ -202,7 +202,7 @@ where `~/.ssh/id_rsa` is a path to SSH private key associated with your git acco
1. Install Ruby 2.5 runtime

```
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/ruby-2.5/runtime.yaml
tm deploy task -f https://raw.githubusercontent.com/triggermesh/knative-lambda-runtime/master/ruby25/runtime.yaml
```

2. Deploy example function
Expand Down Expand Up @@ -258,18 +258,17 @@ curl http://aws-java-sample-java-function.default.dev.triggermesh.io -d '{"event
{"message":"Hello, the current time is Tue Apr 07 13:59:17 GMT 2020"}
```

### Run in Docker
### Run in Docker

For cases in which the use of additional components (tm CLI, Tekton, Knative, k8s) is undesirable, it is possible to build a KLR function as a standalone Docker container and run it in any environment. To do this, you should extract the Dockerfile from the runtime you are interested in, put it in the directory with your function, update the handler variable, and build the container. Here are Dockerfile definitions for all runtimes:

- [go](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/go-1.x/runtime.yaml#L44-L68)
- [java](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/java8/runtime.yaml#L43-L53)
- [node-10](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/node-10.x/runtime.yaml#L43-L48)
- [node-4](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/node-4.x/runtime.yaml#L43-L48)
- [python-2](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/python-2.7/runtime.yaml#L43-L50)
- [python-3](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/python-3.7/runtime.yaml#L43-L50)
- [ruby-2](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/ruby-2.5/runtime.yaml#L43-L47)

- [go](https://github.com/triggermesh/knative-lambda-runtime/blob/master/go/runtime.yaml#L44-L68)
- [java](https://github.com/triggermesh/knative-lambda-runtime/blob/master/java8/runtime.yaml#L43-L53)
- [node-10](https://github.com/triggermesh/knative-lambda-runtime/blob/master/node10/runtime.yaml#L43-L48)
- [node-4](https://github.com/triggermesh/knative-lambda-runtime/blob/master/node4/runtime.yaml#L43-L48)
- [python-2](https://github.com/triggermesh/knative-lambda-runtime/blob/master/python27/runtime.yaml#L43-L50)
- [python-3](https://github.com/triggermesh/knative-lambda-runtime/blob/master/python37/runtime.yaml#L43-L50)
- [ruby-2](https://github.com/triggermesh/knative-lambda-runtime/blob/master/ruby25/runtime.yaml#L43-L47)

Let's build a Python 3.7 function as an example:

Expand All @@ -294,7 +293,7 @@ def endpoint(event, context):
EOF
```

2. Extract the runtime's [Dockerfile](https://github.com/triggermesh/knative-lambda-runtime/blob/9a74ce1ac03d56d233cfc7a46d84f2c5e5f2685a/python-3.7/runtime.yaml#L43-L50), store it in the same directory, and update the `_HANDLER` variable:
2. Extract the runtime's [Dockerfile](https://github.com/triggermesh/knative-lambda-runtime/blob/master/python37/runtime.yaml#L43-L50), store it in the same directory, and update the `_HANDLER` variable:

```
cat > Dockerfile <<EOF
Expand All @@ -313,11 +312,11 @@ The `_HANDLER` variable in most cases consists of the filename without the file
```
docker build -t python-klr-image .
docker run -d --rm --name python-klr-container python-klr-image
# following command will work if you use Docker bridge network and you have jq tool
# otherwise, you should get the container address manually
# following command will work if you use Docker bridge network and you have jq tool
# otherwise, you should get the container address manually
curl $(docker inspect python-klr-container | jq .[].NetworkSettings.Networks.bridge.IPAddress -r):8080
```

The response will contain a JSON document with the current time.

4. Cleanup:
Expand Down
48 changes: 22 additions & 26 deletions cloudbuild.yaml
Original file line number Diff line number Diff line change
@@ -1,32 +1,28 @@
steps:
## Build python37 runtime
- name: 'gcr.io/cloud-builders/docker'
args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-python37:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-python37:latest', '-f' ,'./python-3.7/Dockerfile', '.' ]

## Build python27 runtime
- name: 'gcr.io/cloud-builders/docker'
args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-python27:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-python27:latest', '-f' ,'./python-2.7/Dockerfile', '.' ]
- name: gcr.io/kaniko-project/executor:latest
args:
- --dockerfile=Dockerfile
- --build-arg=VERSION=${_KANIKO_IMAGE_TAG}
- --destination=gcr.io/$PROJECT_ID/${_RUNTIME}:${COMMIT_SHA}
- --destination=gcr.io/$PROJECT_ID/${_RUNTIME}:${_KANIKO_IMAGE_TAG}
- --cache-repo=gcr.io/$PROJECT_ID/${_RUNTIME}/cache
- --cache=${_KANIKO_USE_BUILD_CACHE}
- --no-push=${_KANIKO_NO_PUSH}
- ${_KANIKO_EXTRA_ARGS}
waitFor: ['-']

## Build node4 runtime
- name: 'gcr.io/cloud-builders/docker'
args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-node4:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-node4:latest', '-f' ,'./node-4.x/Dockerfile', '.' ]
timeout: 600s

## Build node10 runtime
- name: 'gcr.io/cloud-builders/docker'
args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-node10:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-node10:latest', '-f' ,'./node-10.x/Dockerfile', '.' ]
substitutions:
_RUNTIME:
_KANIKO_IMAGE_TAG: latest
_KANIKO_NO_PUSH: 'false'
_KANIKO_USE_BUILD_CACHE: 'true'
_KANIKO_EXTRA_ARGS:

## Build ruby25 runtime
- name: 'gcr.io/cloud-builders/docker'
args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-ruby25:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-ruby25:latest', '-f' ,'./ruby-2.5/Dockerfile', '.' ]
options:
substitution_option: ALLOW_LOOSE

## Build java8 runtime
- name: 'gcr.io/cloud-builders/docker'
args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-java8:$REVISION_ID', '-t', 'gcr.io/$PROJECT_ID/knative-lambda-java8:latest', 'java8' ]

images:
- 'gcr.io/$PROJECT_ID/knative-lambda-python37'
- 'gcr.io/$PROJECT_ID/knative-lambda-python27'
- 'gcr.io/$PROJECT_ID/knative-lambda-node4'
- 'gcr.io/$PROJECT_ID/knative-lambda-node10'
- 'gcr.io/$PROJECT_ID/knative-lambda-ruby25'
- 'gcr.io/$PROJECT_ID/knative-lambda-java8'
tags:
- knative-lambda-runtime
File renamed without changes.
12 changes: 6 additions & 6 deletions go-1.x/runtime.yaml → go/runtime.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ spec:
- name: IMAGE
description: Where to store resulting image
- name: SSH_KEY
description: SSH key
description: SSH key
default: "placeholder"
- name: DIRECTORY
description: The subdirectory of the workspace/repo
Expand All @@ -42,11 +42,11 @@ spec:
cd /workspace/workspace/$(inputs.params.DIRECTORY)
cat <<EOF > Dockerfile
FROM golang:alpine
# Skip known public key check to be able to pull from private repositories
# Skip known public key check to be able to pull from private repositories
ENV GIT_SSH_COMMAND "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
RUN apk --no-cache add git ca-certificates openssh \
&& go get github.com/triggermesh/aws-custom-runtime \
&& go get github.com/triggermesh/knative-lambda-runtime/go-1.x \
&& go get github.com/triggermesh/knative-lambda-runtime/go \
&& go get github.com/golang/dep/...
WORKDIR /go/src/handler
COPY . .
Expand All @@ -63,7 +63,7 @@ spec:
ENV LAMBDA_TASK_ROOT "/opt"
ENV _HANDLER "handler"
COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=0 /go/bin/go-1.x /opt/bootstrap
COPY --from=0 /go/bin/go /opt/bootstrap
COPY --from=0 /go/bin/ /opt
ENTRYPOINT ["/opt/aws-custom-runtime"]
EOF
Expand All @@ -73,8 +73,8 @@ spec:
- --context=/workspace/workspace/$(inputs.params.DIRECTORY)
- --dockerfile=Dockerfile
- --destination=$(inputs.params.IMAGE)
# Workaround not to use default config which requires gcloud credentials
# to pull base image from public gcr registry
# Workaround not to use default config which requires gcloud credentials
# to pull base image from public gcr registry
# https://groups.google.com/d/msg/kaniko-users/r5yoP_Ejm_c/ExoEXksDBAAJ
env:
- name: DOCKER_CONFIG
Expand Down
5 changes: 5 additions & 0 deletions java8/.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Dockerfile
.dockerignore

**/*.md
**/.gitignore
17 changes: 11 additions & 6 deletions java8/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
FROM alpine:3 as downloader

RUN apk --no-cache add curl \
&& DOWNLOAD_URL=$(curl -sSf https://api.github.com/repos/triggermesh/aws-custom-runtime/releases/latest | grep "browser_download_url.*-linux-amd64" | cut -d: -f 2,3 | tr -d \") \
&& curl -sSfL ${DOWNLOAD_URL} -o /opt/aws-custom-runtime \
&& chmod +x /opt/aws-custom-runtime

FROM openjdk:8-jre-alpine

COPY runtime /var/runtime
RUN apk add --no-cache libc6-compat

RUN apk update && apk add --no-cache libc6-compat curl \
&& API_VERSION=$(curl -sI https://github.com/triggermesh/aws-custom-runtime/releases/latest | grep -i "Location:" | awk -F "/" '{print $NF}' | tr -d "\r") \
&& curl -sL https://github.com/triggermesh/aws-custom-runtime/releases/download/${API_VERSION}/aws-custom-runtime > /var/runtime/mockserver \
&& chmod +x /var/runtime/mockserver
COPY runtime /var/runtime
COPY --from=downloader /opt/aws-custom-runtime /var/runtime/mockserver

ENV LD_LIBRARY_PATH /lib64:/usr/lib64:/var/runtime:/var/runtime/lib:/var/task:/var/task/lib:/opt/lib
ENV LAMBDA_RUNTIME_DIR /var/runtime
Expand All @@ -23,4 +28,4 @@ ENTRYPOINT ["/usr/bin/java", \
"-Xshare:auto", \
"-XX:-TieredCompilation", \
"-jar", \
"/var/runtime/lib/LambdaJavaRTEntry-1.0.jar"]
"/var/runtime/lib/LambdaJavaRTEntry-1.0.jar"]
12 changes: 0 additions & 12 deletions node-10.x/Dockerfile

This file was deleted.

12 changes: 0 additions & 12 deletions node-4.x/Dockerfile

This file was deleted.

5 changes: 5 additions & 0 deletions node10/.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Dockerfile
.dockerignore

**/*.md
**/.gitignore
Loading

0 comments on commit bda5601

Please sign in to comment.