Git Init Container for cloning #69

* create init container that can be used to checkout git repositories
with https and ssh

Author: jfaltermeier

.vscode/extensions.json

@@ -1,5 +1,6 @@
 {
     "recommendations": [
-        "hashicorp.terraform"
+        "hashicorp.terraform",
+        "ms-python.python"
     ]
-}
+}

doc/docs/Building-Internal.md

@@ -61,3 +61,10 @@ Build and push the operator with:
 docker build --no-cache -t theiacloud/theia-cloud-operator:latest -f dockerfiles/operator/Dockerfile .
 docker push theiacloud/theia-cloud-operator:latest
 ```
+
+Build and pish the git-init container:
+
+```bash
+docker build -t theiacloud/theia-cloud-git-init:latest -f dockerfiles/git-init/Dockerfile .
+docker push theiacloud/theia-cloud-git-init:latest
+```

dockerfiles/git-init/Dockerfile

@@ -0,0 +1,16 @@
+FROM debian:11-slim
+
+RUN apt update && \
+    apt install python git -y && \
+    apt clean
+
+WORKDIR /tmp
+COPY python/git-init/entrypoint.sh .
+COPY python/git-init/ssh-keyscan.sh .
+COPY python/git-init/git-init.py .
+COPY python/git-init/git-askpw.py .
+
+ENV GIT_ASKPASS=/tmp/git-askpw.py
+
+ENTRYPOINT [ "/tmp/entrypoint.sh" ]
+CMD ["-h"]

python/git-init/README.md

@@ -0,0 +1,56 @@
+# Git Init Container
+
+## Scenarios
+
+- HTTP(S)
+  - No Auth
+  - Ask for password only
+  - Ask for username and password
+- SSH
+  - No Auth
+  - Ask for password
+
+## Testing
+
+### Build init container
+
+```bash
+docker build -t theiacloud/theia-cloud-git-init:local -f dockerfiles/git-init/Dockerfile .
+```
+
+### Generate Test SSH Key Pair
+
+```bash
+# don't save in ~/.ssh/... but e.g. in ~/tmp/ssh/id_theiacloud
+ssh-keygen -t ed25519 -C "Test TC Git Init SSH Keypair"
+```
+
+### Test Checkout
+
+```bash
+# Adjust URLs and Password/PATs below
+# keep spaces in front to avoid command being added to bash history
+ export HTTP_PUBLIC=https://github.com/eclipsesource/theia-cloud.git
+ export HTTP_PRIVATE=https://gitlab.eclipse.org/username/my.repository.git
+ export HTTP_PRIVATE_WITH_USERNAME=https://[email protected]/username/my.repository.git
+ export HTTP_PRIVATE_WITH_USERNAME_AND_PASSWORD=https://username:[email protected]/username/my.repository.git
+ export HTTP_USERNAME=username
+ export HTTP_PASSWORD=pat
+ export SSH_PASSWORD=sshpw
+ export SSH_REPO="[email protected]:username/my.repository.git"
+
+# HTTPS Public
+docker run --rm theiacloud/theia-cloud-git-init:local "$HTTP_PUBLIC" "/tmp/my-repo"
+
+# HTTPS Private
+docker run --env GIT_PROMPT1=$HTTP_USERNAME --env GIT_PROMPT2=$HTTP_PASSWORD --rm theiacloud/theia-cloud-git-init:local "$HTTP_PRIVATE" "/tmp/my-repo"
+
+# HTTPS Private with Username
+docker run --env GIT_PROMPT1=$HTTP_PASSWORD --rm theiacloud/theia-cloud-git-init:local "$HTTP_PRIVATE_WITH_USERNAME" "/tmp/my-repo"
+
+# HTTPS Private with Username and Password
+docker run --rm theiacloud/theia-cloud-git-init:local "$HTTP_PRIVATE_WITH_USERNAME_AND_PASSWORD" "/tmp/my-repo"
+
+# SSH
+docker run --env GIT_PROMPT1=$SSH_PASSWORD -v ~/tmp/ssh/:/etc/theia-cloud-ssh --rm theiacloud/theia-cloud-git-init:local "$SSH_REPO" "/tmp/my-repo"
+```

python/git-init/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+eval `ssh-agent`
+mkdir $HOME/.ssh
+touch $HOME/.ssh/known_hosts
+[ -e /etc/theia-cloud-ssh/id_theiacloud ] && { sleep 1; echo $GIT_PROMPT1; } | script -q /dev/null -c 'ssh-add /etc/theia-cloud-ssh/id_theiacloud'
+python git-init.py "$@"

python/git-init/git-askpw.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+import os
+
+path = "/tmp/theia-cloud-askpw"
+
+if os.path.isfile(path):
+    prompt2 = os.environ['GIT_PROMPT2']
+    print(prompt2)
+else:
+    prompt1 = os.environ['GIT_PROMPT1']
+    print(prompt1)
+    os.mknod(path)

python/git-init/git-init.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+
+import argparse
+import subprocess
+import os
+import sys
+
+debugLogging = False
+sshKey = "/etc/theia-cloud-ssh/id_theiacloud"
+NL = "\n"
+
+
+def runProcess(args):
+    process = subprocess.Popen(
+        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    process.wait()
+    out = stdout.decode('ascii')
+    if len(out) > 0:
+        sys.stdout.write(out + NL)
+    if process.returncode != 0:
+        sys.stderr.write(stderr.decode('ascii') + NL)
+    return process.returncode
+
+def getHostname(repository):
+    # remove protocol, if any
+    split = repository.split("://", 1)
+    if len(split) == 1:
+        repository = split[0]
+    else:
+        repository = split[1]
+    if debugLogging:
+        sys.stdout.write("getHostname 1: " + repository + NL)
+
+    # remove path, if any
+    split = repository.split("/", 1)
+    repository = split[0]
+    if debugLogging:
+        sys.stdout.write("getHostname 2: " + repository + NL)
+
+    # remove user information, if any
+    split = repository.split("@", 1)
+    if len(split) == 1:
+        repository = split[0]
+    else:
+        repository = split[1]
+    if debugLogging:
+        sys.stdout.write("getHostname 3: " + repository + NL)
+
+    # remove trailing information, if any
+    split = repository.split(":", 1)
+    repository = split[0]
+    if debugLogging:
+        sys.stdout.write("getHostname 4: " + repository + NL)
+
+    return repository
+
+parser = argparse.ArgumentParser()
+parser.add_argument("repository", help="The repository URL", type=str)
+parser.add_argument("directory", help="The directory to clone into", type=str)
+args = parser.parse_args()
+
+# Set up git credential helper
+code = runProcess(["git", "config", "--global", "credential.helper", "store"])
+if code != 0:
+    exit(code)
+
+# Check if SSH key is available, if so prepare clone with SSH
+if os.path.isfile(sshKey):
+    # Add know host
+    code = runProcess(["/tmp/ssh-keyscan.sh", getHostname(args.repository)])
+    if code != 0:
+        exit(code)
+
+    if debugLogging:
+        runProcess(["ssh-add", "-l"])
+        runProcess(["cat", "/root/.ssh/known_hosts"])
+
+# Clone repository
+code = runProcess(["git", "clone", args.repository, args.directory])
+if code != 0:
+    exit(code)
+
+if debugLogging:
+    runProcess(["ls", "-al", args.directory])

python/git-init/ssh-keyscan.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+ssh-keyscan -H $@ >> /root/.ssh/known_hosts