feat(examples): Add Apache Spark

apache-spark/all-in-one/Dockerfile

@@ -0,0 +1,65 @@
+FROM spark:3.5.2-scala2.12-java17-ubuntu AS build
+
+RUN mkdir /tmp/work
+RUN mkdir -p /tmp/work/etc
+RUN echo "apache-spark" > /tmp/work/etc/hostname
+RUN echo "172.16.3.1 apache-spark.kraft.host apache-spark" > /tmp/work/etc/hosts
+RUN echo "127.0.0.1 localhost.localdomain localhost" >> /tmp/work/etc/hosts
+RUN mkdir -p /tmp/work/tmp
+RUN mkdir -p /tmp/work/opt/spark/logs
+
+FROM scratch
+
+# Copy Spark and Java.
+COPY --from=build /opt /opt
+
+# Copy core libaries.
+COPY --from=build /opt/java/openjdk/lib/libjli.so /opt/java/openjdk/lib/libjli.so
+COPY --from=build /lib/x86_64-linux-gnu/libpthread.so.0 /lib/x86_64-linux-gnu/libpthread.so.0
+COPY --from=build /lib/x86_64-linux-gnu/libdl.so.2 /lib/x86_64-linux-gnu/libdl.so.2
+COPY --from=build /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu/libc.so.6
+COPY --from=build /lib/x86_64-linux-gnu/libsystemd.so.0 /lib/x86_64-linux-gnu/libsystemd.so.0
+COPY --from=build /lib/x86_64-linux-gnu/liblzma.so.5 /lib/x86_64-linux-gnu/liblzma.so.5
+COPY --from=build /lib/x86_64-linux-gnu/libzstd.so.1 /lib/x86_64-linux-gnu/libzstd.so.1
+COPY --from=build /lib/x86_64-linux-gnu/liblz4.so.1 /lib/x86_64-linux-gnu/liblz4.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libcap.so.2 /lib/x86_64-linux-gnu/libcap.so.2
+COPY --from=build /lib/x86_64-linux-gnu/libgcrypt.so.20 /lib/x86_64-linux-gnu/libgcrypt.so.20
+COPY --from=build /lib/x86_64-linux-gnu/libgpg-error.so.0 /lib/x86_64-linux-gnu/libgpg-error.so.0
+COPY --from=build /lib/x86_64-linux-gnu/librt.so.1 /lib/x86_64-linux-gnu/librt.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libm.so.6 /lib/x86_64-linux-gnu/libm.so.6
+COPY --from=build /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2
+
+# Copy Bash and rquired tools.
+COPY --from=build /bin/bash /bin/bash
+COPY --from=build /bin/sh /bin/sh
+COPY --from=build /usr/bin/env /usr/bin/env
+COPY --from=build /usr/bin/dirname /usr/bin/dirname
+COPY --from=build /usr/bin/grep /usr/bin/grep
+COPY --from=build /bin/ps /bin/ps
+COPY --from=build /usr/bin/sleep /usr/bin/sleep
+COPY --from=build /usr/bin/mkdir /usr/bin/mkdir
+COPY --from=build /usr/bin/chown /usr/bin/chown
+COPY --from=build /usr/bin/tail /usr/bin/tail
+COPY --from=build /usr/bin/sed /usr/bin/sed
+COPY --from=build /usr/bin/hostname /usr/bin/hostname
+COPY --from=build /usr/bin/uname /usr/bin/uname
+COPY --from=build /usr/bin/nohup /usr/bin/nohup
+COPY --from=build /usr/bin/nice /usr/bin/nice
+COPY --from=build /usr/bin/head /usr/bin/head
+COPY --from=build /usr/bin/rm /usr/bin/rm
+
+# Copy libraries required for tools.
+COPY --from=build /lib/x86_64-linux-gnu/libpcre.so.3 /lib/x86_64-linux-gnu/libpcre.so.3
+COPY --from=build /lib/x86_64-linux-gnu/libpcre2-8.so.0 /lib/x86_64-linux-gnu/libpcre2-8.so.0
+COPY --from=build /lib/x86_64-linux-gnu/libtinfo.so.6 /lib/x86_64-linux-gnu/libtinfo.so.6
+COPY --from=build /lib/x86_64-linux-gnu/libprocps.so.8 /lib/x86_64-linux-gnu/libprocps.so.8
+COPY --from=build /lib/x86_64-linux-gnu/libselinux.so.1 /lib/x86_64-linux-gnu/libselinux.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libacl.so.1 /lib/x86_64-linux-gnu/libacl.so.1
+
+# Copy user database.
+COPY --from=build /etc/passwd /etc/passwd
+
+# Create runtime directories.
+COPY --from=build /tmp/work/ /
+
+COPY ./wrapper.sh /usr/bin/wrapper.sh

apache-spark/all-in-one/Kraftfile

@@ -0,0 +1,12 @@
+spec: v0.6
+
+runtime: base-compat:latest
+
+labels:
+  cloud.unikraft.v1.instances/scale_to_zero.policy: "idle"
+  cloud.unikraft.v1.instances/scale_to_zero.stateful: "true"
+  cloud.unikraft.v1.instances/scale_to_zero.cooldown_time_ms: 3000
+
+rootfs: ./Dockerfile
+
+cmd: ["/usr/bin/wrapper.sh"]

apache-spark/all-in-one/deploy.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+kraft cloud deploy --scale-to-zero off -M 4096 -p 443:8080/http+tls -p 7077:7077/tls --name apache-spark --subdomain apache-spark .

apache-spark/all-in-one/kill-socat.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sudo pkill -f socat

apache-spark/all-in-one/launch.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+kraft cloud inst create --scale-to-zero off --start -M 4096 -p 443:8080/http+tls -p 444:8081/tls -p 7077:7077/tls --name apache-spark --subdomain apache-spark razvan.unikraft.io/apache-spark

apache-spark/all-in-one/listen-socat.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+socat tcp-listen:7077,bind=127.0.0.1,fork,reuseaddr openssl:apache-spark.fra0.kraft.host:7077 &

apache-spark/all-in-one/logs.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+kraft cloud inst logs apache-spark

apache-spark/all-in-one/query.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+curl -X POST https://spark.fra0.kraft.host/v1/submissions/create \
+--header "Content-Type:application/json;charset=UTF-8" \
+--data '{
+  "appResource": "",
+  "sparkProperties": {
+    "spark.master": "spark://master:7077",
+    "spark.app.name": "Spark Pi",
+    "spark.driver.memory": "1g",
+    "spark.driver.cores": "1",
+    "spark.jars": ""
+  },
+  "clientSparkVersion": "",
+  "mainClass": "org.apache.spark.deploy.SparkSubmit",
+  "environmentVariables": { },
+  "action": "CreateSubmissionRequest",
+  "appArgs": [ "/opt/spark/examples/src/main/python/pi.py", "10" ]
+}'

apache-spark/all-in-one/rm.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+kraft cloud inst rm apache-spark

apache-spark/all-in-one/wrapper.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+export JAVA_HOME=/opt/java/openjdk/
+hostname apache-spark
+export SPARK_NO_DAEMONIZE=1
+/opt/spark/sbin/start-master.sh &
+/opt/spark/sbin/start-worker.sh spark://172.16.3.1:7077

apache-spark/driver/Dockerfile

@@ -0,0 +1,66 @@
+FROM spark:3.5.2-scala2.12-java17-ubuntu AS build
+
+RUN mkdir /tmp/work
+RUN mkdir -p /tmp/work/etc
+RUN echo "apache-spark-driver" > /tmp/work/etc/hostname
+RUN echo "172.16.3.1 apache-spark.kraft.host apache-spark" > /tmp/work/etc/hosts
+RUN echo "127.0.0.1 localhost.localdomain localhost" >> /tmp/work/etc/hosts
+RUN echo "127.0.0.1 apache-spark-driver.kraft.host apache-spark-driver" >> /tmp/work/etc/hosts
+RUN mkdir -p /tmp/work/tmp
+RUN mkdir -p /tmp/work/opt/spark/logs
+
+FROM scratch
+
+# Copy Spark and Java.
+COPY --from=build /opt /opt
+
+# Copy core libaries.
+COPY --from=build /opt/java/openjdk/lib/libjli.so /opt/java/openjdk/lib/libjli.so
+COPY --from=build /lib/x86_64-linux-gnu/libpthread.so.0 /lib/x86_64-linux-gnu/libpthread.so.0
+COPY --from=build /lib/x86_64-linux-gnu/libdl.so.2 /lib/x86_64-linux-gnu/libdl.so.2
+COPY --from=build /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu/libc.so.6
+COPY --from=build /lib/x86_64-linux-gnu/libsystemd.so.0 /lib/x86_64-linux-gnu/libsystemd.so.0
+COPY --from=build /lib/x86_64-linux-gnu/liblzma.so.5 /lib/x86_64-linux-gnu/liblzma.so.5
+COPY --from=build /lib/x86_64-linux-gnu/libzstd.so.1 /lib/x86_64-linux-gnu/libzstd.so.1
+COPY --from=build /lib/x86_64-linux-gnu/liblz4.so.1 /lib/x86_64-linux-gnu/liblz4.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libcap.so.2 /lib/x86_64-linux-gnu/libcap.so.2
+COPY --from=build /lib/x86_64-linux-gnu/libgcrypt.so.20 /lib/x86_64-linux-gnu/libgcrypt.so.20
+COPY --from=build /lib/x86_64-linux-gnu/libgpg-error.so.0 /lib/x86_64-linux-gnu/libgpg-error.so.0
+COPY --from=build /lib/x86_64-linux-gnu/librt.so.1 /lib/x86_64-linux-gnu/librt.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libm.so.6 /lib/x86_64-linux-gnu/libm.so.6
+COPY --from=build /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2
+
+# Copy Bash and rquired tools.
+COPY --from=build /bin/bash /bin/bash
+COPY --from=build /bin/sh /bin/sh
+COPY --from=build /usr/bin/env /usr/bin/env
+COPY --from=build /usr/bin/dirname /usr/bin/dirname
+COPY --from=build /usr/bin/grep /usr/bin/grep
+COPY --from=build /bin/ps /bin/ps
+COPY --from=build /usr/bin/sleep /usr/bin/sleep
+COPY --from=build /usr/bin/mkdir /usr/bin/mkdir
+COPY --from=build /usr/bin/chown /usr/bin/chown
+COPY --from=build /usr/bin/tail /usr/bin/tail
+COPY --from=build /usr/bin/sed /usr/bin/sed
+COPY --from=build /usr/bin/hostname /usr/bin/hostname
+COPY --from=build /usr/bin/uname /usr/bin/uname
+COPY --from=build /usr/bin/nohup /usr/bin/nohup
+COPY --from=build /usr/bin/nice /usr/bin/nice
+COPY --from=build /usr/bin/head /usr/bin/head
+COPY --from=build /usr/bin/rm /usr/bin/rm
+
+# Copy libraries required for tools.
+COPY --from=build /lib/x86_64-linux-gnu/libpcre.so.3 /lib/x86_64-linux-gnu/libpcre.so.3
+COPY --from=build /lib/x86_64-linux-gnu/libpcre2-8.so.0 /lib/x86_64-linux-gnu/libpcre2-8.so.0
+COPY --from=build /lib/x86_64-linux-gnu/libtinfo.so.6 /lib/x86_64-linux-gnu/libtinfo.so.6
+COPY --from=build /lib/x86_64-linux-gnu/libprocps.so.8 /lib/x86_64-linux-gnu/libprocps.so.8
+COPY --from=build /lib/x86_64-linux-gnu/libselinux.so.1 /lib/x86_64-linux-gnu/libselinux.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libacl.so.1 /lib/x86_64-linux-gnu/libacl.so.1
+
+# Copy user database.
+COPY --from=build /etc/passwd /etc/passwd
+
+# Create runtime directories.
+COPY --from=build /tmp/work/ /
+
+COPY ./wrapper.sh /usr/bin/wrapper.sh

apache-spark/driver/Kraftfile

@@ -0,0 +1,12 @@
+spec: v0.6
+
+runtime: base-compat:latest
+
+labels:
+  cloud.unikraft.v1.instances/scale_to_zero.policy: "off"
+  cloud.unikraft.v1.instances/scale_to_zero.stateful: "false"
+  cloud.unikraft.v1.instances/scale_to_zero.cooldown_time_ms: 3000
+
+rootfs: ./Dockerfile
+
+cmd: ["/usr/bin/wrapper.sh"]

apache-spark/driver/deploy.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+kraft cloud deploy --scale-to-zero off -M 4096 -p 443--scale-to-zero off :8080/http+tls -p 7077:7077/tls --name apache-spark-driver --subdomain apache-spark-driver .

apache-spark/driver/launch.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+kraft cloud instance create --scale-to-zero off --start -M 4096 --name apache-spark-driver --subdomain apache-spark-driver razvan.unikraft.io/apache-spark-driver

apache-spark/driver/logs.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+kraft cloud inst logs apache-spark-driver

apache-spark/driver/rm.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+kraft cloud inst rm apache-spark-driver

apache-spark/driver/wrapper.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+export JAVA_HOME=/opt/java/openjdk/
+hostname apache-spark-driver
+export MASTER=spark://apache-spark.kraft.host:7077
+/opt/spark/bin/run-example --conf spark.scheduler.minRegisteredResourcesRatio=0 SparkPi

apache-spark/master/Dockerfile

@@ -0,0 +1,63 @@
+FROM spark:3.5.2-scala2.12-java17-ubuntu AS build
+
+RUN mkdir /tmp/work
+RUN mkdir -p /tmp/work/etc
+RUN echo "apache-spark-master" > /tmp/work/etc/hostname
+RUN echo "172.16.3.2 apache-spark-master.internal apache-spark-master" > /tmp/work/etc/hosts
+RUN mkdir -p /tmp/work/tmp
+RUN mkdir -p /tmp/work/opt/spark/logs
+
+FROM scratch
+
+# Copy Spark and Java.
+COPY --from=build /opt /opt
+
+# Copy core libaries.
+COPY --from=build /opt/java/openjdk/lib/libjli.so /opt/java/openjdk/lib/libjli.so
+COPY --from=build /lib/x86_64-linux-gnu/libpthread.so.0 /lib/x86_64-linux-gnu/libpthread.so.0
+COPY --from=build /lib/x86_64-linux-gnu/libdl.so.2 /lib/x86_64-linux-gnu/libdl.so.2
+COPY --from=build /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu/libc.so.6
+COPY --from=build /lib/x86_64-linux-gnu/libsystemd.so.0 /lib/x86_64-linux-gnu/libsystemd.so.0
+COPY --from=build /lib/x86_64-linux-gnu/liblzma.so.5 /lib/x86_64-linux-gnu/liblzma.so.5
+COPY --from=build /lib/x86_64-linux-gnu/libzstd.so.1 /lib/x86_64-linux-gnu/libzstd.so.1
+COPY --from=build /lib/x86_64-linux-gnu/liblz4.so.1 /lib/x86_64-linux-gnu/liblz4.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libcap.so.2 /lib/x86_64-linux-gnu/libcap.so.2
+COPY --from=build /lib/x86_64-linux-gnu/libgcrypt.so.20 /lib/x86_64-linux-gnu/libgcrypt.so.20
+COPY --from=build /lib/x86_64-linux-gnu/libgpg-error.so.0 /lib/x86_64-linux-gnu/libgpg-error.so.0
+COPY --from=build /lib/x86_64-linux-gnu/librt.so.1 /lib/x86_64-linux-gnu/librt.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libm.so.6 /lib/x86_64-linux-gnu/libm.so.6
+COPY --from=build /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2
+
+# Copy Bash and rquired tools.
+COPY --from=build /bin/bash /bin/bash
+COPY --from=build /bin/sh /bin/sh
+COPY --from=build /usr/bin/env /usr/bin/env
+COPY --from=build /usr/bin/dirname /usr/bin/dirname
+COPY --from=build /usr/bin/grep /usr/bin/grep
+COPY --from=build /bin/ps /bin/ps
+COPY --from=build /usr/bin/sleep /usr/bin/sleep
+COPY --from=build /usr/bin/mkdir /usr/bin/mkdir
+COPY --from=build /usr/bin/chown /usr/bin/chown
+COPY --from=build /usr/bin/tail /usr/bin/tail
+COPY --from=build /usr/bin/sed /usr/bin/sed
+COPY --from=build /usr/bin/hostname /usr/bin/hostname
+COPY --from=build /usr/bin/uname /usr/bin/uname
+COPY --from=build /usr/bin/nohup /usr/bin/nohup
+COPY --from=build /usr/bin/nice /usr/bin/nice
+COPY --from=build /usr/bin/head /usr/bin/head
+
+# Copy libraries required for tools.
+COPY --from=build /lib/x86_64-linux-gnu/libpcre.so.3 /lib/x86_64-linux-gnu/libpcre.so.3
+COPY --from=build /lib/x86_64-linux-gnu/libpcre2-8.so.0 /lib/x86_64-linux-gnu/libpcre2-8.so.0
+COPY --from=build /lib/x86_64-linux-gnu/libtinfo.so.6 /lib/x86_64-linux-gnu/libtinfo.so.6
+COPY --from=build /lib/x86_64-linux-gnu/libprocps.so.8 /lib/x86_64-linux-gnu/libprocps.so.8
+COPY --from=build /lib/x86_64-linux-gnu/libselinux.so.1 /lib/x86_64-linux-gnu/libselinux.so.1
+COPY --from=build /lib/x86_64-linux-gnu/libacl.so.1 /lib/x86_64-linux-gnu/libacl.so.1
+
+# Copy user database.
+COPY --from=build /etc/passwd /etc/passwd
+
+# Create runtime directories.
+COPY --from=build /tmp/work/ /
+
+COPY ./wrapper.sh /usr/bin/wrapper.sh

apache-spark/master/Kraftfile

@@ -0,0 +1,12 @@
+spec: v0.6
+
+runtime: base-compat:latest
+
+labels:
+  cloud.unikraft.v1.instances/scale_to_zero.policy: "idle"
+  cloud.unikraft.v1.instances/scale_to_zero.stateful: "true"
+  cloud.unikraft.v1.instances/scale_to_zero.cooldown_time_ms: 1000
+
+rootfs: ./Dockerfile
+
+cmd: ["/usr/bin/wrapper.sh", "/opt/spark/sbin/start-master.sh"]

apache-spark/master/Makefile

@@ -0,0 +1,42 @@
+IMAGE_NAME ?= unikraft-spark-master
+CONTAINER_NAME ?= $(IMAGE_NAME)
+CMD ?= /bin/bash
+
+build:
+	docker build -f Dockerfile -t $(IMAGE_NAME) .
+
+run_anon: build
+	docker run --rm --interactive --tty $(IMAGE_NAME) $(CMD)
+
+create: build
+	-docker container inspect $(CONTAINER_NAME) > /dev/null 2>&1 || docker create --name $(CONTAINER_NAME) --tty $(IMAGE_NAME) $(CMD)
+
+run: create
+	-test "$(shell docker container inspect -f '{{{{.State.Running}}}}' $(CONTAINER_NAME) 2> /dev/null)" = "false" || docker exec --interactive --tty $(CONTAINER_NAME) python3 && docker start --interactive $(CONTAINER_NAME)
+
+stop:
+	-test "$(shell docker container inspect -f '{{{{.State.Running}}}}' $(CONTAINER_NAME) 2> /dev/null)" = "true" && docker stop $(CONTAINER_NAME)
+
+clean: stop
+	-docker container inspect $(CONTAINER_NAME) > /dev/null 2>&1 && docker rm $(CONTAINER_NAME)
+
+export: cleanfs
+	mkdir rootfs
+	docker build -o rootfs -f Dockerfile .
+
+rootfs: export
+
+rootfs.cpio: rootfs
+	cd rootfs/ && find -depth -print | tac | bsdcpio -o --format newc > ../rootfs.cpio
+
+initrd: rootfs.cpio
+
+cleanfs:
+	-test -d rootfs && sudo umount rootfs/dev ; sudo umount rootfs/proc ; sudo rm -fr rootfs
+
+run_chroot:
+	-mkdir rootfs/proc; sudo mount -t proc proc rootfs/proc
+	-mkdir rootfs/dev; sudo mount -t devtmpfs dev rootfs/dev
+	sudo chroot rootfs $(CMD)
+
+.PHONY: build build run_anon create run stop clean export initrd cleanfs

apache-spark/master/wrapper.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+export JAVA_HOME=/opt/java/openjdk/
+export SPARK_NO_DAEMONIZE=1
+hostname apache-spark-master
+#ln -sfn /proc/self/fd /dev/fd
+exec "$@"