diff --git a/.github/workflows/reusable-multi-node.yaml b/.github/workflows/reusable-multi-node.yaml index 4d6560a..dd33804 100644 --- a/.github/workflows/reusable-multi-node.yaml +++ b/.github/workflows/reusable-multi-node.yaml @@ -41,10 +41,10 @@ jobs: env: LIMA_TEMPLATE: "${{ matrix.lima_template }}" CONTAINER_ENGINE: "${{ matrix.container_engine }}" - U7S_PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}" - U7S_PORT_FLANNEL: "${{ inputs.flannel_port }}" - U7S_PORT_KUBELET: "${{ inputs.kubelet_port }}" - U7S_PORT_ETCD: "${{ inputs.etcd_port }}" + PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}" + PORT_FLANNEL: "${{ inputs.flannel_port }}" + PORT_KUBELET: "${{ inputs.kubelet_port }}" + PORT_ETCD: "${{ inputs.etcd_port }}" KUBECONFIG: ./kubeconfig steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reusable-single-node.yaml b/.github/workflows/reusable-single-node.yaml index 115896d..28b7001 100644 --- a/.github/workflows/reusable-single-node.yaml +++ b/.github/workflows/reusable-single-node.yaml @@ -35,10 +35,10 @@ jobs: timeout-minutes: 40 env: CONTAINER_ENGINE: "${{ inputs.container_engine }}" - U7S_PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}" - U7S_PORT_FLANNEL: "${{ inputs.flannel_port }}" - U7S_PORT_KUBELET: "${{ inputs.kubelet_port }}" - U7S_PORT_ETCD: "${{ inputs.etcd_port }}" + PORT_KUBE_APISERVER: "${{ inputs.kube_apiserver_port }}" + PORT_FLANNEL: "${{ inputs.flannel_port }}" + PORT_KUBELET: "${{ inputs.kubelet_port }}" + PORT_ETCD: "${{ inputs.etcd_port }}" KUBECONFIG: ./kubeconfig steps: - uses: actions/checkout@v4 diff --git a/Makefile b/Makefile index 52d0de5..4a55ca8 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,15 @@ HOSTNAME ?= $(shell hostname) export HOSTNAME := $(HOSTNAME) # Change ports for different kubernetes services -export U7S_PORT_ETCD ?= 2379 -export U7S_PORT_KUBELET ?= 10250 -export U7S_PORT_FLANNEL ?= 8472 -export U7S_PORT_KUBE_APISERVER ?= 6443 +PORT_ETCD ?= 2379 +PORT_KUBELET ?= 10250 +PORT_FLANNEL ?= 8472 +PORT_KUBE_APISERVER ?= 6443 + +export U7S_PORT_ETCD := $(PORT_ETCD) +export U7S_PORT_KUBELET := $(PORT_KUBELET) +export U7S_PORT_FLANNEL := $(PORT_FLANNEL) +export U7S_PORT_KUBE_APISERVER := $(PORT_KUBE_APISERVER) HOST_IP ?= $(shell ip --json route get 1 | jq -r .[0].prefsrc) NODE_NAME ?= u7s-$(HOSTNAME) diff --git a/README.md b/README.md index 93f89bd..7148527 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,21 @@ kubectl taint nodes --all node-role.kubernetes.io/control-plane- The container engine defaults to Docker. To change the container engine, set `export CONTAINER_ENGINE=podman` or `export CONTAINER_ENGINE=nerdctl`. +### Customization + +The following environment variables are recognized: + +Name | Type | Default value +----------------------|---------|---------------------------------------------------------------- +`CONTAINER_ENGINE` | String | automatically resolved to "docker", "podman", or "nerdctl" +`HOST_IP` | String | automatically resolved to the host's IP address +`NODE_NAME` | String | "u7s-" + the host's hostname +`NODE_SUBNET` | String | "10.100.%d.0/24" (%d is computed from the hash of the hostname) +`PORT_ETCD` | Integer | 2379 +`PORT_KUBELET` | Integer | 10250 +`PORT_FLANNEL` | Integer | 8472 +`PORT_KUBE_APISERVER` | Integer | 6443 + ## Limitations - Node ports cannot be exposed automatically. Edit [`docker-compose.yaml`](./docker-compose.yaml) for exposing additional node ports. - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files. @@ -153,6 +168,23 @@ make up > > The support for bypass4netns is still experimental -### Misc +### Multi-tenancy + +Multiple users on the hosts may create their own instances of Usernetes, but the port numbers have to be changed to avoid conflicts. + +```bash +# Default: 2379 +export PORT_ETCD=12379 +# Default: 10250 +export PORT_KUBELET=20250 +# Default: 8472 +export PORT_FLANNEL=18472 +# Default: 6443 +export PORT_KUBE_APISERVER=16443 + +make up +``` + +### Rootful mode - Although Usernetes (Gen2) is designed to be used with Rootless Docker, it should work with the regular "rootful" Docker too. This might be useful for some people who are looking for "multi-host" version of [`kind`](https://kind.sigs.k8s.io/) and [minikube](https://minikube.sigs.k8s.io/). diff --git a/hack/create-cluster-lima.sh b/hack/create-cluster-lima.sh index cd8d6ed..ba3d230 100755 --- a/hack/create-cluster-lima.sh +++ b/hack/create-cluster-lima.sh @@ -6,10 +6,10 @@ set -eux -o pipefail : "${LIMA_TEMPLATE:=template://default}" : "${CONTAINER_ENGINE:=docker}" : "${LOCKDOWN_SUDO:=1}" -: "${U7S_PORT_KUBE_APISERVER:=6443}" -: "${U7S_PORT_ETCD:=2379}" -: "${U7S_PORT_FLANNEL:=8472}" -: "${U7S_PORT_KUBELET:=10250}" +: "${PORT_KUBE_APISERVER:=6443}" +: "${PORT_ETCD:=2379}" +: "${PORT_FLANNEL:=8472}" +: "${PORT_KUBELET:=10250}" guest_home="/home/${USER}.linux" @@ -35,23 +35,23 @@ for host in host0 host1; do ${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.rootless.sh" done -U7S_SERVICE_PORTS="U7S_PORT_KUBE_APISERVER=${U7S_PORT_KUBE_APISERVER} U7S_PORT_ETCD=${U7S_PORT_ETCD} U7S_PORT_FLANNEL=${U7S_PORT_FLANNEL} U7S_PORT_KUBELET=${U7S_PORT_KUBELET}" +SERVICE_PORTS="PORT_KUBE_APISERVER=${PORT_KUBE_APISERVER} PORT_ETCD=${PORT_ETCD} PORT_FLANNEL=${PORT_FLANNEL} PORT_KUBELET=${PORT_KUBELET}" # Launch a Kubernetes node inside a Rootless Docker host for host in host0 host1; do - ${LIMACTL} shell "${host}" ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" up + ${LIMACTL} shell "${host}" ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" up done # Bootstrap a cluster with host0 -${LIMACTL} shell host0 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command +${LIMACTL} shell host0 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command # Let host1 join the cluster ${LIMACTL} copy host0:~/usernetes/join-command host1:~/usernetes/join-command -${LIMACTL} shell host1 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join -${LIMACTL} shell host0 ${U7S_SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip +${LIMACTL} shell host1 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join +${LIMACTL} shell host0 ${SERVICE_PORTS} CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip # Enable kubectl -ssh -q -f -N -L ${U7S_PORT_KUBE_APISERVER}:127.0.0.1:${U7S_PORT_KUBE_APISERVER} -F ~/.lima/host0/ssh.config lima-host0 +ssh -q -f -N -L ${PORT_KUBE_APISERVER}:127.0.0.1:${PORT_KUBE_APISERVER} -F ~/.lima/host0/ssh.config lima-host0 ${LIMACTL} copy host0:${guest_home}/usernetes/kubeconfig ./kubeconfig KUBECONFIG="$(pwd)/kubeconfig" export KUBECONFIG