Skip to content

Manual feature backport of cert rotation for whereabouts #2812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: release-4.15
Choose a base branch
from
Open

Manual feature backport of cert rotation for whereabouts #2812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
276 changes: 256 additions & 20 deletions bindata/network/multus/multus.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,23 @@ data:
fi
rm -Rf $UPGRADE_DIRECTORY
---
apiVersion: v1
kind: ConfigMap
metadata:
name: whereabouts-flatfile-config
namespace: openshift-multus
data:
whereabouts.conf: |
{
"datastore": "kubernetes",
"kubernetes": {
"kubeconfig": "/etc/kubernetes/cni/net.d/whereabouts.d/whereabouts.kubeconfig"
},
"reconciler_cron_expression": "30 4 * * *",
"log_level": "verbose",
"configuration_path": "/etc/kubernetes/cni/net.d/whereabouts.d"
}
---
kind: ConfigMap
apiVersion: v1
metadata:
Expand Down Expand Up @@ -157,7 +174,6 @@ data:
"bootstrapKubeconfig": "{{ .KubeletKubeconfigPath }}",
"certDir": "/etc/cni/multus/certs",
"certDuration": "24h"
},
{{ end }}
"cniConfigDir": "/host/etc/cni/net.d",
"multusConfigFile": "auto",
Expand Down Expand Up @@ -325,7 +341,7 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
terminationGracePeriodSeconds: 10
terminationGracePeriodSeconds: 30
volumes:
- name: system-cni-dir
hostPath:
Expand Down Expand Up @@ -540,21 +556,43 @@ spec:

CNI_BIN_DIR=${CNI_BIN_DIR:-"/host/opt/cni/bin/"}
WHEREABOUTS_KUBECONFIG_FILE_HOST=${WHEREABOUTS_KUBECONFIG_FILE_HOST:-"/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig"}
CNI_CONF_DIR=${CNI_CONF_DIR:-"/host/etc/cni/net.d"}
CNI_CONF_DIR=${CNI_CONF_DIR:-"/host{{ .SystemCNIConfDir }}"}
WHEREABOUTS_RECONCILER_CRON=${WHEREABOUTS_RECONCILER_CRON:-30 4 * * *}

# Make a whereabouts.d directory (for our kubeconfig)

mkdir -p $CNI_CONF_DIR/whereabouts.d
WHEREABOUTS_KUBECONFIG=$CNI_CONF_DIR/whereabouts.d/whereabouts.kubeconfig
WHEREABOUTS_GLOBALCONFIG=$CNI_CONF_DIR/whereabouts.d/whereabouts.conf
WHEREABOUTS_CONF_FILE=$CNI_CONF_DIR/whereabouts.d/whereabouts.conf
WHEREABOUTS_KUBECONFIG_LITERAL=$(echo "$WHEREABOUTS_KUBECONFIG" | sed -e s'|/host||')

# Write the nodename to the whereabouts.d directory for standardized hostname reference across cloud providers
echo $NODENAME > $CNI_CONF_DIR/whereabouts.d/nodename

# ------------------------------- Generate a "kube-config"
SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
SERVICE_ACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
SERVICE_ACCOUNT_TOKEN_PATH=$SERVICE_ACCOUNT_PATH/token
SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}


function log()
{
echo "$(date -Iseconds) ${1}"
}

function error()
{
log "ERR: {$1}"
}

function warn()
{
log "WARN: {$1}"
}


function generateKubeConfig {
# Check if we're running as a k8s pod.
if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then
# We're running as a k8d pod - expect some variables.
Expand All @@ -571,25 +609,31 @@ spec:
TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
fi

# Kubernetes service address must be wrapped if it is IPv6 address
KUBERNETES_SERVICE_HOST_WRAP=$KUBERNETES_SERVICE_HOST
if [ "$KUBERNETES_SERVICE_HOST_WRAP" != "${KUBERNETES_SERVICE_HOST_WRAP#*:[0-9a-fA-F]}" ]; then
KUBERNETES_SERVICE_HOST_WRAP=\[$KUBERNETES_SERVICE_HOST_WRAP\]
fi

# Write a kubeconfig file for the CNI plugin. Do this
# to skip TLS verification for now. We should eventually support
# writing more complete kubeconfig files. This is only used
# if the provided CNI network config references it.
touch $WHEREABOUTS_KUBECONFIG
chmod ${KUBECONFIG_MODE:-600} $WHEREABOUTS_KUBECONFIG
cat > $WHEREABOUTS_KUBECONFIG <<EOF
# Kubeconfig file for Multus CNI plugin.
# Kubeconfig file for the Whereabouts CNI plugin.
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST_WRAP}:${KUBERNETES_SERVICE_PORT}
$TLS_CFG
users:
- name: whereabouts
user:
token: "${SERVICEACCOUNT_TOKEN}"
token: "${SERVICE_ACCOUNT_TOKEN}"
contexts:
- name: whereabouts-context
context:
Expand All @@ -599,28 +643,46 @@ spec:
current-context: whereabouts-context
EOF

# Kubeconfig file for Whereabouts CNI plugin.
cat > $WHEREABOUTS_GLOBALCONFIG <<EOF
# Copy the config from ConfigMap to the desired directory
cp /etc/whereabouts/config/whereabouts.conf $WHEREABOUTS_CONF_FILE
chmod 600 $WHEREABOUTS_CONF_FILE

else
warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
fi

}

function generateWhereaboutsConf {

touch $WHEREABOUTS_CONF_FILE
chmod ${KUBECONFIG_MODE:-600} $WHEREABOUTS_CONF_FILE
cat > $WHEREABOUTS_CONF_FILE <<EOF
{
"datastore": "kubernetes",
"kubernetes": {
"kubeconfig": "/etc/kubernetes/cni/net.d/whereabouts.d/whereabouts.kubeconfig"
"kubeconfig": "${WHEREABOUTS_KUBECONFIG_LITERAL}"
},
"reconciler_cron_expression": "30 4 * * *",
"log_level": "debug"
"reconciler_cron_expression": "${WHEREABOUTS_RECONCILER_CRON}"
}
EOF
chmod 600 $WHEREABOUTS_GLOBALCONFIG

else
warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
fi
}

generateKubeConfig
# ------------------ end Generate a "kube-config"

# ----------------- Generate a whereabouts conf
# removed because we have the configmap
#generateWhereaboutsConf
# ---------------- End generate a whereabouts conf


# copy whereabouts to the cni bin dir
# SKIPPED DUE TO FIPS COPY.
# cp -f /whereabouts $CNI_BIN_DIR

# ---------------------- end Generate a "kube-config".
# ---------------------- end generate a "kube-config".

# Unless told otherwise, sleep forever.
# This prevents Kubernetes from restarting the pod repeatedly.
Expand All @@ -639,7 +701,14 @@ spec:
name: cnibin
- name: system-cni-dir
mountPath: /host/etc/cni/net.d
- name: whereabouts-flatfile-configmap
mountPath: /etc/whereabouts/config
env:
- name: NODENAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: KUBERNETES_SERVICE_PORT
value: "{{.KUBERNETES_SERVICE_PORT}}"
- name: KUBERNETES_SERVICE_HOST
Expand Down Expand Up @@ -692,7 +761,10 @@ spec:
configMap:
name: {{.CniSysctlAllowlist}}
defaultMode: 0744
{{if .RenderIpReconciler}}
- name: whereabouts-flatfile-configmap
configMap:
name: whereabouts-flatfile-config
{{if .RenderWhereaboutsAuxillary}}
---
apiVersion: apps/v1
kind: DaemonSet
Expand Down Expand Up @@ -768,4 +840,168 @@ spec:
items:
- key: reconciler_cron_expression
path: config
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: whereabouts-token-watcher
namespace: openshift-multus
annotations:
kubernetes.io/description: |
This deamon watches over the whereabouts service account token and CA
file for changes and will regenerate a kubeconfig if changes are seen
relase.openshift.io.version: "{{.ReleaseVersion}}"
spec:
selector:
matchLabels:
app: whereabouts-token-watcher
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 10%
template:
metadata:
annotations:
target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
# prevent blocks when node critical pods get evicted prior to workloads
cluster-autoscaler.kubernetes.io/enable-ds-eviction: "false"
labels:
app: whereabouts-token-watcher
spec:
nodeSelector:
kubernetes.io/os: linux
priorityClassName: "system-node-critical"
tolerations:
- operator: Exists
serviceAccountName: multus-ancillary-tools
containers:
- name: whereabouts-token-watcher
image: {{.WhereaboutsImage}}
command:
- /bin/sh
- -c
- |
#!/bin/sh

set -u -e

SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
SERVICE_ACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
SERVICE_ACCOUNT_TOKEN_PATH=$SERVICE_ACCOUNT_PATH/token
SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}

function generateKubeConfig {
# Check if we're running as a k8s pod.
if [ -f "$SERVICE_ACCOUNT_PATH/token" ]; then
# We're running as a k8d pod - expect some variables.
if [ -z ${KUBERNETES_SERVICE_HOST} ]; then
error "KUBERNETES_SERVICE_HOST not set"; exit 1;
fi
if [ -z ${KUBERNETES_SERVICE_PORT} ]; then
error "KUBERNETES_SERVICE_PORT not set"; exit 1;
fi

if [ "$SKIP_TLS_VERIFY" == "true" ]; then
TLS_CFG="insecure-skip-tls-verify: true"
elif [ -f "$KUBE_CA_FILE" ]; then
TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
fi

# Kubernetes service address must be wrapped if it is IPv6 address
KUBERNETES_SERVICE_HOST_WRAP=$KUBERNETES_SERVICE_HOST
if [ "$KUBERNETES_SERVICE_HOST_WRAP" != "${KUBERNETES_SERVICE_HOST_WRAP#*:[0-9a-fA-F]}" ]; then
KUBERNETES_SERVICE_HOST_WRAP=\[$KUBERNETES_SERVICE_HOST_WRAP\]
fi
# Write a kubeconfig file for the CNI plugin. Do this
# to skip TLS verification for now. We should eventually support
# writing more complete kubeconfig files. This is only used
# if the provided CNI network config references it.
touch $WHEREABOUTS_KUBECONFIG
chmod ${KUBECONFIG_MODE:-600} $WHEREABOUTS_KUBECONFIG
cat > $WHEREABOUTS_KUBECONFIG <<EOF
# Kubeconfig file for the Whereabouts CNI plugin.
apiVersion: v1
kind: Config
clusters:
- name: local
cluster:
server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST_WRAP}:${KUBERNETES_SERVICE_PORT}
$TLS_CFG
users:
- name: whereabouts
user:
token: "${SERVICE_ACCOUNT_TOKEN}"
contexts:
- name: whereabouts-context
context:
cluster: local
user: whereabouts
namespace: ${WHEREABOUTS_NAMESPACE}
current-context: whereabouts-context
EOF

else
warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
fi

}

function get_token_md5sum {
md5sum "$SERVICE_ACCOUNT_TOKEN_PATH" | awk '{print $1}'
}

function get_ca_file_md5sum {
if [ ! -f "$KUBE_CA_FILE" ]; then
echo ""
return
fi
md5sum "$KUBE_CA_FILE" | awk '{print $1}'
}

export LAST_SERVICEACCOUNT_MD5SUM="$(get_token_md5sum)"
export LAST_KUBE_CA_FILE_MD5SUM="$(get_ca_file_md5sum)"

echo "Sleep and Watching for service account token and CA file changes..."
# enter sleep/watch loop
while true; do
# Check the md5sum of the service account token and ca.
svcaccountsum="$(get_token_md5sum)"
casum="$(get_ca_file_md5sum)"
if [ "$svcaccountsum" != "$LAST_SERVICEACCOUNT_MD5SUM" ] || ! [ "$SKIP_TLS_VERIFY" == "true" ] && [ "$casum" != "$LAST_KUBE_CA_FILE_MD5SUM" ]; then
log "Detected service account or CA file change, regenerating kubeconfig..."
generateKubeConfig
LAST_SERVICEACCOUNT_MD5SUM="$svcaccountsum"
LAST_KUBE_CA_FILE_MD5SUM="$casum"
fi

sleep 1s
done

resources:
requests:
cpu: 10m
memory: 10Mi
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- name: whereabouts-flatfile-configmap
mountPath: /etc/wherabouts/config
env:
- name: KUBERNETES_SERVICE_PORT
value: "{{.KUBERNETES_SERVICE_PORT}}"
- name: KUBERNETES_SERVICE_HOST
value: "{{.KUBERNETES_SERVICE_HOST}}"
- name: CNI_BIN_DIR
value: "/host/opt/cni/bin/"
- name: CNI_CONF_DIR
value: "/host/etc/cni/net.d"
- name: SLEEP
value: "false"
- name: WHEREABOUTS_NAMESPACE
value: "openshift-multus"
volumes:
- name: whereabouts-flatfile-configmap
configMap:
name: whereabouts-flatfile-config
{{- end}}
---
Loading