From 6cd104d5e89f4e73cf7e5b0771afb5ff3f30eca1 Mon Sep 17 00:00:00 2001 From: Joshua Fernandes Date: Mon, 25 Sep 2023 22:24:15 +1000 Subject: [PATCH] fixes for azure-workload-identity - use helm in minikube + fix aws no roles after last commit (#205) --- aws/README.md | 35 ++---------- aws/scripts/bootstrap.sh | 39 +++++++++++++ azure/arm/azuredeploy.json | 54 +++--------------- azure/scripts/bootstrap.sh | 56 +++---------------- helm/README.md | 4 +- .../templates/genesis-job-cleanup.yaml | 8 ++- .../templates/genesis-job-init.yaml | 6 +- .../templates/genesis-service-account.yaml | 45 +++++++++++++++ helm/charts/besu-genesis/values.yaml | 6 +- .../templates/node-hooks-pre-delete.yaml | 6 +- .../templates/node-hooks-pre-install.yaml | 4 +- .../templates/node-hooks-service-account.yaml | 54 ++++++++++++++++++ .../templates/node-service-account.yaml | 45 +++++++++++++++ .../besu-node/templates/node-statefulset.yaml | 4 +- helm/charts/besu-node/values.yaml | 6 +- .../templates/genesis-job-cleanup.yaml | 6 +- .../templates/genesis-job-init.yaml | 6 +- .../templates/genesis-service-account.yaml | 47 ++++++++++++++++ helm/charts/goquorum-genesis/values.yaml | 7 +-- .../templates/node-hooks-pre-delete.yaml | 6 +- .../templates/node-hooks-pre-install.yaml | 8 ++- .../templates/node-hooks-service-account.yaml | 54 ++++++++++++++++++ .../templates/node-service-account.yaml | 44 +++++++++++++++ .../templates/node-statefulset.yaml | 8 ++- helm/charts/goquorum-node/values.yaml | 6 +- helm/values/bootnode.yml | 6 +- helm/values/genesis-besu.yml | 6 +- helm/values/genesis-goquorum.yml | 6 +- helm/values/reader.yml | 6 +- helm/values/txnode.yml | 6 +- helm/values/validator.yml | 6 +- 31 files changed, 425 insertions(+), 175 deletions(-) create mode 100755 aws/scripts/bootstrap.sh create mode 100644 helm/charts/besu-genesis/templates/genesis-service-account.yaml create mode 100644 helm/charts/besu-node/templates/node-hooks-service-account.yaml create mode 100644 helm/charts/besu-node/templates/node-service-account.yaml create mode 100644 helm/charts/goquorum-genesis/templates/genesis-service-account.yaml create mode 100644 helm/charts/goquorum-node/templates/node-hooks-service-account.yaml create mode 100644 helm/charts/goquorum-node/templates/node-service-account.yaml diff --git a/aws/README.md b/aws/README.md index 475b40ab..15b964d6 100644 --- a/aws/README.md +++ b/aws/README.md @@ -49,43 +49,20 @@ aws sts get-caller-identity aws eks --region AWS_REGION update-kubeconfig --name CLUSTER_NAME ``` -4. Provision EFS CSI Driver (optional) - -The `cluster.yml` file that is included in this folder uses the EBS drivers but also deploys the EFS IAM policies ie you still need to install the EFS CSI drivers. This can be done following the [AWS Docs ](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) - -5. [Provision Secrets Drivers](https://github.com/aws/secrets-store-csi-driver-provider-aws) +4. [Provision Secrets Drivers](https://github.com/aws/secrets-store-csi-driver-provider-aws) Once the deployment has completed, please provision the Secrets Manager identity and the CSI drivers -Use `quorum` (or equivalent) for `NAMESPACE` below and update `AWS_REGION` and `CLUSTER_NAME` to match your settings from step 2. - ```bash - -helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts -helm install --namespace kube-system --create-namespace csi-secrets-store secrets-store-csi-driver/secrets-store-csi-driver -kubectl apply -f https://raw.githubusercontent.com/aws/secrets-store-csi-driver-provider-aws/main/deployment/aws-provider-installer.yaml - -POLICY_ARN=$(aws --region AWS_REGION --query Policy.Arn --output text iam create-policy --policy-name quorum-node-secrets-mgr-policy --policy-document '{ - "Version": "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Action": ["secretsmanager:CreateSecret","secretsmanager:UpdateSecret","secretsmanager:DescribeSecret","secretsmanager:GetSecretValue","secretsmanager:PutSecretValue","secretsmanager:ReplicateSecretToRegions","secretsmanager:TagResource"], - "Resource": ["arn:aws:secretsmanager:AWS_REGION:AWS_ACCOUNT:secret:goquorum-node-*", "arn:aws:secretsmanager:AWS_REGION:AWS_ACCOUNT:secret:besu-node-*"] - } ] -}') - - -If you have deployed the above policy before, you can acquire its ARN: -POLICY_ARN=$(aws iam list-policies --scope Local \ ---query 'Policies[?PolicyName==`quorum-node-secrets-mgr-policy`].Arn' \ ---output text) - -eksctl create iamserviceaccount --name quorum-node-secrets-sa --namespace NAMESPACE --region=AWS_REGION --cluster CLUSTER_NAME --attach-policy-arn "$POLICY_ARN" --approve --override-existing-serviceaccounts +./scripts/bootstrap.sh "AWS_REGION" "AWS_ACCOUNT" "CLUSTER_NAME" "AKS_NAMESPACE" ``` -| ⚠️ **Note**: The above command creates a service account called `quorum-node-secrets-sa`. Please use the same in the values.yml files under the `aws` map. If you would like to change the name of the service account, please remember to do it in both places | +| ⚠️ **Note**: The above command creates a service account called `quorum-sa`. Please use the same in the values.yml files under the `aws` map. If you would like to change the name of the service account, please remember to do it in both places | | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +5. Provision EFS CSI Driver (optional) + +The `cluster.yml` file that is included in this folder uses the EBS drivers but also deploys the EFS IAM policies ie you still need to install the EFS CSI drivers. This can be done following the [AWS Docs ](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) 6. Deploy the charts as per the `helm` folder readme files diff --git a/aws/scripts/bootstrap.sh b/aws/scripts/bootstrap.sh new file mode 100755 index 00000000..483cb795 --- /dev/null +++ b/aws/scripts/bootstrap.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Run as: +# ./bootstrap.sh "AWS_REGION" "AWS_ACCOUNT" "CLUSTER_NAME" "AKS_NAMESPACE" +# + +set -eux + +AWS_REGION=${1:-rg} +AWS_ACCOUNT=${2:-account} +CLUSTER_NAME=${3:-cluster} +# quourum +AKS_NAMESPACE=${4:-quorum} + +echo "aws get-credentials ..." +aws sts get-caller-identity +aws eks --region "${AWS_REGION}" update-kubeconfig --name "${CLUSTER_NAME}" + +helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts +helm install --namespace kube-system --create-namespace csi-secrets-store secrets-store-csi-driver/secrets-store-csi-driver +kubectl apply -f https://raw.githubusercontent.com/aws/secrets-store-csi-driver-provider-aws/main/deployment/aws-provider-installer.yaml + +# If you have deployed the above policy before, acquire its ARN: +POLICY_ARN=$(aws iam list-policies --scope Local --query 'Policies[?PolicyName==`quorum-node-secrets-mgr-policy`].Arn' --output text) +if [ $? -eq 1 ] +then + echo "Deploy the policy" + POLICY_ARN=$(aws --region $AWS_REGION --query Policy.Arn --output text iam create-policy --policy-name quorum-node-secrets-mgr-policy --policy-document '{ + "Version": "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Action": ["secretsmanager:CreateSecret","secretsmanager:UpdateSecret","secretsmanager:DescribeSecret","secretsmanager:GetSecretValue","secretsmanager:PutSecretValue","secretsmanager:ReplicateSecretToRegions","secretsmanager:TagResource"], + "Resource": ["arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT:secret:goquorum-node-*", "arn:aws:secretsmanager:$AWS_REGION:$AWS_ACCOUNT:secret:besu-node-*"] + } ] + }') +fi + +eksctl create iamserviceaccount --name quorum-sa --namespace "${NAMESPACE}" --region="${AWS_REGION}" --cluster "${CLUSTER_NAME}" --attach-policy-arn "$POLICY_ARN" --approve --override-existing-serviceaccounts +echo "Done... " diff --git a/azure/arm/azuredeploy.json b/azure/arm/azuredeploy.json index 78499119..68a50fcb 100644 --- a/azure/arm/azuredeploy.json +++ b/azure/arm/azuredeploy.json @@ -41,7 +41,7 @@ "variables": { "apiVersionVirtualNetworks": "2020-05-01", "apiVersionNetworkSecurityGroups": "2018-11-01", - "apiVersionManangedClusters": "2020-09-01", + "apiVersionManangedClusters": "2023-06-01", "apiVersionUserManangedIdentity": "2018-11-30", "apiVersionRoleAssignments": "2018-09-01-preview", "apiVersionKeyVault": "2018-02-14", @@ -73,9 +73,7 @@ "monitoringMetricsPublisherRoleId": "3913510d-42f4-4e42-8a64-420c390055eb", "readerRoleId": "acdd72a7-3385-48ef-bd42-f606fba81ae7", "aksClusterAdminRoleId": "0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8", - "networkContributorRoleId": "4d97b98b-1d4f-4787-a291-c67834d212e7", - "serviceBusReceiverRoleId": "4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0", - "serviceBusSenderRoleId": "69a216fc-b8fb-44d8-bc22-1f3c2cd27a39" + "networkContributorRoleId": "4d97b98b-1d4f-4787-a291-c67834d212e7" }, "resources": [ { @@ -222,34 +220,6 @@ "principalType": "ServicePrincipal" } }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "[variables('apiVersionRoleAssignments')]", - "name": "[guid(resourceGroup().id, deployment().name, variables('opsManagedIdentity'), '-sb-receiver-ra')]", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('opsManagedIdentity'))]" - ], - "properties": { - "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('serviceBusReceiverRoleId'))]", - "principalId": "[reference(variables('opsManagedIdentity')).principalId]", - "scope": "[resourceGroup().id]", - "principalType": "ServicePrincipal" - } - }, - { - "type": "Microsoft.Authorization/roleAssignments", - "apiVersion": "[variables('apiVersionRoleAssignments')]", - "name": "[guid(resourceGroup().id, deployment().name, variables('opsManagedIdentity'), '-sb-sender-ra')]", - "dependsOn": [ - "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('opsManagedIdentity'))]" - ], - "properties": { - "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('serviceBusSenderRoleId'))]", - "principalId": "[reference(variables('opsManagedIdentity')).principalId]", - "scope": "[resourceGroup().id]", - "principalType": "ServicePrincipal" - } - }, { "type": "Microsoft.Authorization/roleAssignments", "apiVersion": "[variables('apiVersionRoleAssignments')]", @@ -360,19 +330,13 @@ "servicePrincipalProfile": { "clientId": "msi" }, - "podIdentityProfile": { - "enabled": true, - "userAssignedIdentities": [ - { - "name": "quorum-pod-identity", - "namespace": "default", - "identity": { - "resourceId": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('opsManagedIdentity'))]", - "clientId": "[reference(variables('opsManagedIdentity')).clientId]", - "objectId": "[reference(variables('opsManagedIdentity')).principalId]" - } - } - ] + "oidcIssuerProfile": { + "enabled": true + }, + "securityProfile": { + "workloadIdentity": { + "enabled": true + } }, "networkProfile": { "networkPlugin": "azure", diff --git a/azure/scripts/bootstrap.sh b/azure/scripts/bootstrap.sh index 1660e7f7..af67477a 100755 --- a/azure/scripts/bootstrap.sh +++ b/azure/scripts/bootstrap.sh @@ -1,8 +1,5 @@ #!/bin/bash # -# This bootstraps the ops vm to run helm charts on the aks cluster. -# This is for dev only at present, and this functionality will be moved to a lambda function -# # Run as: # ./bootstrap.sh "AKS_RESOURCE_GROUP" "AKS_CLUSTER_NAME" "AKS_MANAGED_IDENTITY" "AKS_NAMESPACE" # @@ -13,7 +10,7 @@ AKS_CLUSTER_NAME=${2:-cluster} AKS_MANAGED_IDENTITY=${3:-identity} # quourum AKS_NAMESPACE=${4:-quorum} -SA_NAME=${5:-quorum} +SA_NAME=${5:-quorum-sa} echo "az get-credentials ..." # if running this on a VM/Function/etc use a managed identity @@ -22,61 +19,26 @@ echo "az get-credentials ..." az login # https://learn.microsoft.com/en-us/azure/aks/use-oidc-issuer -echo "Update the cluster to use oidc issuer and workload identity ... " -az aks update -g myResourceGroup -n myAKSCluster --enable-oidc-issuer --enable-workload-identity - -echo "Provisioning AAD pod-identity... " +echo "Get the oidc issuer and workload identity ID from the cluster... " AKS_MANAGED_IDENTITY_RESOURCE_ID=$(az identity show --name "$AKS_MANAGED_IDENTITY" --resource-group "$AKS_RESOURCE_GROUP" | jq -r '.id') -AKS_OIDC_ISSUER=$(az aks show --name "$AKS_MANAGED_IDENTITY" --resource-group "$AKS_RESOURCE_GROUP" --query "oidcIssuerProfile.issuerUrl" -otsv) +AKS_OIDC_ISSUER=$(az aks show -n $AKS_CLUSTER_NAME -g "$AKS_RESOURCE_GROUP" --query "oidcIssuerProfile.issuerUrl" -otsv) -# https://learn.microsoft.com/en-us/azure/aks/workload-identity-deploy-cluster +# https://learn.microsoft.com/en-gb/azure/aks/workload-identity-deploy-cluster#create-kubernetes-service-account cat <