Skip to content

Install Docker CLI after workflows files are created #629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Install Docker CLI after workflows files are created #629

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 26 additions & 23 deletions template/v2/dirs/etc/sagemaker-ui/workflows/start-workflows-container.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ handle_workflows_startup_error() {
detailed_status="Error creating directories"
;;
3)
detailed_status="Error installing docker"
detailed_status="Error copying prerequisite files"
;;
4)
detailed_status="Error copying prerequisite files"
detailed_status="Error installing docker"
;;
5)
detailed_status="Error starting workflows image"
Expand All @@ -42,6 +42,7 @@ DZ_DOMAIN_ID=$(jq -r '.AdditionalMetadata.DataZoneDomainId' < $RESOURCE_METADATA
DZ_PROJECT_ID=$(jq -r '.AdditionalMetadata.DataZoneProjectId' < $RESOURCE_METADATA_FILE)
DZ_ENV_ID=$(jq -r '.AdditionalMetadata.DataZoneEnvironmentId' < $RESOURCE_METADATA_FILE)
DZ_DOMAIN_REGION=$(jq -r '.AdditionalMetadata.DataZoneDomainRegion' < $RESOURCE_METADATA_FILE)
DZ_ENDPOINT=$(jq -r '.AdditionalMetadata.DataZoneEndpoint' < $RESOURCE_METADATA_FILE)
DZ_PROJECT_S3PATH=$(jq -r '.AdditionalMetadata.ProjectS3Path' < $RESOURCE_METADATA_FILE)
WORKFLOW_DAG_PATH="/home/sagemaker-user/${HOME_FOLDER_NAME}/workflows/dags"
WORKFLOW_CONFIG_PATH="/home/sagemaker-user/${HOME_FOLDER_NAME}/workflows/config"
Expand All @@ -67,7 +68,7 @@ if [ ! -f "${WORKFLOW_HEALTH_PATH}/status.json" ]; then
fi

# Only start local runner if Workflows blueprint is enabled
if [ "$(python /etc/sagemaker-ui/workflows/workflow_client.py check-blueprint --domain-id "$DZ_DOMAIN_ID")" = "False" ]; then
if [ "$(python /etc/sagemaker-ui/workflows/workflow_client.py check-blueprint --region "$DZ_DOMAIN_REGION" --domain-id "$DZ_DOMAIN_ID" --endpoint "$DZ_ENDPOINT")" = "False" ]; then
echo "Workflows blueprint is not enabled. Workflows will not start."
handle_workflows_startup_error 0
fi
Expand All @@ -93,26 +94,6 @@ mkdir -p $WORKFLOW_STARTUP_PATH
mkdir -p $WORKFLOW_OUTPUT_PATH
) || handle_workflows_startup_error 2

(
# Set the status of the status file to 'starting'
python /etc/sagemaker-ui/workflows/workflow_client.py update-local-runner-status --status 'starting' --detailed-status 'Installing prerequisites'

# Workflows execution environment install
sudo apt-get update
sudo install -m 0755 -d /etc/apt/keyrings
sudo rm -f /etc/apt/keyrings/docker.gpg
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
sudo chmod a+r /etc/apt/keyrings/docker.gpg
echo \
"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update
VERSION_ID=$(cat /etc/os-release | grep -oP 'VERSION_ID=".*"' | cut -d'"' -f2)
VERSION_STRING=$(sudo apt-cache madison docker-ce | awk '{ print $3 }' | grep -i $VERSION_ID | head -n 1)
sudo apt-get install docker-ce-cli=$VERSION_STRING docker-compose-plugin=2.29.2-1~ubuntu.22.04~jammy -y --allow-downgrades
) || handle_workflows_startup_error 3

(
# Set status to copying files
python /etc/sagemaker-ui/workflows/workflow_client.py update-local-runner-status --status 'starting' --detailed-status 'Copying files'
Expand All @@ -131,6 +112,7 @@ cp $WORKFLOW_REQUIREMENTS_SOURCE_PATH $WORKFLOW_REQUIREMENTS_PATH
cp /etc/sagemaker-ui/workflows/startup/startup.sh $WORKFLOW_STARTUP_PATH

# Append user's custom startup script, if exists
echo -e "\n\n" >> "${WORKFLOW_STARTUP_PATH}startup.sh"
if [ -f $USER_STARTUP_FILE ]; then
tail -n +2 $USER_STARTUP_FILE >> "${WORKFLOW_STARTUP_PATH}startup.sh"
else
Expand All @@ -143,6 +125,7 @@ else
fi

# Append user's custom requirements, if exists
echo -e "\n\n" >> "${WORKFLOW_REQUIREMENTS_PATH}requirements.txt"
if [ -f $USER_REQUIREMENTS_FILE ]; then
cat $USER_REQUIREMENTS_FILE >> "${WORKFLOW_REQUIREMENTS_PATH}requirements.txt"
else
Expand All @@ -158,6 +141,26 @@ if [ -d $USER_PLUGINS_FOLDER ]; then
cp -r $USER_PLUGINS_FOLDER/* $WORKFLOW_PLUGINS_PATH
fi

) || handle_workflows_startup_error 3

(
# Set the status of the status file to 'starting'
python /etc/sagemaker-ui/workflows/workflow_client.py update-local-runner-status --status 'starting' --detailed-status 'Installing prerequisites'

# Workflows execution environment install
sudo apt-get update
sudo install -m 0755 -d /etc/apt/keyrings
sudo rm -f /etc/apt/keyrings/docker.gpg
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
sudo chmod a+r /etc/apt/keyrings/docker.gpg
echo \
"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update
VERSION_ID=$(cat /etc/os-release | grep -oP 'VERSION_ID=".*"' | cut -d'"' -f2)
VERSION_STRING=$(sudo apt-cache madison docker-ce | awk '{ print $3 }' | grep -i $VERSION_ID | head -n 1)
sudo apt-get install docker-ce-cli=$VERSION_STRING docker-compose-plugin=2.29.2-1~ubuntu.22.04~jammy -y --allow-downgrades
) || handle_workflows_startup_error 4

(
Expand Down