package linux-libc-dev upgrade

dlc_developer_config.toml

@@ -1,6 +1,6 @@
 [dev]
 # Set to "huggingface", for example, if you are a huggingface developer. Default is ""
-partner_developer = ""
+partner_developer = "huggingface"
 # Please only set it to true if you are preparing an EI related PR
 # Do remember to revert it back to false before merging any PR (including EI dedicated PR)
 ei_mode = false
@@ -37,12 +37,12 @@ deep_canary_mode = false
 [build]
 # Add in frameworks you would like to build. By default, builds are disabled unless you specify building an image.
 # available frameworks - ["autogluon", "huggingface_tensorflow", "huggingface_pytorch", "huggingface_tensorflow_trcomp", "huggingface_pytorch_trcomp", "pytorch_trcomp", "tensorflow", "pytorch", "stabilityai_pytorch"]
-build_frameworks = []
+build_frameworks = ["huggingface_pytorch"]
 
 
 # By default we build both training and inference containers. Set true/false values to determine which to build.
 build_training = true
-build_inference = true
+build_inference = false
 
 # Set do_build to "false" to skip builds and test the latest image built by this PR
 # Note: at least one build is required to set do_build to "false"

huggingface/pytorch/training/docker/2.5/py3/cu124/Dockerfile.gpu

@@ -49,28 +49,16 @@ RUN pip install --no-cache-dir \
     peft==${PEFT_VERSION} \
     flash-attn==${FLASH_ATTN_VERSION}
 
-# Override conflicting versions to satisfy datasets
-RUN pip install --no-cache-dir dill==0.3.8 multiprocess==0.70.16 \
- && pip install --no-cache-dir pathos==0.3.3 --no-deps \
- && PATHOS_META=$(find /opt/conda/lib -type f -path "*pathos-0.3.3.dist-info/METADATA") \
- && sed -i 's/dill.*/dill/' $PATHOS_META \
- && sed -i 's/multiprocess.*/multiprocess/' $PATHOS_META
-
+RUN apt-get update \
+# TODO: Remove upgrade statements once packages are updated in base image
+&& apt-get -y upgrade --only-upgrade systemd openssl cryptsetup libkrb5-3 linux-libc-dev git-lfs \
+&& apt install -y git \
+&& apt-get clean \
+&& rm -rf /var/lib/apt/lists/*
 
 # hf_transfer will be a built-in feature, remove the env variable then
 ENV HF_HUB_ENABLE_HF_TRANSFER="1"
-
-RUN apt-get update \
- # TODO: Remove upgrade statements once packages are updated in base image
- && apt-get -y upgrade --only-upgrade systemd openssl cryptsetup libkrb5-3 \
- && apt-get install -y git git-lfs wget tar \
- && wget https://go.dev/dl/go1.22.3.linux-amd64.tar.gz \
- && rm -rf /usr/local/go \
- && tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz \
- && ln -s /usr/local/go/bin/go /usr/bin/go \
- && rm go1.22.3.linux-amd64.tar.gz \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
+ENV HF_HUB_USER_AGENT_ORIGIN="aws:sagemaker:gpu-cuda:training"
 
 COPY cuda-compatibility-lib.sh /usr/local/bin/cuda-compatibility-lib.sh
 RUN chmod +x /usr/local/bin/cuda-compatibility-lib.sh