Skip to content

Commit fbcfda3

Browse files
mattshaxmattshax
authored
Fix FIPS selftest crash by pinning opencv-python-headless (#96)
* Use sparse checkout and targeted LFS fetch for HuggingFace model cloning Replace bare `git lfs pull` (which downloads all LFS files including duplicate .bin weights) with sparse checkout + targeted LFS fetch. This prevents hanging on large models by excluding .bin/.gguf/.onnx files, adding progress output, retry logic, and concurrent transfer configuration. Falls back to .bin if no safetensors are found. * Fix FIPS selftest crash by pinning opencv-python-headless to 4.12.0.88 opencv-python-headless >=4.13 bundles a FIPS-enabled OpenSSL 1.1.1k from CentOS/RHEL. On FIPS-enabled HPC hosts (where Singularity exposes /proc/sys/crypto/fips_enabled=1), this bundled library triggers a fatal FIPS selftest failure at import time, crashing vllm serve. Pin to 4.12.0.88 which does not bundle OpenSSL. Also remove ineffective FIPS workarounds from both containers: - OPENSSL_CONF=/dev/null (targeted system OpenSSL 3.0.2, not the culprit) - OPENSSL_FORCE_FIPS_MODE=0 (not a real OpenSSL variable) - apt-get reinstall libssl-dev openssl (wrong library) - pip rebuild cryptography from source (unnecessary) Refs: opencv/opencv-python#1184 Refs: opencv/opencv-python#1191
1 parent 6337e5f commit fbcfda3

4 files changed

Lines changed: 93 additions & 6 deletions

File tree

‎singularity/Singularity.rag‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@ mkdir -p /app
1515
cd /app
1616
apt-get update && apt-get install -y --no-install-recommends git wget curl build-essential ca-certificates && rm -rf /var/lib/apt/lists/*
1717
pip3 install --no-cache-dir chromadb "fastapi>=0.110,<0.112" "uvicorn[standard]>=0.29,<0.31" "transformers>=4.41,<4.45" "sentence-transformers>=2.6,<3" "chromadb==0.5.4" "watchdog>=3,<5" "pypdf>=4,<5" requests httpx pyyaml
18+
1819
HF_HOME=/root/.cache/huggingface
1920
chmod +x singularity-entrypoint.sh
2021

2122
%environment
2223
export HF_HOME=/root/.cache/huggingface
24+
export LD_LIBRARY_PATH=/opt/conda/lib:/usr/local/lib:/usr/lib/x86_64-linux-gnu:/usr/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
2325

2426
%runscript
2527
/bin/bash singularity-entrypoint.sh

‎singularity/Singularity.vllm‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@ From: vllm/vllm-openai:latest
1212
pkg-config cmake ninja-build
1313
rm -rf /var/lib/apt/lists/*
1414

15+
# FIPS workaround: opencv-python-headless >=4.13 bundles a FIPS-enabled
16+
# OpenSSL 1.1.1k from CentOS/RHEL that crashes on FIPS-enabled hosts.
17+
# Pin to 4.12.0.88 which does not bundle OpenSSL.
18+
# See: https://github.com/opencv/opencv-python/issues/1184
19+
pip install opencv-python-headless==4.12.0.88
20+
1521
# Pick a Python interpreter that actually exists in the base image
1622
if command -v python >/dev/null 2>&1; then
1723
PY=python
@@ -37,6 +43,9 @@ print("Transformers:", transformers.__version__)
3743
print("vLLM:", vllm.__version__)
3844
PY
3945

46+
%environment
47+
export LD_LIBRARY_PATH=/usr/local/lib:/usr/lib/x86_64-linux-gnu:/usr/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
48+
4049
%runscript
4150
mkdir -p /app
4251
cd /app

‎workflow.yaml‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -468,10 +468,48 @@ jobs:
468468
REPO_URL="https://huggingface.co/$MODEL_ID"
469469
[[ -n "$HF_TOKEN" ]] && REPO_URL="https://user:${HF_TOKEN}@huggingface.co/$MODEL_ID"
470470
471-
# Clone with LFS and pull large files
472-
GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 "$REPO_URL" "$TARGET_DIR"
471+
# Clone without LFS or checkout, then sparse-checkout only safetensors
472+
GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 --no-checkout "$REPO_URL" "$TARGET_DIR"
473473
cd "$TARGET_DIR"
474-
git lfs pull
474+
475+
# Sparse checkout: exclude bin/gguf/onnx/pth to only get safetensors weights
476+
git sparse-checkout init --no-cone
477+
git sparse-checkout set '/*' '!*.bin' '!*.gguf' '!*.onnx' '!consolidated*.pth'
478+
git checkout
479+
480+
# Configure LFS for reliability with large models
481+
git lfs install --local
482+
git config lfs.concurrenttransfers 4
483+
git config lfs.transfer.maxretries 10
484+
git config lfs.transfer.maxretrydelay 30
485+
486+
# Fetch only safetensors LFS objects with progress and retry logic
487+
attempt=1
488+
max_attempts=3
489+
while true; do
490+
echo "LFS fetch attempt ${attempt}/${max_attempts}..."
491+
if git lfs fetch --progress --include="*.safetensors"; then
492+
break
493+
fi
494+
if [[ $attempt -ge $max_attempts ]]; then
495+
echo "ERROR: LFS fetch failed after ${max_attempts} attempts"
496+
exit 1
497+
fi
498+
sleep $((attempt * 5))
499+
((attempt++))
500+
done
501+
echo "Checking out LFS files..."
502+
git lfs checkout --include="*.safetensors"
503+
504+
# Fallback: if model only ships bin weights, re-include and fetch those
505+
if ! ls *.safetensors 1>/dev/null 2>&1; then
506+
echo "No safetensors files found, falling back to bin weights..."
507+
git sparse-checkout set '/*'
508+
git checkout
509+
git lfs fetch --progress --include="*.bin"
510+
git lfs checkout --include="*.bin"
511+
fi
512+
475513
cd ..
476514
477515
# Verify model weights exist (not just LFS pointers)

‎yamls/hsp.yaml‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -468,10 +468,48 @@ jobs:
468468
REPO_URL="https://huggingface.co/$MODEL_ID"
469469
[[ -n "$HF_TOKEN" ]] && REPO_URL="https://user:${HF_TOKEN}@huggingface.co/$MODEL_ID"
470470
471-
# Clone with LFS and pull large files
472-
GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 "$REPO_URL" "$TARGET_DIR"
471+
# Clone without LFS or checkout, then sparse-checkout only safetensors
472+
GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 --no-checkout "$REPO_URL" "$TARGET_DIR"
473473
cd "$TARGET_DIR"
474-
git lfs pull
474+
475+
# Sparse checkout: exclude bin/gguf/onnx/pth to only get safetensors weights
476+
git sparse-checkout init --no-cone
477+
git sparse-checkout set '/*' '!*.bin' '!*.gguf' '!*.onnx' '!consolidated*.pth'
478+
git checkout
479+
480+
# Configure LFS for reliability with large models
481+
git lfs install --local
482+
git config lfs.concurrenttransfers 4
483+
git config lfs.transfer.maxretries 10
484+
git config lfs.transfer.maxretrydelay 30
485+
486+
# Fetch only safetensors LFS objects with progress and retry logic
487+
attempt=1
488+
max_attempts=3
489+
while true; do
490+
echo "LFS fetch attempt ${attempt}/${max_attempts}..."
491+
if git lfs fetch --progress --include="*.safetensors"; then
492+
break
493+
fi
494+
if [[ $attempt -ge $max_attempts ]]; then
495+
echo "ERROR: LFS fetch failed after ${max_attempts} attempts"
496+
exit 1
497+
fi
498+
sleep $((attempt * 5))
499+
((attempt++))
500+
done
501+
echo "Checking out LFS files..."
502+
git lfs checkout --include="*.safetensors"
503+
504+
# Fallback: if model only ships bin weights, re-include and fetch those
505+
if ! ls *.safetensors 1>/dev/null 2>&1; then
506+
echo "No safetensors files found, falling back to bin weights..."
507+
git sparse-checkout set '/*'
508+
git checkout
509+
git lfs fetch --progress --include="*.bin"
510+
git lfs checkout --include="*.bin"
511+
fi
512+
475513
cd ..
476514
477515
# Verify model weights exist (not just LFS pointers)

0 commit comments

Comments
 (0)