Fix bump blob bootstrap flow

.final_builds/jobs/database-backup-restorer/index.yml

@@ -19,6 +19,10 @@ builds:
     version: 2f98fb38dc705679c7bcac5c3cfa7fc27d834df0199111580963cb6b4fc679b2
     blobstore_id: ff46d340-72df-46b1-41b5-113c6c22455f
     sha1: sha256:a2b841768b139e9129e8ba0e75a8db61df95aa074c73e007f54b0f648fa77798
+  3a3bb77fbaa6415535b3e0bad6ab83bfc70efbac9cbcf3a8166b1eaef58dd2a9:
+    version: 3a3bb77fbaa6415535b3e0bad6ab83bfc70efbac9cbcf3a8166b1eaef58dd2a9
+    blobstore_id: 36556d6c-f4d9-4698-6f6b-b010e0429e89
+    sha1: sha256:4e9a9b3c679e0eb18d0d465ad293c568a906f698c1dd7fc61acd77dc7549c6a8
   3ae2b8039f826d1709a4ec165d2ec958efc03d70:
     version: 3ae2b8039f826d1709a4ec165d2ec958efc03d70
     blobstore_id: 9e022592-31ef-4908-4ae8-cc1583e5362a

.final_builds/packages/database-backup-restorer-postgres-15/index.yml

@@ -19,6 +19,10 @@ builds:
     version: 3c9b445ffd0db7887aae2be2c59f0f7a4eb0c393398eaa156b0e6a2610614501
     blobstore_id: 91fa5c8e-148d-4f7b-4435-1206137d6cd2
     sha1: sha256:84a1a0a86ff545f25a0a8025bb17d7cff5bf7a20ec7db5a6e6bd78a5f837fd84
+  3cd9eb77945198428371b95864be62ce93d4c38f7df7e64272ad58112de301eb:
+    version: 3cd9eb77945198428371b95864be62ce93d4c38f7df7e64272ad58112de301eb
+    blobstore_id: 073330b9-14f1-423f-7717-c2881417969d
+    sha1: sha256:a070ab1104a66e7acffa7e0c52a6889eb22c0577edeb248fa28a3bb683851fbd
   6a4454134e4f34c77674b103f95dc315e13cee6d843462828f88aa519736db10:
     version: 6a4454134e4f34c77674b103f95dc315e13cee6d843462828f88aa519736db10
     blobstore_id: 99fc8638-0798-4499-64ec-5a13732690ce

.final_builds/packages/database-backup-restorer-postgres-16/index.yml

@@ -0,0 +1,6 @@
+builds:
+  0f7a70c489ad66cc581db8b7b1f12eb0d895c3a538a590235e42ebbbeed2f754:
+    version: 0f7a70c489ad66cc581db8b7b1f12eb0d895c3a538a590235e42ebbbeed2f754
+    blobstore_id: d77b739d-cfbd-4d4b-702d-c7abd8740021
+    sha1: sha256:e17a727e526bfb9a292e307b28cd8cbe67bfd1e8e5ab94c8b1b2234c070a0b4e
+format-version: "2"

.final_builds/packages/database-backup-restorer/index.yml

@@ -315,6 +315,10 @@ builds:
     version: d02eb7564eaf8ebc09186591b64c311bd7879865f1ae2ed8c8d0e91ccf5d7220
     blobstore_id: 3e033f13-e1e3-410f-64a5-9a5c780a47dd
     sha1: sha256:d6747096fb2e21c75ef44ff957a36e1070863d49ef071f113728393c8e1b02fe
+  d0abf2f70a9339ab7307a3828fce6594f73476de8aadf14d5704bbcd60dfa45f:
+    version: d0abf2f70a9339ab7307a3828fce6594f73476de8aadf14d5704bbcd60dfa45f
+    blobstore_id: 4f9ffcf5-bce8-42e4-64ed-74df5d74599c
+    sha1: sha256:6f2c6f7e8dd05679db9db844e0232deb7afcae685f48604f87dc3f68086517cd
   d23f85c07f901b4f29a0ad127194246942ee3fc9:
     version: d23f85c07f901b4f29a0ad127194246942ee3fc9
     blobstore_id: f3053599-ae73-4fd5-5f71-707364cf7655

.final_builds/packages/s3-blobstore-backup-restorer/index.yml

@@ -55,6 +55,10 @@ builds:
     version: 1fac0669e31dce64ae96661395590d1f07301c79d185b843ecb6149cd01d506c
     blobstore_id: f95cb2ec-1d7a-4f3d-7478-ab1ec54c2714
     sha1: sha256:2108e570e2a5c111adb038e1b6ea99d2156e01a0830900f0260528c8ceb396c6
+  23173e0684232e472c7b3695300895b5189864a745c95854971bd3bdeac92f1b:
+    version: 23173e0684232e472c7b3695300895b5189864a745c95854971bd3bdeac92f1b
+    blobstore_id: 8c7dad40-213e-46c1-4a38-85221e7e0f45
+    sha1: sha256:1b74d52b139be180e5b5a575031dbb849cd2eb41fc0ccf8468ff770dffed31af
   257410456bca84981069962bc571554fe64c7f53:
     version: 257410456bca84981069962bc571554fe64c7f53
     blobstore_id: c3fa4c2c-91d2-4bb5-5604-54af5e8ec8b9

ci/pipeline.yml

@@ -403,6 +403,7 @@ resources:
   icon: github
   source:
     uri: https://github.com/cloudfoundry/backup-and-restore-sdk-release.git
+    branch: *backup_and_restore_sdk_branch
     username: bosh-admin-bot
     password: *git_token
     disable_ci_skip: true
@@ -415,19 +416,19 @@ resources:
   icon: github
   source:
     uri: https://github.com/cloudfoundry/backup-and-restore-sdk-release.git
+    branch: *backup_and_restore_sdk_branch
     username: bosh-admin-bot
     password: *git_token
-    branch: *backup_and_restore_sdk_branch
     ignore_paths: [ "ci" ]
 
 - name: backup-and-restore-sdk-release-ci
   type: git
   icon: github
   source:
     uri: https://github.com/cloudfoundry/backup-and-restore-sdk-release.git
+    branch: *backup_and_restore_sdk_branch
     username: bosh-admin-bot
     password: *git_token
-    branch: *backup_and_restore_sdk_branch
     paths: [ "ci" ]
 
 - name: bosh-shared-ci
@@ -882,8 +883,8 @@ jobs:
   - in_parallel:
     - get: backup-and-restore-sdk-release-ci
     - get: backup-and-restore-sdk-release
-#      passed:
-#        - claim-cf-deployment-airgapped
+      passed:
+        - claim-cf-deployment-airgapped
     - get: release-tarball
       trigger: true
       resource: dev-release
@@ -922,10 +923,10 @@ jobs:
     - get: bosh-integration-image
     - get: backup-and-restore-sdk-release-ci
     - get: backup-and-restore-sdk-release
-#      passed:
-#      - deploy-database-sdk-on-airgapped-env
-#      - deploy-postgres
-#      - deploy-mysql
+      passed:
+      - deploy-database-sdk-on-airgapped-env
+      - deploy-postgres
+      - deploy-mysql
     - get: cert-store
       resource: gcp-db-certs-zip
     - get: cf-deployment-env-airgapped

ci/tasks/bump-blob/task.sh

@@ -61,9 +61,9 @@ EOF
   # it could accidentially remove a blob that we actually still need.
 
  if [[ -n ${KEEP_BLOBS_FILTER} ]]; then
-    CURRENT_FILENAME=$(bosh blobs --column=path | grep "$BLOB_NAME_WITH_PATH"| grep -v "${KEEP_BLOBS_FILTER/,/\\|}" | xargs) # xargs is used to trim the trailing spaces
+    CURRENT_FILENAME=$(bosh blobs --column=path | grep "$BLOB_NAME_WITH_PATH" | { grep -v "${KEEP_BLOBS_FILTER/,/\\|}" || true; } | xargs) # xargs is used to trim the trailing spaces
  else
-    CURRENT_FILENAME=$(bosh blobs --column=path | grep "$BLOB_NAME_WITH_PATH"| xargs)
+    CURRENT_FILENAME=$(bosh blobs --column=path | { grep "$BLOB_NAME_WITH_PATH" || true; } | xargs)
  fi
   # On 2023-11-30, ksemenov tested the version-getting regex as follows:
   # $ for f in autoconf-2.71.tar.gz automake-1.16.5.tar.xz libtool-2.4.7.tar.xz json-c/json-c-0.17-nodoc.tar.gz jq/jq-1.7-linux-amd64; do
@@ -145,20 +145,24 @@ EOF
     NEW_BLOBNAME="${BLOB_DIRECTORY%/}/${NEW_FILENAME#../distributed-package/}"
   fi
 
-  echo "removing old blob(s) $CURRENT_FILENAME"
-  # check if we have " "(spaces) in CURRENT_FILENAME. It would indicate that we are about to remove more than one blob.
-  if [[ $(tr -dc ' ' <<<"$CURRENT_FILENAME" | wc -c) -gt 0 ]]; then
-    if [[ $REMOVE_MULTIPLE_BLOBS == true ]]; then
-      echo "${CURRENT_FILENAME}" | xargs -n1 bosh remove-blob
+  if [[ -z "${CURRENT_FILENAME}" ]]; then
+    echo "No existing blob found for ${BLOB_NAME_WITH_PATH} (first-time add). Adding ${NEW_BLOBNAME}."
+  else
+    echo "removing old blob(s) $CURRENT_FILENAME"
+    # check if we have " "(spaces) in CURRENT_FILENAME. It would indicate that we are about to remove more than one blob.
+    if [[ $(tr -dc ' ' <<<"$CURRENT_FILENAME" | wc -c) -gt 0 ]]; then
+      if [[ $REMOVE_MULTIPLE_BLOBS == true ]]; then
+        echo "${CURRENT_FILENAME}" | xargs -n1 bosh remove-blob
+      else
+        echo -e "It seems this would remove more than one blob. But remove multiple blobs is set to: $REMOVE_MULTIPLE_BLOBS\n bailing out to avoid removing the wrong blob"
+        exit 1
+      fi
     else
-      echo -e "It seems this would remove more than one blob. But remove multiple blobs is set to: $REMOVE_MULTIPLE_BLOBS\n bailing out to avoid removing the wrong blob"
-      exit 1
+      bosh remove-blob "${CURRENT_FILENAME}"
     fi
-  else
-    bosh remove-blob "${CURRENT_FILENAME}"
   fi
 
-  echo "updating ${BLOB_NAME} blob from ${CURRENT_VERSION} to ${NEW_VERSION}"
+  echo "updating ${BLOB_NAME} blob from ${CURRENT_VERSION:-<none>} to ${NEW_VERSION}"
   bosh add-blob "../distributed-package/${NEW_FILENAME}" "$NEW_BLOBNAME"
 
   # check if the blob we just added was already present. If the size and the checksum didn't change, we just added the same blob because the bumper was rerun for any reason. In that case the diff look similar to:
@@ -187,7 +191,7 @@ EOF
   bosh upload-blobs
   git add config/blobs.yml
 
-  if [[ -n "${UPDATE_REFERENCES:-""}" ]]
+  if [[ -n "${UPDATE_REFERENCES:-""}" && -n "${CURRENT_VERSION}" ]]
   then
     if [[ "$UPDATE_REFERENCES" =~ ^only-variable-name: ]]; then
       VARIABLE_NAME="${UPDATE_REFERENCES#only-variable-name:}"
@@ -202,5 +206,9 @@ EOF
 
   git config user.name "${GIT_USERNAME}"
   git config user.email "${GIT_EMAIL}"
-  git commit -m "Bump ${BLOB_NAME} from ${CURRENT_VERSION} to ${NEW_VERSION}"
+  if [[ -n "${CURRENT_VERSION}" ]]; then
+    git commit -m "Bump ${BLOB_NAME} from ${CURRENT_VERSION} to ${NEW_VERSION}"
+  else
+    git commit -m "Add ${BLOB_NAME} ${NEW_VERSION}"
+  fi
 popd

packages/database-backup-restorer-postgres-16/packaging

@@ -7,7 +7,7 @@ mkdir -p postgresql-${POSTGRES16_VERSION}
 tar xzf postgres/postgresql-${POSTGRES16_VERSION}.tar.gz --strip-components=1 -C postgresql-${POSTGRES16_VERSION}
 
 pushd postgresql-${POSTGRES16_VERSION}
-  ./configure --prefix=${BOSH_INSTALL_TARGET} --with-openssl
+  ICU_CFLAGS=" " ICU_LIBS="-L/usr/lib/x86_64-linux-gnu -licui18n -licuuc -licudata" ./configure --prefix=${BOSH_INSTALL_TARGET} --with-openssl
 
   pushd src/bin/pg_config
     make

releases/backup-and-restore-sdk/backup-and-restore-sdk-1.19.50.yml

@@ -0,0 +1,117 @@
+name: backup-and-restore-sdk
+version: 1.19.50
+commit_hash: 38b049748
+uncommitted_changes: false
+jobs:
+- name: azure-blobstore-backup-restorer
+  version: f88fccebfb6347564f17c7b5f59d89681b3dbb18e172562de88f112577d0aebc
+  fingerprint: f88fccebfb6347564f17c7b5f59d89681b3dbb18e172562de88f112577d0aebc
+  sha1: sha256:8cbb25b5a9290963c61270ab65968c0e1ba1b748bd4282e1b4cf375e95442337
+  packages:
+  - azure-blobstore-backup-restorer
+- name: database-backup-restorer
+  version: 3a3bb77fbaa6415535b3e0bad6ab83bfc70efbac9cbcf3a8166b1eaef58dd2a9
+  fingerprint: 3a3bb77fbaa6415535b3e0bad6ab83bfc70efbac9cbcf3a8166b1eaef58dd2a9
+  sha1: sha256:4e9a9b3c679e0eb18d0d465ad293c568a906f698c1dd7fc61acd77dc7549c6a8
+  packages:
+  - database-backup-restorer
+  - database-backup-restorer-postgres-13
+  - database-backup-restorer-postgres-15
+  - database-backup-restorer-postgres-16
+  - database-backup-restorer-mariadb
+  - database-backup-restorer-mysql-8.0
+  - database-backup-restorer-mysql-5.7
+- name: gcs-blobstore-backup-restorer
+  version: e1dbf2608c7c2eebb5cc2a2f6b4a3499ed423f24f28337a125dce15df42e5bef
+  fingerprint: e1dbf2608c7c2eebb5cc2a2f6b4a3499ed423f24f28337a125dce15df42e5bef
+  sha1: sha256:4f5630350240e0402917ff0afc5bef2c0a19b76cd20b24efce44629f20bfae2c
+  packages:
+  - gcs-blobstore-backup-restorer
+- name: s3-unversioned-blobstore-backup-restorer
+  version: 93bbadecdc56cb2cf8d577159dca3e9c813956936d0c80211b98c99ab9aaa59c
+  fingerprint: 93bbadecdc56cb2cf8d577159dca3e9c813956936d0c80211b98c99ab9aaa59c
+  sha1: sha256:56725d4007a22dd4da1048cdbefe9fd4a02b77f49af1d711d70fad6320f0c676
+  packages:
+  - s3-blobstore-backup-restorer
+- name: s3-versioned-blobstore-backup-restorer
+  version: ac5b9c179835f9d0a0d064dbde502d2ef891dd77bdeb89f16fe78b6394b1ee51
+  fingerprint: ac5b9c179835f9d0a0d064dbde502d2ef891dd77bdeb89f16fe78b6394b1ee51
+  sha1: sha256:f4c98bdf199721552568ce7b2fecead128d6b42ab7e39ab56903dfc707eea827
+  packages:
+  - s3-blobstore-backup-restorer
+packages:
+- name: azure-blobstore-backup-restorer
+  version: 4d0913dff07f5d412201b3764a259f589f467ef3043e4811114762781377eead
+  fingerprint: 4d0913dff07f5d412201b3764a259f589f467ef3043e4811114762781377eead
+  sha1: sha256:fdef4a34a7121403dd20b33ba2b4e77b20f99366bf454ac9021cacc9e11fa635
+  dependencies:
+  - golang-1-linux
+- name: database-backup-restorer
+  version: d0abf2f70a9339ab7307a3828fce6594f73476de8aadf14d5704bbcd60dfa45f
+  fingerprint: d0abf2f70a9339ab7307a3828fce6594f73476de8aadf14d5704bbcd60dfa45f
+  sha1: sha256:6f2c6f7e8dd05679db9db844e0232deb7afcae685f48604f87dc3f68086517cd
+  dependencies:
+  - golang-1-linux
+- name: database-backup-restorer-boost
+  version: 05f72399bdd8d91643f42ac411ba65befb78ac0334484dbc3ca95c5286ab7680
+  fingerprint: 05f72399bdd8d91643f42ac411ba65befb78ac0334484dbc3ca95c5286ab7680
+  sha1: sha256:b228c7a30024fafd1d9769695ef1c8e1f417d273293da15a5cc306d25ef89fc5
+  dependencies: []
+- name: database-backup-restorer-mariadb
+  version: e842ead2345f7a4611f6f903429987071a753b395346125c86d412584a234194
+  fingerprint: e842ead2345f7a4611f6f903429987071a753b395346125c86d412584a234194
+  sha1: sha256:94c83b7be95ab8581fe770d7c9d1563a1bc4ed1cf75bcfa573c7084059b65da4
+  dependencies:
+  - libpcre2
+- name: database-backup-restorer-mysql-5.7
+  version: 58c0d2d885422d79f127870a96323180678b7bd1f7febab89cee23b1c0b637b9
+  fingerprint: 58c0d2d885422d79f127870a96323180678b7bd1f7febab89cee23b1c0b637b9
+  sha1: sha256:b0a7a7dd0a2970f9fc6b14a5519a2d7a463c832d1b47280ea1855751c3a0722b
+  dependencies:
+  - database-backup-restorer-boost
+- name: database-backup-restorer-mysql-8.0
+  version: 89a18bffd917a7edd1b96aa8e806ae9e469d678b7631e7776a3f73f7ec53b1f6
+  fingerprint: 89a18bffd917a7edd1b96aa8e806ae9e469d678b7631e7776a3f73f7ec53b1f6
+  sha1: sha256:d83fb8a6a573bf4b59c6d179e13d1ed96fb3eb06dce1a2c03cced8f679a8519e
+  dependencies: []
+- name: database-backup-restorer-postgres-13
+  version: 8fe3ae54de5e9dfcd5246a73717404b9054394c51304e488df64ff0a01ee61b6
+  fingerprint: 8fe3ae54de5e9dfcd5246a73717404b9054394c51304e488df64ff0a01ee61b6
+  sha1: sha256:887d1d336b04aef15e90d57b5e64df356ebcd32a0c4c15bea036f083ab7f6df8
+  dependencies: []
+- name: database-backup-restorer-postgres-15
+  version: 3cd9eb77945198428371b95864be62ce93d4c38f7df7e64272ad58112de301eb
+  fingerprint: 3cd9eb77945198428371b95864be62ce93d4c38f7df7e64272ad58112de301eb
+  sha1: sha256:a070ab1104a66e7acffa7e0c52a6889eb22c0577edeb248fa28a3bb683851fbd
+  dependencies: []
+- name: database-backup-restorer-postgres-16
+  version: 0f7a70c489ad66cc581db8b7b1f12eb0d895c3a538a590235e42ebbbeed2f754
+  fingerprint: 0f7a70c489ad66cc581db8b7b1f12eb0d895c3a538a590235e42ebbbeed2f754
+  sha1: sha256:e17a727e526bfb9a292e307b28cd8cbe67bfd1e8e5ab94c8b1b2234c070a0b4e
+  dependencies: []
+- name: gcs-blobstore-backup-restorer
+  version: 831cd2fa8f504b89457148f1564120e732054414888eda792dd6c5e1995211be
+  fingerprint: 831cd2fa8f504b89457148f1564120e732054414888eda792dd6c5e1995211be
+  sha1: sha256:81dd22ca4c18fac2fd98ba90fd9e10682c8d2af054c40507bebc074abf48a033
+  dependencies:
+  - golang-1-linux
+- name: golang-1-linux
+  version: 8f8128af0520fb107b1f9b339fc0a79eeb70b16efd67bc0e41d770798b2543e0
+  fingerprint: 8f8128af0520fb107b1f9b339fc0a79eeb70b16efd67bc0e41d770798b2543e0
+  sha1: sha256:f339b17b80583082ec7f4dfe7b4e511dcfda4b66e365ebe0e5a95679ee79478a
+  dependencies: []
+- name: libpcre2
+  version: d78b2b2f4a94258de9debb3e11bb4334fbd45d69b14cf6a490eb26323c6b6cad
+  fingerprint: d78b2b2f4a94258de9debb3e11bb4334fbd45d69b14cf6a490eb26323c6b6cad
+  sha1: sha256:5372ec6ffa0ef6313ce93896406d98af502d8f69034c7e96f4d7650826a8dafb
+  dependencies: []
+- name: s3-blobstore-backup-restorer
+  version: 23173e0684232e472c7b3695300895b5189864a745c95854971bd3bdeac92f1b
+  fingerprint: 23173e0684232e472c7b3695300895b5189864a745c95854971bd3bdeac92f1b
+  sha1: sha256:1b74d52b139be180e5b5a575031dbb849cd2eb41fc0ccf8468ff770dffed31af
+  dependencies:
+  - golang-1-linux
+license:
+  version: de99d82e62206a72c9e0b838fa79b9b67a7b1cef5b82eaf5037fba3bc6117c21
+  fingerprint: de99d82e62206a72c9e0b838fa79b9b67a7b1cef5b82eaf5037fba3bc6117c21
+  sha1: sha256:66daaae31d1a514da5aee35f036f063a5c76f4719d0e7c4bd0f0531bb7112100

releases/backup-and-restore-sdk/index.yml

@@ -87,6 +87,8 @@ builds:
     version: 1.18.72
   3764ea5d-979c-477f-7ba7-26dbda1467d6:
     version: 1.18.26
+  37d668ea-d32c-4e63-6d42-f2f62c451126:
+    version: 1.19.50
   38218638-8631-4b1d-6af0-8ac48d346a53:
     version: 1.15.0
   3837934e-2e63-48fb-43de-9a76665ba6a2: