ci: Adding S3 storage

Igalia · Dec 17, 2024 · e48c793 · e48c793
1 parent eab13f5
commit e48c793
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 7 deletions.
diff --git a/.github/actions/bitbake-build/action.yml b/.github/actions/bitbake-build/action.yml
@@ -24,18 +24,41 @@ runs:
       shell: bash
       run:  |
         cd ~/yocto-meta-wpe-image-${{ inputs.repo_release }}
-        rm -rf tmp
         ulimit -n 4096
-        #source sources/meta-wpe-image/scripts/setup-environment ${{ inputs.bitbake_source }}
-        source setup-environment ${{ inputs.bitbake_source }}
+        source sources/meta-wpe-image/scripts/setup-environment ${{ inputs.bitbake_source }}
+        MAX_RETRIES=5
+        export ${{ inputs.bitbake_prefix }}
+        for i in $(seq 1 $MAX_RETRIES); do
+            echo "Attempt $i to fetch sources for $IMAGE..."
+            bitbake ${{ inputs.bitbake_target }} --runall=fetch
+
+            if bitbake -c fetch ${{ inputs.bitbake_target }} --dry-run | grep -q "already fetched"; then
+                echo "All sources fetched successfully."
+                break
+            fi
+
+            if [ $i -lt $MAX_RETRIES ]; then
+                echo "Retrying fetch process..."
+            else
+                echo "Max retries reached. Some sources may still be missing."
+            fi
+        done
         ${{ inputs.bitbake_prefix }} bitbake ${{ inputs.bitbake_target }}
+    - name: Generate timestamped prefix
+      id: timestamp
+      shell: bash
+      run: |
+        TIMESTAMP=$(date +"%Y%m%d%H%M")
+        echo "prefix=${TIMESTAMP}" >> $GITHUB_OUTPUT
     - name: Artifacts
       if: always()
       shell: bash
-      run:  |
-        ls -lh ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/*/tmp/deploy/images/*/
+      run: |
+        ls -lh ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/build-${{ inputs.bitbake_target }}/tmp/deploy/images/${{ inputs.bitbake_target }}/
+        PREFIX=${{ steps.timestamp.outputs.prefix }}
+        s3cmd put --recursive ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/build-${{ inputs.bitbake_target }}/tmp/deploy/images/${{ inputs.bitbake_target }}  s3://yocto/${PREFIX}-${{ inputs.bitbake_target }}
     - name: Clean the tmp dir
       if: always()
       shell: bash
       run:  |
-        rm -rf ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/*/tmp
+        rm -rf ~/yocto-meta-wpe-image-${{ inputs.repo_release }}/builds/build-${{ inputs.bitbake_target }}/tmp
diff --git a/.github/actions/s3-configure/action.yml b/.github/actions/s3-configure/action.yml
@@ -0,0 +1,28 @@
+inputs:
+  access_key:
+    description: "S3 Access Key"
+    required: true
+  secret_key:
+    description: "S3 Secret Key"
+    required: true
+  host_base:
+    description: "S3 Host base"
+    required: false
+    default: "ams3.digitaloceanspaces.com"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Configure s3cmd
+      shell: bash
+      run: |
+        cat > ~/.s3cfg <<EOL
+        [default]
+        access_key = ${{ inputs.access_key }}
+        secret_key = ${{ inputs.secret_key }}
+        host_base = ${{ inputs.host_base }}
+        host_bucket = %(bucket)s.${{ inputs.host_base }}
+        use_https = True
+        gpg_command = /usr/bin/gpg
+        EOL
+
diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml
@@ -6,6 +6,11 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/s3-configure
+        with:
+          access_key: ${{ secrets.S3_ACCESS_KEY }}
+          secret_key: ${{ secrets.S3_SECRET_KEY }}
+          host_base: ${{ secrets.S3_HOST_BASE }}
       - uses: ./.github/actions/bitbake-repo
         with:
           repo_release: 'scarthgap'
@@ -25,7 +30,7 @@ jobs:
       #- uses: actions/checkout@v3
       - uses: ./.github/actions/bitbake-build
         with:
-          bitbake_source: 'raspberrypi3-mesa-wpe-${{ matrix.wpe_vers }} ${{ matrix.machine }} poky-wayland layers.raspberrypi conf.wpe-${{ matrix.wpe_vers }} --update-config'
+          bitbake_source: 'build-${{ matrix.machine }} ${{ matrix.machine }} poky-wayland layers.raspberrypi conf.wpe-${{ matrix.wpe_vers }} --update-config'
           repo_release: ${{ matrix.yocto_rel }}
 
   # scarthgap-raspberrypi3-mesa-weston-wpe-nightly: