Merge branch 'main' into fds-improve-multipartitioners-checks

adap · Dec 4, 2024 · 46471fe · 46471fe
2 parents 2514e29 + b57e9e7
commit 46471fe
Show file tree

Hide file tree

Showing 362 changed files with 81,262 additions and 57,801 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,16 +1,16 @@
 # Details: https://help.github.com/en/articles/about-code-owners
 
 # Default code owners
-* @danieljanes @tanertopal
+* @tanertopal @danieljanes
 
 # README.md
 README.md @jafermarq @tanertopal @danieljanes
 
 # Flower Baselines
-/baselines @jafermarq @danieljanes
+/baselines @jafermarq @tanertopal @danieljanes
 
 # Flower Benchmarks
-/benchmarks @jafermarq @danieljanes
+/benchmarks @jafermarq @tanertopal @danieljanes
 
 # Flower Datasets
 /datasets @jafermarq @tanertopal @danieljanes

diff --git a/.github/workflows/_docker-build.yml b/.github/workflows/_docker-build.yml
@@ -79,7 +79,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
         with:
           images: ${{ inputs.namespace-repository }}
 
@@ -93,10 +93,10 @@ jobs:
           password: ${{ secrets.dockerhub-token }}
 
       - name: Build and push
-        uses: Wandalen/wretry.action@6feedb7dedadeb826de0f45ff482b53b379a7844 # v3.5.0
+        uses: Wandalen/wretry.action@ffdd254f4eaf1562b8a2c66aeaa37f1ff2231179 # v3.7.3
         id: build
         with:
-          action: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
+          action: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
           attempt_limit: 60 # 60 attempts * (9 secs delay + 1 sec retry) = ~10 mins
           attempt_delay: 9000 # 9 secs
           with: |
@@ -139,7 +139,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
         with:
           images: ${{ inputs.namespace-repository }}
           tags: ${{ inputs.tags }}

diff --git a/.github/workflows/docker-build-main.yml b/.github/workflows/docker-build-main.yml
@@ -14,7 +14,7 @@ jobs:
     outputs:
       pip-version: ${{ steps.versions.outputs.pip-version }}
       setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
-      flwr-version-ref: ${{ steps.versions.outputs.flwr-version-ref }}
+      matrix: ${{ steps.versions.outputs.matrix }}
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
@@ -25,21 +25,26 @@ jobs:
         run: |
           echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
           echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
-          echo "flwr-version-ref=git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" >> "$GITHUB_OUTPUT"
+          FLWR_VERSION_REF="git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}"
+          python dev/build-docker-image-matrix.py --flwr-version "${FLWR_VERSION_REF}" --matrix unstable > matrix.json
+          echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
 
   build-docker-base-images:
     name: Build base images
     if: github.repository == 'adap/flower'
     uses: ./.github/workflows/_docker-build.yml
     needs: parameters
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.parameters.outputs.matrix).base }}
     with:
-      namespace-repository: flwr/base
-      file-dir: src/docker/base/ubuntu
+      namespace-repository: ${{ matrix.images.namespace_repository }}
+      file-dir: ${{ matrix.images.file_dir }}
       build-args: |
         PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
         SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
-        FLWR_VERSION_REF=${{ needs.parameters.outputs.flwr-version-ref }}
-      tags: unstable
+        ${{ matrix.images.build_args_encoded }}
+      tags: ${{ matrix.images.tags_encoded }}
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -48,22 +53,15 @@ jobs:
     name: Build binary images
     if: github.repository == 'adap/flower'
     uses: ./.github/workflows/_docker-build.yml
-    needs: build-docker-base-images
+    needs: [parameters, build-docker-base-images]
     strategy:
       fail-fast: false
-      matrix:
-        images: [
-          { repository: "flwr/superlink", file_dir: "src/docker/superlink" },
-          { repository: "flwr/supernode", file_dir: "src/docker/supernode" },
-          { repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
-          { repository: "flwr/superexec", file_dir: "src/docker/superexec" },
-          { repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
-        ]
+      matrix: ${{ fromJson(needs.parameters.outputs.matrix).binary }}
     with:
-      namespace-repository: ${{ matrix.images.repository }}
+      namespace-repository: ${{ matrix.images.namespace_repository }}
       file-dir: ${{ matrix.images.file_dir }}
-      build-args: BASE_IMAGE=unstable
-      tags: unstable
+      build-args: BASE_IMAGE=${{ matrix.images.base_image }}
+      tags: ${{ matrix.images.tags_encoded }}
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/docker-readme.yml b/.github/workflows/docker-readme.yml
@@ -24,7 +24,7 @@ jobs:
             list-files: "json"
             filters: |
               readme:
-                - 'src/docker/**/README.md'
+                - added|modified: 'src/docker/**/README.md'
 
     update:
       if: ${{ needs.collect.outputs.readme_files != '' && toJson(fromJson(needs.collect.outputs.readme_files)) != '[]' }}

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -67,7 +67,7 @@ jobs:
           - connection: insecure
             authentication: client-auth
     name: |
-      SuperExec / 
+      Exec API / 
       Python ${{ matrix.python-version }} /
       ${{ matrix.connection }} / 
       ${{ matrix.authentication }} / 
@@ -102,12 +102,12 @@ jobs:
             python -m pip install "${WHEEL_URL}"
           fi
       - name: >
-          Run SuperExec test / 
+          Run Exec API test / 
           ${{ matrix.connection }} / 
           ${{ matrix.authentication }} / 
           ${{ matrix.engine }} 
         working-directory: e2e/${{ matrix.directory }}
-        run: ./../test_superexec.sh "${{ matrix.connection }}" "${{ matrix.authentication}}" "${{ matrix.engine }}"
+        run: ./../test_exec_api.sh "${{ matrix.connection }}" "${{ matrix.authentication}}" "${{ matrix.engine }}"
 
   frameworks:
     runs-on: ubuntu-22.04
@@ -347,3 +347,59 @@ jobs:
           cd tmp-${{ matrix.framework }}
           flwr build
           flwr install *.fab
+
+  numpy:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    needs: wheel
+    strategy:
+      fail-fast: false
+      matrix:
+        numpy-version: ["1.26"]
+        python-version: ["3.11"]
+        directory: [e2e-bare-auth]
+        connection: [insecure]
+        engine: [deployment-engine, simulation-engine]
+        authentication: [no-auth]
+    name: |
+      NumPy ${{ matrix.numpy-version }} / 
+      Python ${{ matrix.python-version }} /
+      ${{ matrix.connection }} / 
+      ${{ matrix.authentication }} / 
+      ${{ matrix.engine }} 
+    defaults:
+      run:
+        working-directory: e2e/${{ matrix.directory }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Bootstrap
+        uses: ./.github/actions/bootstrap
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-skip: 'true'
+      - name: Install Flower from repo
+        if: ${{ github.repository != 'adap/flower' || github.event.pull_request.head.repo.fork || github.actor == 'dependabot[bot]' }}
+        working-directory: ./
+        run: |
+          if [[ "${{ matrix.engine }}" == "simulation-engine" ]]; then
+            python -m pip install ".[simulation]" "numpy>=${{ matrix.numpy-version }},<2.0"
+          else
+            python -m pip install . "numpy>=${{ matrix.numpy-version }},<2.0"
+          fi
+      - name: Download and install Flower wheel from artifact store
+        if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
+        run: |
+          # Define base URL for wheel file
+          WHEEL_URL="https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}"
+          if [[ "${{ matrix.engine }}" == "simulation-engine" ]]; then
+            python -m pip install "flwr[simulation] @ ${WHEEL_URL}" "numpy>=${{ matrix.numpy-version }},<2.0"
+          else
+            python -m pip install "${WHEEL_URL}" "numpy>=${{ matrix.numpy-version }},<2.0"
+          fi
+      - name: >
+          Run Flower - NumPy 1.26 test / 
+          ${{ matrix.connection }} / 
+          ${{ matrix.authentication }} / 
+          ${{ matrix.engine }} 
+        working-directory: e2e/${{ matrix.directory }}
+        run: ./../test_exec_api.sh "${{ matrix.connection }}" "${{ matrix.authentication}}" "${{ matrix.engine }}"
diff --git a/.github/workflows/framework-release.yml b/.github/workflows/framework-release.yml
@@ -71,7 +71,7 @@ jobs:
 
       - id: matrix
         run: |
-           python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" > matrix.json
+           python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" --matrix stable > matrix.json
            echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
 
   build-base-images:
@@ -86,13 +86,10 @@ jobs:
       namespace-repository: ${{ matrix.images.namespace_repository }}
       file-dir: ${{ matrix.images.file_dir }}
       build-args: |
-        PYTHON_VERSION=${{ matrix.images.python_version }}
         PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
         SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
-        DISTRO=${{ matrix.images.distro.name }}
-        DISTRO_VERSION=${{ matrix.images.distro.version }}
-        FLWR_VERSION=${{ matrix.images.flwr_version }}
-      tags: ${{ matrix.images.tag }}
+        ${{ matrix.images.build_args_encoded }}
+      tags: ${{ matrix.images.tags_encoded }}
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -109,7 +106,7 @@ jobs:
       namespace-repository: ${{ matrix.images.namespace_repository }}
       file-dir: ${{ matrix.images.file_dir }}
       build-args: BASE_IMAGE=${{ matrix.images.base_image }}
-      tags: ${{ matrix.images.tags }}
+      tags: ${{ matrix.images.tags_encoded }}
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
@@ -13,11 +13,10 @@ jobs:
     name: Relase nightly on PyPI
     if: github.repository == 'adap/flower'
     outputs:
-      name: ${{ steps.release.outputs.name }}
-      version: ${{ steps.release.outputs.version }}
       skip: ${{ steps.release.outputs.skip }}
       pip-version: ${{ steps.release.outputs.pip-version }}
       setuptools-version: ${{ steps.release.outputs.setuptools-version }}
+      matrix: ${{ steps.release.outputs.matrix }}
     steps:
       - uses: actions/checkout@v4
       - name: Bootstrap
@@ -33,27 +32,30 @@ jobs:
             echo "skip=true" >> $GITHUB_OUTPUT
           fi
 
-          echo "name=$(poetry version | awk {'print $1'})" >> $GITHUB_OUTPUT
-          echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT
           echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
           echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
 
+          NAME=$(poetry version | awk {'print $1'})
+          VERSION=$(poetry version -s)
+          python dev/build-docker-image-matrix.py --flwr-version "${VERSION}" --matrix nightly --flwr-package "${NAME}" > matrix.json
+          echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
+
   build-docker-base-images:
     name: Build nightly base images
     if: github.repository == 'adap/flower' && needs.release-nightly.outputs.skip != 'true'
     uses: ./.github/workflows/_docker-build.yml
     needs: release-nightly
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.release-nightly.outputs.matrix).base }}
     with:
-      namespace-repository: flwr/base
-      file-dir: src/docker/base/ubuntu
+      namespace-repository: ${{ matrix.images.namespace_repository }}
+      file-dir: ${{ matrix.images.file_dir }}
       build-args: |
         PIP_VERSION=${{ needs.release-nightly.outputs.pip-version }}
         SETUPTOOLS_VERSION=${{ needs.release-nightly.outputs.setuptools-version }}
-        FLWR_VERSION=${{ needs.release-nightly.outputs.version }}
-        FLWR_PACKAGE=${{ needs.release-nightly.outputs.name }}
-      tags: |
-        ${{ needs.release-nightly.outputs.version }}
-        nightly
+        ${{ matrix.images.build_args_encoded }}
+      tags: ${{ matrix.images.tags_encoded }}
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -65,21 +67,12 @@ jobs:
     needs: [release-nightly, build-docker-base-images]
     strategy:
       fail-fast: false
-      matrix:
-        images: [
-          { repository: "flwr/superlink", file_dir: "src/docker/superlink" },
-          { repository: "flwr/supernode", file_dir: "src/docker/supernode" },
-          { repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
-          { repository: "flwr/superexec", file_dir: "src/docker/superexec" },
-          { repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
-        ]
+      matrix: ${{ fromJson(needs.release-nightly.outputs.matrix).binary }}
     with:
-      namespace-repository: ${{ matrix.images.repository }}
+      namespace-repository: ${{ matrix.images.namespace_repository }}
       file-dir: ${{ matrix.images.file_dir }}
-      build-args: BASE_IMAGE=${{ needs.release-nightly.outputs.version }}
-      tags: |
-        ${{ needs.release-nightly.outputs.version }}
-        nightly
+      build-args: BASE_IMAGE=${{ matrix.images.base_image }}
+      tags: ${{ matrix.images.tags_encoded }}
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/update_translations.yml b/.github/workflows/update_translations.yml
@@ -12,11 +12,17 @@ jobs:
       contents: write
       pull-requests: write
     env:
-      branch-name: auto-update-trans-text
+      base-branch: main # The base branch for the PR
     name: Update text
     steps:
       - uses: actions/checkout@v4
 
+      - name: Generate unique branch name
+        id: generate_branch
+        run: |
+          export BRANCH_NAME="auto-update-trans-text-$(date +'%Y%m%d-%H%M%S')"
+          echo "branch-name=$BRANCH_NAME" >> $GITHUB_ENV
+
       - name: Bootstrap
         uses: ./.github/actions/bootstrap
         with:
@@ -65,14 +71,15 @@ jobs:
         uses: ad-m/github-push-action@master
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          branch: '${{ env.branch-name }}'
+          branch: ${{ env.branch-name }}
 
       - name: Create Pull Request
         if: steps.calculate_diff.outputs.additions > 228 && steps.calculate_diff.outputs.deletions > 60
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          branch: '${{ env.branch-name }}'
+          branch: ${{ env.branch-name }}
+          base: ${{ env.base-branch }}
           delete-branch: true
           title: 'docs(framework:skip) Update source texts for translations (automated)'
           body: 'This PR is auto-generated to update text and language files.'

diff --git a/.gitignore b/.gitignore
@@ -17,6 +17,9 @@ examples/**/dataset/**
 # Flower Baselines
 baselines/datasets/leaf
 
+# Exclude ee package
+src/py/flwr/ee
+
 # macOS
 .DS_Store
 
@@ -183,3 +186,6 @@ app/src/main/assets
 /captures
 .externalNativeBuild
 .cxx
+
+# Pyright
+pyrightconfig.json
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# Flower: A Friendly Federated Learning Framework
+# Flower: A Friendly Federated AI Framework
 
 <p align="center">
   <a href="https://flower.ai/">
@@ -21,7 +21,7 @@
 [![Docker Hub](https://img.shields.io/badge/Docker%20Hub-flwr-blue)](https://hub.docker.com/u/flwr)
 [![Slack](https://img.shields.io/badge/Chat-Slack-red)](https://flower.ai/join-slack)
 
-Flower (`flwr`) is a framework for building federated learning systems. The
+Flower (`flwr`) is a framework for building federated AI systems. The
 design of Flower is based on a few guiding principles:
 
 - **Customizable**: Federated learning systems vary wildly from one use case to