diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 7e0910c449e9..8c635c516450 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,16 +1,16 @@
# Details: https://help.github.com/en/articles/about-code-owners
# Default code owners
-* @danieljanes @tanertopal
+* @tanertopal @danieljanes
# README.md
README.md @jafermarq @tanertopal @danieljanes
# Flower Baselines
-/baselines @jafermarq @danieljanes
+/baselines @jafermarq @tanertopal @danieljanes
# Flower Benchmarks
-/benchmarks @jafermarq @danieljanes
+/benchmarks @jafermarq @tanertopal @danieljanes
# Flower Datasets
/datasets @jafermarq @tanertopal @danieljanes
diff --git a/.github/workflows/_docker-build.yml b/.github/workflows/_docker-build.yml
index b5c27c9b4834..4528baea9536 100644
--- a/.github/workflows/_docker-build.yml
+++ b/.github/workflows/_docker-build.yml
@@ -79,7 +79,7 @@ jobs:
- name: Extract metadata (tags, labels) for Docker
id: meta
- uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+ uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
with:
images: ${{ inputs.namespace-repository }}
@@ -93,10 +93,10 @@ jobs:
password: ${{ secrets.dockerhub-token }}
- name: Build and push
- uses: Wandalen/wretry.action@6feedb7dedadeb826de0f45ff482b53b379a7844 # v3.5.0
+ uses: Wandalen/wretry.action@ffdd254f4eaf1562b8a2c66aeaa37f1ff2231179 # v3.7.3
id: build
with:
- action: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
+ action: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
attempt_limit: 60 # 60 attempts * (9 secs delay + 1 sec retry) = ~10 mins
attempt_delay: 9000 # 9 secs
with: |
@@ -139,7 +139,7 @@ jobs:
- name: Docker meta
id: meta
- uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+ uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
with:
images: ${{ inputs.namespace-repository }}
tags: ${{ inputs.tags }}
diff --git a/.github/workflows/docker-build-main.yml b/.github/workflows/docker-build-main.yml
index 81ef845eae29..38a9cd56942b 100644
--- a/.github/workflows/docker-build-main.yml
+++ b/.github/workflows/docker-build-main.yml
@@ -14,7 +14,7 @@ jobs:
outputs:
pip-version: ${{ steps.versions.outputs.pip-version }}
setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
- flwr-version-ref: ${{ steps.versions.outputs.flwr-version-ref }}
+ matrix: ${{ steps.versions.outputs.matrix }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -25,21 +25,26 @@ jobs:
run: |
echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
- echo "flwr-version-ref=git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" >> "$GITHUB_OUTPUT"
+ FLWR_VERSION_REF="git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}"
+ python dev/build-docker-image-matrix.py --flwr-version "${FLWR_VERSION_REF}" --matrix unstable > matrix.json
+ echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-docker-base-images:
name: Build base images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
needs: parameters
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJson(needs.parameters.outputs.matrix).base }}
with:
- namespace-repository: flwr/base
- file-dir: src/docker/base/ubuntu
+ namespace-repository: ${{ matrix.images.namespace_repository }}
+ file-dir: ${{ matrix.images.file_dir }}
build-args: |
PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
- FLWR_VERSION_REF=${{ needs.parameters.outputs.flwr-version-ref }}
- tags: unstable
+ ${{ matrix.images.build_args_encoded }}
+ tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -48,22 +53,15 @@ jobs:
name: Build binary images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
- needs: build-docker-base-images
+ needs: [parameters, build-docker-base-images]
strategy:
fail-fast: false
- matrix:
- images: [
- { repository: "flwr/superlink", file_dir: "src/docker/superlink" },
- { repository: "flwr/supernode", file_dir: "src/docker/supernode" },
- { repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
- { repository: "flwr/superexec", file_dir: "src/docker/superexec" },
- { repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
- ]
+ matrix: ${{ fromJson(needs.parameters.outputs.matrix).binary }}
with:
- namespace-repository: ${{ matrix.images.repository }}
+ namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
- build-args: BASE_IMAGE=unstable
- tags: unstable
+ build-args: BASE_IMAGE=${{ matrix.images.base_image }}
+ tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/docker-readme.yml b/.github/workflows/docker-readme.yml
index 29dd787d638e..9e156e835056 100644
--- a/.github/workflows/docker-readme.yml
+++ b/.github/workflows/docker-readme.yml
@@ -24,7 +24,7 @@ jobs:
list-files: "json"
filters: |
readme:
- - 'src/docker/**/README.md'
+ - added|modified: 'src/docker/**/README.md'
update:
if: ${{ needs.collect.outputs.readme_files != '' && toJson(fromJson(needs.collect.outputs.readme_files)) != '[]' }}
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 012f584561ac..0f462d9a49da 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -67,7 +67,7 @@ jobs:
- connection: insecure
authentication: client-auth
name: |
- SuperExec /
+ Exec API /
Python ${{ matrix.python-version }} /
${{ matrix.connection }} /
${{ matrix.authentication }} /
@@ -102,12 +102,12 @@ jobs:
python -m pip install "${WHEEL_URL}"
fi
- name: >
- Run SuperExec test /
+ Run Exec API test /
${{ matrix.connection }} /
${{ matrix.authentication }} /
${{ matrix.engine }}
working-directory: e2e/${{ matrix.directory }}
- run: ./../test_superexec.sh "${{ matrix.connection }}" "${{ matrix.authentication}}" "${{ matrix.engine }}"
+ run: ./../test_exec_api.sh "${{ matrix.connection }}" "${{ matrix.authentication}}" "${{ matrix.engine }}"
frameworks:
runs-on: ubuntu-22.04
@@ -347,3 +347,59 @@ jobs:
cd tmp-${{ matrix.framework }}
flwr build
flwr install *.fab
+
+ numpy:
+ runs-on: ubuntu-22.04
+ timeout-minutes: 10
+ needs: wheel
+ strategy:
+ fail-fast: false
+ matrix:
+ numpy-version: ["1.26"]
+ python-version: ["3.11"]
+ directory: [e2e-bare-auth]
+ connection: [insecure]
+ engine: [deployment-engine, simulation-engine]
+ authentication: [no-auth]
+ name: |
+ NumPy ${{ matrix.numpy-version }} /
+ Python ${{ matrix.python-version }} /
+ ${{ matrix.connection }} /
+ ${{ matrix.authentication }} /
+ ${{ matrix.engine }}
+ defaults:
+ run:
+ working-directory: e2e/${{ matrix.directory }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Bootstrap
+ uses: ./.github/actions/bootstrap
+ with:
+ python-version: ${{ matrix.python-version }}
+ poetry-skip: 'true'
+ - name: Install Flower from repo
+ if: ${{ github.repository != 'adap/flower' || github.event.pull_request.head.repo.fork || github.actor == 'dependabot[bot]' }}
+ working-directory: ./
+ run: |
+ if [[ "${{ matrix.engine }}" == "simulation-engine" ]]; then
+ python -m pip install ".[simulation]" "numpy>=${{ matrix.numpy-version }},<2.0"
+ else
+ python -m pip install . "numpy>=${{ matrix.numpy-version }},<2.0"
+ fi
+ - name: Download and install Flower wheel from artifact store
+ if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
+ run: |
+ # Define base URL for wheel file
+ WHEEL_URL="https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}"
+ if [[ "${{ matrix.engine }}" == "simulation-engine" ]]; then
+ python -m pip install "flwr[simulation] @ ${WHEEL_URL}" "numpy>=${{ matrix.numpy-version }},<2.0"
+ else
+ python -m pip install "${WHEEL_URL}" "numpy>=${{ matrix.numpy-version }},<2.0"
+ fi
+ - name: >
+ Run Flower - NumPy 1.26 test /
+ ${{ matrix.connection }} /
+ ${{ matrix.authentication }} /
+ ${{ matrix.engine }}
+ working-directory: e2e/${{ matrix.directory }}
+ run: ./../test_exec_api.sh "${{ matrix.connection }}" "${{ matrix.authentication}}" "${{ matrix.engine }}"
diff --git a/.github/workflows/framework-release.yml b/.github/workflows/framework-release.yml
index e608329872de..6af0c281882b 100644
--- a/.github/workflows/framework-release.yml
+++ b/.github/workflows/framework-release.yml
@@ -71,7 +71,7 @@ jobs:
- id: matrix
run: |
- python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" > matrix.json
+ python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" --matrix stable > matrix.json
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-base-images:
@@ -86,13 +86,10 @@ jobs:
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: |
- PYTHON_VERSION=${{ matrix.images.python_version }}
PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
- DISTRO=${{ matrix.images.distro.name }}
- DISTRO_VERSION=${{ matrix.images.distro.version }}
- FLWR_VERSION=${{ matrix.images.flwr_version }}
- tags: ${{ matrix.images.tag }}
+ ${{ matrix.images.build_args_encoded }}
+ tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -109,7 +106,7 @@ jobs:
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=${{ matrix.images.base_image }}
- tags: ${{ matrix.images.tags }}
+ tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index fcefff300cb7..d1de7bed531e 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -13,11 +13,10 @@ jobs:
name: Relase nightly on PyPI
if: github.repository == 'adap/flower'
outputs:
- name: ${{ steps.release.outputs.name }}
- version: ${{ steps.release.outputs.version }}
skip: ${{ steps.release.outputs.skip }}
pip-version: ${{ steps.release.outputs.pip-version }}
setuptools-version: ${{ steps.release.outputs.setuptools-version }}
+ matrix: ${{ steps.release.outputs.matrix }}
steps:
- uses: actions/checkout@v4
- name: Bootstrap
@@ -33,27 +32,30 @@ jobs:
echo "skip=true" >> $GITHUB_OUTPUT
fi
- echo "name=$(poetry version | awk {'print $1'})" >> $GITHUB_OUTPUT
- echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT
echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
+ NAME=$(poetry version | awk {'print $1'})
+ VERSION=$(poetry version -s)
+ python dev/build-docker-image-matrix.py --flwr-version "${VERSION}" --matrix nightly --flwr-package "${NAME}" > matrix.json
+ echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
+
build-docker-base-images:
name: Build nightly base images
if: github.repository == 'adap/flower' && needs.release-nightly.outputs.skip != 'true'
uses: ./.github/workflows/_docker-build.yml
needs: release-nightly
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJson(needs.release-nightly.outputs.matrix).base }}
with:
- namespace-repository: flwr/base
- file-dir: src/docker/base/ubuntu
+ namespace-repository: ${{ matrix.images.namespace_repository }}
+ file-dir: ${{ matrix.images.file_dir }}
build-args: |
PIP_VERSION=${{ needs.release-nightly.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.release-nightly.outputs.setuptools-version }}
- FLWR_VERSION=${{ needs.release-nightly.outputs.version }}
- FLWR_PACKAGE=${{ needs.release-nightly.outputs.name }}
- tags: |
- ${{ needs.release-nightly.outputs.version }}
- nightly
+ ${{ matrix.images.build_args_encoded }}
+ tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -65,21 +67,12 @@ jobs:
needs: [release-nightly, build-docker-base-images]
strategy:
fail-fast: false
- matrix:
- images: [
- { repository: "flwr/superlink", file_dir: "src/docker/superlink" },
- { repository: "flwr/supernode", file_dir: "src/docker/supernode" },
- { repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
- { repository: "flwr/superexec", file_dir: "src/docker/superexec" },
- { repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
- ]
+ matrix: ${{ fromJson(needs.release-nightly.outputs.matrix).binary }}
with:
- namespace-repository: ${{ matrix.images.repository }}
+ namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
- build-args: BASE_IMAGE=${{ needs.release-nightly.outputs.version }}
- tags: |
- ${{ needs.release-nightly.outputs.version }}
- nightly
+ build-args: BASE_IMAGE=${{ matrix.images.base_image }}
+ tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/update_translations.yml b/.github/workflows/update_translations.yml
index 9419f4aaef25..9a5391a40438 100644
--- a/.github/workflows/update_translations.yml
+++ b/.github/workflows/update_translations.yml
@@ -12,11 +12,17 @@ jobs:
contents: write
pull-requests: write
env:
- branch-name: auto-update-trans-text
+ base-branch: main # The base branch for the PR
name: Update text
steps:
- uses: actions/checkout@v4
+ - name: Generate unique branch name
+ id: generate_branch
+ run: |
+ export BRANCH_NAME="auto-update-trans-text-$(date +'%Y%m%d-%H%M%S')"
+ echo "branch-name=$BRANCH_NAME" >> $GITHUB_ENV
+
- name: Bootstrap
uses: ./.github/actions/bootstrap
with:
@@ -65,14 +71,15 @@ jobs:
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- branch: '${{ env.branch-name }}'
+ branch: ${{ env.branch-name }}
- name: Create Pull Request
if: steps.calculate_diff.outputs.additions > 228 && steps.calculate_diff.outputs.deletions > 60
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
- branch: '${{ env.branch-name }}'
+ branch: ${{ env.branch-name }}
+ base: ${{ env.base-branch }}
delete-branch: true
title: 'docs(framework:skip) Update source texts for translations (automated)'
body: 'This PR is auto-generated to update text and language files.'
diff --git a/.gitignore b/.gitignore
index b0962c2783f0..96789cbf6e00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,9 @@ examples/**/dataset/**
# Flower Baselines
baselines/datasets/leaf
+# Exclude ee package
+src/py/flwr/ee
+
# macOS
.DS_Store
@@ -183,3 +186,6 @@ app/src/main/assets
/captures
.externalNativeBuild
.cxx
+
+# Pyright
+pyrightconfig.json
diff --git a/README.md b/README.md
index 7aa73fe609bb..b5c58c6838f0 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Flower: A Friendly Federated Learning Framework
+# Flower: A Friendly Federated AI Framework
@@ -21,7 +21,7 @@
[](https://hub.docker.com/u/flwr)
[](https://flower.ai/join-slack)
-Flower (`flwr`) is a framework for building federated learning systems. The
+Flower (`flwr`) is a framework for building federated AI systems. The
design of Flower is based on a few guiding principles:
- **Customizable**: Federated learning systems vary wildly from one use case to
diff --git a/baselines/doc/source/conf.py b/baselines/doc/source/conf.py
index a2667dbcf006..9d5d4ea7fc92 100644
--- a/baselines/doc/source/conf.py
+++ b/baselines/doc/source/conf.py
@@ -37,7 +37,7 @@
author = "The Flower Authors"
# The full version, including alpha/beta/rc tags
-release = "1.13.0"
+release = "1.14.0"
# -- General configuration ---------------------------------------------------
diff --git a/baselines/doc/source/how-to-contribute-baselines.rst b/baselines/doc/source/how-to-contribute-baselines.rst
index 429ac714c1aa..5838f489d313 100644
--- a/baselines/doc/source/how-to-contribute-baselines.rst
+++ b/baselines/doc/source/how-to-contribute-baselines.rst
@@ -65,7 +65,7 @@ Flower is known and loved for its usability. Therefore, make sure that your base
flwr run .
# Run the baseline overriding the config
- flwr run . --run-config lr=0.01,num-server-rounds=200
+ flwr run . --run-config "lr=0.01 num-server-rounds=200"
We look forward to your contribution!
\ No newline at end of file
diff --git a/baselines/doc/source/index.rst b/baselines/doc/source/index.rst
index 3a19e74b891e..2ca39776dc8e 100644
--- a/baselines/doc/source/index.rst
+++ b/baselines/doc/source/index.rst
@@ -1,7 +1,7 @@
Flower Baselines Documentation
==============================
-Welcome to Flower Baselines' documentation. `Flower `_ is a friendly federated learning framework.
+Welcome to Flower Baselines' documentation. `Flower `_ is a friendly federated AI framework.
Join the Flower Community
diff --git a/baselines/feddebug/.gitignore b/baselines/feddebug/.gitignore
new file mode 100644
index 000000000000..bd99fc54b9d5
--- /dev/null
+++ b/baselines/feddebug/.gitignore
@@ -0,0 +1 @@
+outputs/
\ No newline at end of file
diff --git a/baselines/feddebug/LICENSE b/baselines/feddebug/LICENSE
new file mode 100644
index 000000000000..d64569567334
--- /dev/null
+++ b/baselines/feddebug/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/baselines/feddebug/README.md b/baselines/feddebug/README.md
new file mode 100644
index 000000000000..ca93fd4b4426
--- /dev/null
+++ b/baselines/feddebug/README.md
@@ -0,0 +1,233 @@
+---
+title: FedDebug Systematic Debugging for Federated Learning Applications
+url: https://dl.acm.org/doi/abs/10.1109/ICSE48619.2023.00053
+labels: [malicious client, debugging, fault localization, image classification, data poisoning]
+dataset: [cifar10, mnist]
+---
+
+# FedDebug: Systematic Debugging for Federated Learning Applications
+
+> [!NOTE]
+> If you use this baseline in your work, please remember to cite the original authors of the paper as well as the Flower paper.
+
+**Paper:** [dl.acm.org/doi/abs/10.1109/ICSE48619.2023.00053](https://dl.acm.org/doi/abs/10.1109/ICSE48619.2023.00053)
+
+**Authors:** [Waris Gill](https://people.cs.vt.edu/waris/) (Virginia Tech, USA), [Ali Anwar](https://cse.umn.edu/cs/ali-anwar) (University of Minnesota Twin Cities, USA), [Muhammad Ali Gulzar](https://people.cs.vt.edu/~gulzar/) (Virginia Tech, USA)
+
+**Abstract:** In Federated Learning (FL), clients independently train local models and share them with a central aggregator to build a global model. Impermissibility to access clients' data and collaborative training make FL appealing for applications with data-privacy concerns, such as medical imaging. However, these FL characteristics pose unprecedented challenges for debugging. When a global model's performance deteriorates, identifying the responsible rounds and clients is a major pain point. Developers resort to trial-and-error debugging with subsets of clients, hoping to increase the global model's accuracy or let future FL rounds retune the model, which are time-consuming and costly.
+We design a systematic fault localization framework, FedDebug, that advances the FL debugging on two novel fronts. First, FedDebug enables interactive debugging of realtime collaborative training in FL by leveraging record and replay techniques to construct a simulation that mirrors live FL. FedDebug's _breakpoint_ can help inspect an FL state (round, client, and global model) and move between rounds and clients' models seamlessly, enabling a fine-grained step-by-step inspection. Second, FedDebug automatically identifies the client(s) responsible for lowering the global model's performance without any testing data and labels---both are essential for existing debugging techniques. FedDebug's strengths come from adapting differential testing in conjunction with neuron activations to determine the client(s) deviating from normal behavior. FedDebug achieves 100% accuracy in finding a single faulty client and 90.3% accuracy in finding multiple faulty clients. FedDebug's interactive debugging incurs 1.2% overhead during training, while it localizes a faulty client in only 2.1% of a round's training time. With FedDebug, we bring effective debugging practices to federated learning, improving the quality and productivity of FL application developers.
+
+
+
+
+
+
+
+
+## About this baseline
+
+**What's implemented:**
+FedDebug is a systematic malicious client(s) localization framework designed to advance debugging in Federated Learning (FL). It enables interactive debugging of real-time collaborative training and automatically identifies clients responsible for lowering global model performance without requiring testing data or labels.
+
+This repository implements the FedDebug technique of localizing malicious client(s) in a generic way, allowing it to be used with various fusion techniques (FedAvg, FedProx) and CNN architectures. You can find the original code of FedDebug [here](https://github.com/SEED-VT/FedDebug).
+
+
+**Flower Datasets:** This baseline integrates `flwr-datasets` and tested on CIFAR-10 and MNIST datasets. The code is designed to work with other datasets as well. You can easily extend the code to work with other datasets by following the Flower dataset guidelines.
+
+
+**Hardware Setup:**
+These experiments were run on a machine with 8 CPU cores and an Nvidia Tesla P40 GPU.
+> [!NOTE]
+> This baseline also contains a smaller CNN model (LeNet) to run all these experiments on a CPU. Furthermore, the experiments are also scaled down to obtain representative results of the FedDebug evaluations.
+
+**Contributors:** Waris Gill ([GitHub Profile](https://github.com/warisgill))
+
+## Experimental Setup
+
+**Task:** Image classification, Malicious/Faulty Client(s) Removal, Debugging and Testing
+
+**Models:** This baseline implements two CNN architectures: LeNet and ResNet. Other CNN models (DenseNet, VGG, etc.) are also supported. Check the `conf/base.yaml` file for more details.
+
+**Dataset:** The datasets are partitioned among clients, and each client participates in the training (cross-silo). However, you can easily extend the code to work in cross-device settings. This baseline uses Dirichlet partitioning to partition the datasets among clients for Non-IID experiments. However, the original paper uses quantity-based imbalance approach ([niid_bench](https://arxiv.org/abs/2102.02079)).
+
+| Dataset | #classes | #clients | partitioning method |
+| :------- | :------: | :------: | :-----------------: |
+| CIFAR-10 | 10 | 10 | IID and Non-IID |
+| MNIST | 10 | 10 | IID and Non-IID |
+
+**FL Training Hyperparameters and FedDebug Configuration:**
+Default training hyperparameters are in `conf/base.yaml`.
+
+## Environment Setup
+
+Experiments are conducted with `Python 3.10.14`. It is recommended to use Python 3.10 for the experiments.
+Check the documentation for the different ways of installing `pyenv`, but one easy way is using the [automatic installer](https://github.com/pyenv/pyenv-installer):
+
+```bash
+curl https://pyenv.run | bash # then, don't forget links to your .bashrc/.zshrc
+```
+
+You can then install any Python version with `pyenv install 3.10.14` Then, in order to use FedDebug baseline, you'd do the following:
+
+```bash
+# cd to your feddebug directory (i.e. where the `pyproject.toml` is)
+pyenv local 3.10.14
+poetry env use 3.10.14 # set that version for poetry
+
+# run this from the same directory as the `pyproject.toml` file is
+poetry install
+poetry shell
+
+# check the python version by running the following command
+python --version # it should be >=3.10.14
+```
+
+This will create a basic Python environment with just Flower and additional packages, including those needed for simulation. Now you are inside your environment (pretty much as when you use `virtualenv` or `conda`).
+
+## Running the Experiments
+
+> [!NOTE]
+> You can run almost any evaluation from the paper by changing the parameters in `conf/base.yaml`. Also, you can change the resources (per client CPU and GPU) in `conf/base.yaml` to speed up the simulation. Please check the Flower simulation guide for more details ([Flower Framework main](https://flower.ai/docs/framework/how-to-run-simulations.html)).
+
+The following command will run the default experimental setting in `conf/base.yaml` (LeNet, MNIST, with a total of 10 clients, where client-0 is malicious). FedDebug will identify client-0 as the malicious client. **The experiment took on average 60 seconds to complete.**
+
+```bash
+python -m feddebug.main device=cpu
+```
+
+Output of the last round will show the FedDebug output as follows:
+
+```log
+...
+[2024-10-24 12:25:48,758][flwr][INFO] - ***FedDebug Output Round 5 ***
+[2024-10-24 12:25:48,758][flwr][INFO] - True Malicious Clients (Ground Truth) = ['0']
+[2024-10-24 12:25:48,758][flwr][INFO] - Total Random Inputs = 10
+[2024-10-24 12:25:48,758][flwr][INFO] - Predicted Malicious Clients = {'0': 1.0}
+[2024-10-24 12:25:48,758][flwr][INFO] - FedDebug Localization Accuracy = 100.0
+[2024-10-24 12:25:49,577][flwr][INFO] - fit progress: (5, 0.00015518503449857236, {'accuracy': 0.978, 'loss': 0.00015518503449857236, 'round': 5}, 39.02993568999227)
+[2024-10-24 12:25:49,577][flwr][INFO] - configure_evaluate: no clients selected, skipping evaluation
+[2024-10-24 12:25:49,577][flwr][INFO] -
+[2024-10-24 12:25:49,577][flwr][INFO] - [SUMMARY]
+...
+
+```
+It predicts the malicious client(s) with 100% accuracy. `Predicted Malicious Clients = {'0': 1.0}` means that client-0 is predicted as the malicious client with 1.0 probability. It will also generate a graph `iid-lenet-mnist.png` as shown below:
+
+
+
+
+
+## FedDebug Diverse Experiment Scenarios
+Next, we demonstrate FedDebug experiments across key scenarios: detecting multiple malicious clients (Section 5-B), running with various models, datasets, and devices (including GPU), and examining how neuron activation thresholds impact localization accuracy (Section 5-C). Understanding these scenarios will help you adapt FedDebug to your specific needs and evaluate any configuration you wish to explore from the paper.
+
+
+### 1. Multiple Malicious Clients
+To test the localization of multiple malicious clients, you can change the `total_malicious_clients`. Total Time Taken: 46.7 seconds.
+
+```bash
+python -m feddebug.main device=cpu total_malicious_clients=2 dataset.name=cifar10
+```
+In this scenario, clients 0 and 1 are now malicious. The output will show the FedDebug output as follows:
+
+```log
+...
+[2024-10-24 12:28:14,125][flwr][INFO] - ***FedDebug Output Round 5 ***
+[2024-10-24 12:28:14,125][flwr][INFO] - True Malicious Clients (Ground Truth) = ['0', '1']
+[2024-10-24 12:28:14,125][flwr][INFO] - Total Random Inputs = 10
+[2024-10-24 12:28:14,125][flwr][INFO] - Predicted Malicious Clients = {'0': 1.0, '1': 1.0}
+[2024-10-24 12:28:14,125][flwr][INFO] - FedDebug Localization Accuracy = 100.0
+[2024-10-24 12:28:15,148][flwr][INFO] - fit progress: (5, 0.003398957598209381, {'accuracy': 0.4151, 'loss': 0.003398957598209381, 'round': 5}, 35.81892481799878)
+[2024-10-24 12:28:15,148][flwr][INFO] - configure_evaluate: no clients selected, skipping evaluation
+[2024-10-24 12:28:15,148][flwr][INFO] -
+[2024-10-24 12:28:15,148][flwr][INFO] - [SUMMARY]
+...
+```
+FedDebug predicts the malicious clients with 100% accuracy. `Predicted Malicious Clients = {'0': 1.0, '1': 1.0}` means that clients 0 and 1 are predicted as the malicious clients with 1.0 probability. It will also generate a graph `iid-lenet-cifar10.png` as shown below:
+
+
+
+
+
+### 2. Changing the Model and Device
+To run the experiments with ResNet and `Cuda` with Non-IID distribution you can run the following command. Total Time Taken: 84 seconds.
+
+> [!NOTE]
+> You can run FedDebug with any *model* list in the `conf/base.yaml` file at line 24. Furthermore, you can quickly add additional models in `feddebug/models.py` at line 47.
+
+
+```bash
+python -m feddebug.main device=cuda model=resnet18 distribution=non_iid
+
+```
+Output
+```log
+...
+[2024-10-24 12:13:40,679][flwr][INFO] - ***FedDebug Output Round 5 ***
+[2024-10-24 12:13:40,679][flwr][INFO] - True Malicious Clients (Ground Truth) = ['0']
+[2024-10-24 12:13:40,679][flwr][INFO] - Total Random Inputs = 10
+[2024-10-24 12:13:40,679][flwr][INFO] - Predicted Malicious Clients = {'0': 1.0}
+[2024-10-24 12:13:40,679][flwr][INFO] - FedDebug Localization Accuracy = 100.0
+[2024-10-24 12:13:41,595][flwr][INFO] - fit progress: (5, 0.000987090128660202, {'accuracy': 0.8528, 'loss': 0.000987090128660202, 'round': 5}, 75.3773579710105)
+[2024-10-24 12:13:41,595][flwr][INFO] - configure_evaluate: no clients selected, skipping evaluation
+[2024-10-24 12:13:41,602][flwr][INFO] -
+[2024-10-24 12:13:41,602][flwr][INFO] - [SUMMARY]
+```
+Following is the graph `non_iid-resnet18-mnist.png` generated by the code:
+
+
+
+
+
+### 3. Threshold Impact on Localization
+You can also test the impact of the neuron activation threshold on localization accuracy. A higher threshold decreases the localization accuracy. Total Time Taken: 84 seconds.
+
+```bash
+python -m feddebug.main device=cuda model=resnet18 feddebug.na_t=0.7
+```
+
+```log
+...
+[2024-10-24 12:21:26,923][flwr][INFO] - ***FedDebug Output Round 2 ***
+[2024-10-24 12:21:26,923][flwr][INFO] - True Malicious Clients (Ground Truth) = ['0']
+[2024-10-24 12:21:26,923][flwr][INFO] - Total Random Inputs = 10
+[2024-10-24 12:21:26,923][flwr][INFO] - Predicted Malicious Clients = {'5': 0.7, '0': 0.3}
+[2024-10-24 12:21:26,923][flwr][INFO] - FedDebug Localization Accuracy = 30.0
+[2024-10-24 12:21:27,773][flwr][INFO] - fit progress: (2, 0.001345307207107544, {'accuracy': 0.9497, 'loss': 0.001345307207107544, 'round': 2}, 31.669926984992344)
+[2024-10-24 12:21:27,773][flwr][INFO] - configure_evaluate: no clients selected, skipping evaluation
+[2024-10-24 12:21:27,773][flwr][INFO] -
+```
+
+Following is the graph `iid-resnet18-mnist.png` generated by the code:
+
+
+
+
+> [!WARNING]
+> FedDebug generates random inputs to localize malicious client(s). Thus, results might vary slightly on each run due to randomness.
+
+
+
+## Limitations and Discussion
+Compared to the current baseline, FedDebug was originally evaluated using only a single round of training. It was not initially tested with Dirichlet partitioning for data distribution, which means it may deliver suboptimal performance under different data distribution settings. Enhancing FedDebug's performance could be achieved by generating more effective random inputs, for example, through the use of Generative Adversarial Networks (GANs).
+
+
+## Application of FedDebug
+We used FedDebug to detect `backdoor attacks` in Federated Learning, resulting in [FedDefender](https://dl.acm.org/doi/10.1145/3617574.3617858). The code is implemented using the Flower Framework in [this repository](https://github.com/warisgill/FedDefender). We plan to adapt FedDefender to Flower baseline guidelines soon.
+
+## Citation
+If you have any questions or feedback, feel free to contact me at `waris@vt.edu`. Please cite FedDebug as follows:
+
+```bibtex
+@inproceedings{gill2023feddebug,
+ title={{Feddebug: Systematic Debugging for Federated Learning Applications}},
+ author={Gill, Waris and Anwar, Ali and Gulzar, Muhammad Ali},
+ booktitle={2023 IEEE/ACM 45th International Conference on Software Engineering (ICSE)},
+ pages={512--523},
+ year={2023},
+ organization={IEEE}
+}
+```
+
+
+
+
diff --git a/baselines/feddebug/_static/feddbug-approach.png b/baselines/feddebug/_static/feddbug-approach.png
new file mode 100644
index 000000000000..046e0ec9ceb9
Binary files /dev/null and b/baselines/feddebug/_static/feddbug-approach.png differ
diff --git a/baselines/feddebug/_static/iid-lenet-cifar10.png b/baselines/feddebug/_static/iid-lenet-cifar10.png
new file mode 100644
index 000000000000..a5e0aa87a3c6
Binary files /dev/null and b/baselines/feddebug/_static/iid-lenet-cifar10.png differ
diff --git a/baselines/feddebug/_static/iid-lenet-mnist.png b/baselines/feddebug/_static/iid-lenet-mnist.png
new file mode 100644
index 000000000000..4414c658a3c2
Binary files /dev/null and b/baselines/feddebug/_static/iid-lenet-mnist.png differ
diff --git a/baselines/feddebug/_static/iid-resnet18-mnist.png b/baselines/feddebug/_static/iid-resnet18-mnist.png
new file mode 100644
index 000000000000..0538ce6413a2
Binary files /dev/null and b/baselines/feddebug/_static/iid-resnet18-mnist.png differ
diff --git a/baselines/feddebug/_static/non_iid-resnet18-mnist.png b/baselines/feddebug/_static/non_iid-resnet18-mnist.png
new file mode 100644
index 000000000000..bda745c75942
Binary files /dev/null and b/baselines/feddebug/_static/non_iid-resnet18-mnist.png differ
diff --git a/baselines/feddebug/feddebug/__init__.py b/baselines/feddebug/feddebug/__init__.py
new file mode 100644
index 000000000000..a5e567b59135
--- /dev/null
+++ b/baselines/feddebug/feddebug/__init__.py
@@ -0,0 +1 @@
+"""Template baseline package."""
diff --git a/baselines/feddebug/feddebug/client.py b/baselines/feddebug/feddebug/client.py
new file mode 100644
index 000000000000..3b3138e63156
--- /dev/null
+++ b/baselines/feddebug/feddebug/client.py
@@ -0,0 +1,45 @@
+"""Define your client class and a function to construct such clients.
+
+Please overwrite `flwr.client.NumPyClient` or `flwr.client.Client` and create a function
+to instantiate your client.
+"""
+
+from logging import INFO
+
+import flwr as fl
+from flwr.common.logger import log
+
+from feddebug.models import train_neural_network
+from feddebug.utils import get_parameters, set_parameters
+
+
+class CNNFlowerClient(fl.client.NumPyClient):
+ """Flower client for training a CNN model."""
+
+ def __init__(self, args):
+ """Initialize the client with the given configuration."""
+ self.args = args
+
+ def fit(self, parameters, config):
+ """Train the model on the local dataset."""
+ nk_client_data_points = len(self.args["client_data_train"])
+ model = self.args["model"]
+
+ set_parameters(model, parameters=parameters)
+ train_dict = train_neural_network(
+ {
+ "lr": config["lr"],
+ "epochs": config["local_epochs"],
+ "batch_size": config["batch_size"],
+ "model": model,
+ "train_data": self.args["client_data_train"],
+ "device": self.args["device"],
+ }
+ )
+
+ parameters = get_parameters(model)
+
+ client_train_dict = {"cid": self.args["cid"]} | train_dict
+
+ log(INFO, "Client %s trained.", self.args["cid"])
+ return parameters, nk_client_data_points, client_train_dict
diff --git a/baselines/feddebug/feddebug/conf/base.yaml b/baselines/feddebug/feddebug/conf/base.yaml
new file mode 100644
index 000000000000..4df403ec3e87
--- /dev/null
+++ b/baselines/feddebug/feddebug/conf/base.yaml
@@ -0,0 +1,61 @@
+# General Configuration
+num_rounds: 5
+num_clients: 10
+clients_per_round: ${num_clients}
+
+# Client Configuration
+client:
+ epochs: 3
+ lr: 0.001
+ batch_size: 256
+
+# Adversarial Settings
+noise_rate: 1
+malicious_clients_ids: [0] # Malicious (also called faulty) client IDs (Ground Truth). Default client 0 is malicious.
+total_malicious_clients: null # For inducing multiple malicious clients in Table-2. e.g., 2 means clients [0, 1] are malicious
+
+# FedDebug Configuration
+feddebug:
+ fast: true # to generate randome inputs faster
+ r_inputs: 10 # number of random inputs to generate
+ na_t: 0.00 # neuron activation threshold
+
+# Model Configuration
+model: lenet # Options: lenet, resnet18, resnet34, resnet50, resnet101, resnet152, densenet121, vgg16
+
+# Dataset Configuration
+distribution: 'iid' # Change to "iid" for iid data distribution. Change it to `non_iid` for non-iid data distribution.
+dataset_channels:
+ cifar10: 3 # RGB
+ mnist: 1
+
+dataset_classes:
+ cifar10: 10
+ mnist: 10
+
+dataset:
+ name: mnist
+ num_classes: ${dataset_classes.${dataset.name}}
+ channels: ${dataset_channels.${dataset.name}}
+
+
+# Device and Resource Configuration
+device: cpu
+total_gpus: 1
+total_cpus: 10
+
+client_resources:
+ num_cpus: 2
+ num_gpus: 0.2 # Note that `num_gpus` is only used when the device is set to `cuda` (i.e., `device = cuda`)
+
+
+# Logging Configuration (Hydra)
+hydra:
+ job_logging:
+ root:
+ level: INFO # Set the job logging level to INFO
+ loggers:
+ flwr:
+ level: INFO
+ accelerate.utils.other:
+ level: ERROR
\ No newline at end of file
diff --git a/baselines/feddebug/feddebug/dataset.py b/baselines/feddebug/feddebug/dataset.py
new file mode 100644
index 000000000000..7964ad4083a4
--- /dev/null
+++ b/baselines/feddebug/feddebug/dataset.py
@@ -0,0 +1,165 @@
+"""Handle basic dataset creation.
+
+In case of PyTorch it should return dataloaders for your dataset (for both the clients
+and the server). If you are using a custom dataset class, this module is the place to
+define it. If your dataset requires to be downloaded (and this is not done
+automatically -- e.g. as it is the case for many dataset in TorchVision) and
+partitioned, please include all those functions and logic in the
+`dataset_preparation.py` module. You can use all those functions from functions/methods
+defined here of course.
+"""
+
+import random
+from logging import INFO
+
+import numpy as np
+from flwr.common.logger import log
+from flwr_datasets import FederatedDataset
+from flwr_datasets.partitioner import DirichletPartitioner, IidPartitioner
+from torch.utils.data import DataLoader
+from torchvision import transforms
+
+from feddebug.utils import create_transform
+
+
+def _add_noise_in_data(my_dataset, noise_rate, num_classes):
+ """Introduce label noise by flipping labels based on the noise rate."""
+
+ def flip_labels(batch):
+ labels = np.array(batch["label"])
+ flip_mask = np.random.rand(len(labels)) < noise_rate
+ indices_to_flip = np.where(flip_mask)[0]
+ if len(indices_to_flip) > 0:
+ new_labels = labels[indices_to_flip].copy()
+ for idx in indices_to_flip:
+ current_label = new_labels[idx]
+ possible_labels = list(range(num_classes))
+ possible_labels.remove(current_label)
+ new_labels[idx] = random.choice(possible_labels)
+
+ labels[indices_to_flip] = new_labels
+ batch["label"] = labels
+ return batch
+
+ noisy_dataset = my_dataset.map(
+ flip_labels, batched=True, batch_size=256, num_proc=8
+ ).with_format("torch")
+ return noisy_dataset
+
+
+def _create_dataloader(my_dataset, batch_size=64, shuffle=True):
+ """Create a DataLoader with applied transformations."""
+ col = "image" if "image" in my_dataset.column_names else "img"
+
+ def apply_transforms(batch):
+ batch["image"] = [transform(img) for img in batch[col]]
+ if col != "image":
+ del batch[col]
+ return batch
+
+ temp_transform = create_transform()
+
+ transform = transforms.Compose(
+ [
+ transforms.Resize((32, 32)),
+ transforms.ToTensor(),
+ temp_transform,
+ ]
+ )
+ transformed_dataset = my_dataset.with_transform(apply_transforms)
+ dataloader = DataLoader(transformed_dataset, batch_size=batch_size, shuffle=shuffle)
+ return dataloader
+
+
+class ClientsAndServerDatasets:
+ """Prepare the clients and server datasets for federated learning."""
+
+ def __init__(self, cfg):
+ self.cfg = cfg
+ self.client_id_to_loader = {}
+ self.server_testloader = None
+ self.clients_and_server_raw_data = None
+
+ self._set_distribution_partitioner()
+ self._load_datasets()
+ self._introduce_label_noise()
+
+ def _set_distribution_partitioner(self):
+ """Set the data distribution partitioner based on configuration."""
+ if self.cfg.distribution == "iid":
+ self.data_dist_partitioner_func = self._iid_data_distribution
+ elif self.cfg.distribution == "non_iid":
+ self.data_dist_partitioner_func = self._dirichlet_data_distribution
+ else:
+ raise ValueError(f"Unknown distribution type: {self.cfg.distribution}")
+
+ def _dirichlet_data_distribution(self, target_label_col: str = "label"):
+ """Partition data using Dirichlet distribution."""
+ partitioner = DirichletPartitioner(
+ num_partitions=self.cfg.num_clients,
+ partition_by=target_label_col,
+ alpha=2,
+ min_partition_size=0,
+ self_balancing=True,
+ )
+ return self._partition_helper(partitioner)
+
+ def _iid_data_distribution(self):
+ """Partition data using IID distribution."""
+ partitioner = IidPartitioner(num_partitions=self.cfg.num_clients)
+ return self._partition_helper(partitioner)
+
+ def _partition_helper(self, partitioner):
+ fds = FederatedDataset(
+ dataset=self.cfg.dataset.name, partitioners={"train": partitioner}
+ )
+ server_data = fds.load_split("test")
+ client2data = {
+ f"{cid}": fds.load_partition(cid) for cid in range(self.cfg.num_clients)
+ }
+ return {"client2data": client2data, "server_data": server_data}
+
+ def _load_datasets(self):
+ """Load and partition the datasets based on the partitioner."""
+ self.clients_and_server_raw_data = self.data_dist_partitioner_func()
+ self._create_client_dataloaders()
+ self.server_testloader = _create_dataloader(
+ self.clients_and_server_raw_data["server_data"],
+ batch_size=512,
+ shuffle=False,
+ )
+
+ def _create_client_dataloaders(self):
+ """Create DataLoaders for each client."""
+ self.client_id_to_loader = {
+ client_id: _create_dataloader(
+ client_data, batch_size=self.cfg.client.batch_size
+ )
+ for client_id, client_data in self.clients_and_server_raw_data[
+ "client2data"
+ ].items()
+ if client_id not in self.cfg.malicious_clients_ids
+ }
+
+ def _introduce_label_noise(self):
+ """Introduce label noise to specified faulty clients."""
+ faulty_client_ids = self.cfg.malicious_clients_ids
+ noise_rate = self.cfg.noise_rate
+ num_classes = self.cfg.dataset.num_classes
+ client2data = self.clients_and_server_raw_data["client2data"]
+
+ for client_id in faulty_client_ids:
+ client_ds = client2data[client_id]
+ noisy_dataset = _add_noise_in_data(client_ds, noise_rate, num_classes)
+ self.client_id_to_loader[client_id] = _create_dataloader(
+ noisy_dataset, batch_size=self.cfg.client.batch_size
+ )
+
+ log(INFO, "** All Malicious Clients are: %s **", faulty_client_ids)
+
+ def get_data(self):
+ """Get the prepared client and server DataLoaders."""
+ return {
+ "server_testdata": self.server_testloader,
+ "client2data": self.client_id_to_loader,
+ }
diff --git a/baselines/feddebug/feddebug/dataset_preparation.py b/baselines/feddebug/feddebug/dataset_preparation.py
new file mode 100644
index 000000000000..0120ab9dfb0f
--- /dev/null
+++ b/baselines/feddebug/feddebug/dataset_preparation.py
@@ -0,0 +1,8 @@
+"""Handle the dataset partitioning and (optionally) complex downloads.
+
+Please add here all the necessary logic to either download, uncompress, pre/post-process
+your dataset (or all of the above). If the desired way of running your baseline is to
+first download the dataset and partition it and then run the experiments, please
+uncomment the lines below and tell us in the README.md (see the "Running the Experiment"
+block) that this file should be executed first.
+"""
diff --git a/baselines/feddebug/feddebug/differential_testing.py b/baselines/feddebug/feddebug/differential_testing.py
new file mode 100644
index 000000000000..d68d07b25090
--- /dev/null
+++ b/baselines/feddebug/feddebug/differential_testing.py
@@ -0,0 +1,220 @@
+"""Fed_Debug Differential Testing."""
+
+import itertools
+import time
+from logging import DEBUG
+
+import torch
+import torch.nn.functional as F
+from flwr.common.logger import log
+
+from feddebug.neuron_activation import get_neurons_activations
+from feddebug.utils import create_transform
+
+
+def _predict_func(model, input_tensor):
+ model.eval()
+ logits = model(input_tensor)
+ preds = torch.argmax(F.log_softmax(logits, dim=1), dim=1)
+ pred = preds.item()
+ return pred
+
+
+class InferenceGuidedInputGenerator:
+ """Generate random inputs based on the feedback from the clients."""
+
+ def __init__(
+ self,
+ clients2models,
+ input_shape,
+ transform_func,
+ k_gen_inputs=10,
+ min_nclients_same_pred=3,
+ time_delta=60,
+ faster_input_generation=False,
+ ):
+ self.clients2models = clients2models
+ self.input_shape = input_shape
+ self.transform = transform_func
+ self.k_gen_inputs = k_gen_inputs
+ self.min_nclients_same_pred = min_nclients_same_pred
+ self.time_delta = time_delta
+ self.faster_input_generation = faster_input_generation
+ self.seed = 0
+
+ def _get_random_input(self):
+ torch.manual_seed(self.seed)
+ self.seed += 1
+ img = torch.rand(self.input_shape)
+ if self.transform:
+ return self.transform(img).unsqueeze(0)
+ return img.unsqueeze(0)
+
+ def _simple_random_inputs(self):
+ start_time = time.time()
+ random_inputs = [self._get_random_input() for _ in range(self.k_gen_inputs)]
+ elapsed_time = time.time() - start_time
+ return random_inputs, elapsed_time
+
+ def _generate_feedback_random_inputs(self):
+ print("Generating feedback-based random inputs")
+ random_inputs = []
+ same_prediction_set = set()
+ start_time = time.time()
+ timeout = 60
+
+ while len(random_inputs) < self.k_gen_inputs:
+ img = self._get_random_input()
+ if self.min_nclients_same_pred > 1:
+ self._append_or_not(img, random_inputs, same_prediction_set)
+ else:
+ random_inputs.append(img)
+
+ if time.time() - start_time > timeout:
+ timeout += 60
+ self.min_nclients_same_pred -= 1
+ print(
+ f">> Timeout: Number of distinct inputs: {len(random_inputs)}, "
+ f"decreasing min_nclients_same_pred "
+ f"to {self.min_nclients_same_pred} "
+ f"and extending timeout to {timeout} seconds"
+ )
+
+ elapsed_time = time.time() - start_time
+ return random_inputs, elapsed_time
+
+ def _append_or_not(self, input_tensor, random_inputs, same_prediction_set):
+ preds = [
+ _predict_func(model, input_tensor) for model in self.clients2models.values()
+ ]
+ for ci1, pred1 in enumerate(preds):
+ seq = {ci1}
+ for ci2, pred2 in enumerate(preds):
+ if ci1 != ci2 and pred1 == pred2:
+ seq.add(ci2)
+
+ seq_str = ",".join(map(str, seq))
+ if (
+ seq_str not in same_prediction_set
+ and len(seq) >= self.min_nclients_same_pred
+ ):
+ same_prediction_set.add(seq_str)
+ random_inputs.append(input_tensor)
+
+ def get_inputs(self):
+ """Return generated random inputs."""
+ if self.faster_input_generation or len(self.clients2models) <= 10:
+ return self._simple_random_inputs()
+ return self._generate_feedback_random_inputs()
+
+
+def _torch_intersection(client2tensors):
+ intersect = torch.ones_like(next(iter(client2tensors.values())), dtype=torch.bool)
+ for temp_t in client2tensors.values():
+ intersect = torch.logical_and(intersect, temp_t)
+ return intersect
+
+
+def _generate_leave_one_out_combinations(clients_ids):
+ """Generate and update all subsets of clients with a specified subset size."""
+ subset_size = len(clients_ids) - 1
+ subsets = [set(sub) for sub in itertools.combinations(clients_ids, subset_size)]
+ return subsets
+
+
+class FaultyClientDetector:
+ """Faulty Client Localization using Neuron Activation."""
+
+ def __init__(self, device):
+ self.leave_1_out_combs = None
+ self.device = device
+
+ def _get_clients_ids_with_highest_common_neurons(self, clients2neurons2boolact):
+ def _count_common_neurons(comb):
+ """Return the number of common neurons.
+
+ In PyTorch, boolean values are treated as integers (True as 1 and False as
+ 0), so summing a tensor of boolean values will give you the count of True
+ values.
+ """
+ c2act = {cid: clients2neurons2boolact[cid] for cid in comb}
+ intersect_tensor = _torch_intersection(c2act)
+ return intersect_tensor.sum().item()
+
+ count_of_common_neurons = [
+ _count_common_neurons(comb) for comb in self.leave_1_out_combs
+ ]
+
+ highest_number_of_common_neurons = max(count_of_common_neurons)
+ val_index = count_of_common_neurons.index(highest_number_of_common_neurons)
+ val_clients_ids = self.leave_1_out_combs[val_index]
+ return val_clients_ids
+
+ def get_client_neurons_activations(self, client2model, input_tensor):
+ """Get neuron activations for each client model."""
+ client2acts = {}
+ for cid, model in client2model.items():
+ model = model.to(self.device)
+ neurons_act = get_neurons_activations(model, input_tensor.to(self.device))
+ client2acts[cid] = neurons_act.cpu()
+ model = model.cpu()
+ input_tensor = input_tensor.cpu()
+ return client2acts
+
+ def get_malicious_clients(self, client2acts, na_t, num_bugs):
+ """Identify potential malicious clients based on neuron activations."""
+ potential_faulty_clients = None
+ all_clients_ids = set(client2acts.keys())
+ self.leave_1_out_combs = _generate_leave_one_out_combinations(all_clients_ids)
+ for _ in range(num_bugs):
+ client2_na = {
+ cid: activations > na_t for cid, activations in client2acts.items()
+ }
+ normal_clients_ids = self._get_clients_ids_with_highest_common_neurons(
+ client2_na
+ )
+
+ potential_faulty_clients = all_clients_ids - normal_clients_ids
+ log(DEBUG, "Malicious clients %s", potential_faulty_clients)
+ self.leave_1_out_combs = _generate_leave_one_out_combinations(
+ all_clients_ids - potential_faulty_clients
+ )
+
+ return potential_faulty_clients
+
+
+def differential_testing_fl_clients(
+ client2model,
+ num_bugs,
+ num_inputs,
+ input_shape,
+ na_threshold,
+ faster_input_generation,
+ device,
+):
+ """Differential Testing for FL Clients."""
+ generate_inputs = InferenceGuidedInputGenerator(
+ clients2models=client2model,
+ input_shape=input_shape,
+ transform_func=create_transform(),
+ k_gen_inputs=num_inputs,
+ min_nclients_same_pred=3,
+ faster_input_generation=faster_input_generation,
+ )
+ selected_inputs, _ = generate_inputs.get_inputs()
+
+ predicted_faulty_clients = []
+ localize = FaultyClientDetector(device)
+
+ # Iterate over each random input tensor to detect malicious clients
+ for input_tensor in selected_inputs:
+ # Get neuron activations for each client model
+ client2acts = localize.get_client_neurons_activations(
+ client2model, input_tensor
+ )
+ # Identify potential malicious clients based on activations and thresholds
+ potential_malicious_clients = localize.get_malicious_clients(
+ client2acts, na_threshold, num_bugs
+ )
+ predicted_faulty_clients.append(potential_malicious_clients)
+ return predicted_faulty_clients
diff --git a/baselines/feddebug/feddebug/main.py b/baselines/feddebug/feddebug/main.py
new file mode 100644
index 000000000000..c3937b7f8a3d
--- /dev/null
+++ b/baselines/feddebug/feddebug/main.py
@@ -0,0 +1,171 @@
+"""Create and connect the building blocks for your experiments; start the simulation.
+
+It includes processioning the dataset, instantiate strategy, specify how the global
+model is going to be evaluated, etc. At the end, this script saves the results.
+"""
+
+import time
+from logging import DEBUG, INFO
+from pathlib import Path
+
+import flwr as fl
+import hydra
+import torch
+from flwr.common import ndarrays_to_parameters
+from flwr.common.logger import log
+from hydra.core.hydra_config import HydraConfig
+
+from feddebug import utils
+from feddebug.client import CNNFlowerClient
+from feddebug.dataset import ClientsAndServerDatasets
+from feddebug.models import initialize_model, test
+from feddebug.strategy import FedAvgWithFedDebug
+
+utils.seed_everything(786)
+
+
+def _fit_metrics_aggregation_fn(metrics):
+ """Aggregate metrics recieved from client."""
+ log(INFO, ">> ------------------- Clients Metrics ------------- ")
+ all_logs = {}
+ for nk_points, metric_d in metrics:
+ cid = int(metric_d["cid"])
+ temp_s = (
+ f' Client {metric_d["cid"]}, Loss Train {metric_d["train_loss"]}, '
+ f'Accuracy Train {metric_d["train_accuracy"]}, data_points = {nk_points}'
+ )
+ all_logs[cid] = temp_s
+
+ # sorted by client id from lowest to highest
+ for k in sorted(all_logs.keys()):
+ log(INFO, all_logs[k])
+ return {"loss": 0.0, "accuracy": 0.0}
+
+
+def run_simulation(cfg):
+ """Run the simulation."""
+ if cfg.total_malicious_clients:
+ cfg.malicious_clients_ids = list(range(cfg.total_malicious_clients))
+
+ cfg.malicious_clients_ids = [f"{c}" for c in cfg.malicious_clients_ids]
+
+ save_path = Path(HydraConfig.get().runtime.output_dir)
+
+ exp_key = utils.set_exp_key(cfg)
+
+ log(INFO, " *********** Starting Experiment: %s ***************", exp_key)
+
+ log(DEBUG, "Simulation Configuration: %s", cfg)
+
+ num_bugs = len(cfg.malicious_clients_ids)
+ ds_prep = ClientsAndServerDatasets(cfg)
+ ds_dict = ds_prep.get_data()
+ server_testdata = ds_dict["server_testdata"]
+
+ round2gm_accs = []
+ round2feddebug_accs = []
+
+ def _create_model():
+ return initialize_model(cfg.model, cfg.dataset)
+
+ def _get_fit_config(server_round):
+ return {
+ "server_round": server_round,
+ "local_epochs": cfg.client.epochs,
+ "batch_size": cfg.client.batch_size,
+ "lr": cfg.client.lr,
+ }
+
+ def _get_client(cid):
+ """Give the new client."""
+ client2data = ds_dict["client2data"]
+
+ args = {
+ "cid": cid,
+ "model": _create_model(),
+ "client_data_train": client2data[cid],
+ "device": torch.device(cfg.device),
+ }
+ client = CNNFlowerClient(args).to_client()
+ return client
+
+ def _eval_gm(server_round, parameters, config):
+ gm_model = _create_model()
+ utils.set_parameters(gm_model, parameters)
+ d_res = test(gm_model, server_testdata, device=cfg.device)
+ round2gm_accs.append(d_res["accuracy"])
+ log(DEBUG, "config: %s", config)
+ return d_res["loss"], {
+ "accuracy": d_res["accuracy"],
+ "loss": d_res["loss"],
+ "round": server_round,
+ }
+
+ def _callback_fed_debug_evaluate_fn(server_round, predicted_malicious_clients):
+ true_malicious_clients = cfg.malicious_clients_ids
+
+ log(INFO, "***FedDebug Output Round %s ***", server_round)
+ log(INFO, "True Malicious Clients (Ground Truth) = %s", true_malicious_clients)
+ log(INFO, "Total Random Inputs = %s", cfg.feddebug.r_inputs)
+ localization_accuracy = utils.calculate_localization_accuracy(
+ true_malicious_clients, predicted_malicious_clients
+ )
+
+ mal_probs = {
+ c: v / cfg.feddebug.r_inputs for c, v in predicted_malicious_clients.items()
+ }
+ log(INFO, "Predicted Malicious Clients = %s", mal_probs)
+ log(INFO, "FedDebug Localization Accuracy = %s", localization_accuracy)
+ round2feddebug_accs.append(localization_accuracy)
+
+ initial_net = _create_model()
+ strategy = FedAvgWithFedDebug(
+ num_bugs=num_bugs,
+ num_inputs=cfg.feddebug.r_inputs,
+ input_shape=server_testdata.dataset[0]["image"].clone().detach().shape,
+ na_t=cfg.feddebug.na_t,
+ device=cfg.device,
+ fast=cfg.feddebug.fast,
+ callback_create_model_fn=_create_model,
+ callback_fed_debug_evaluate_fn=_callback_fed_debug_evaluate_fn,
+ accept_failures=False,
+ fraction_fit=0,
+ fraction_evaluate=0.0,
+ min_fit_clients=cfg.clients_per_round,
+ min_evaluate_clients=0,
+ min_available_clients=cfg.num_clients,
+ initial_parameters=ndarrays_to_parameters(
+ ndarrays=utils.get_parameters(initial_net)
+ ),
+ evaluate_fn=_eval_gm,
+ on_fit_config_fn=_get_fit_config,
+ fit_metrics_aggregation_fn=_fit_metrics_aggregation_fn,
+ )
+
+ server_config = fl.server.ServerConfig(num_rounds=cfg.num_rounds)
+
+ client_app = fl.client.ClientApp(client_fn=_get_client)
+ server_app = fl.server.ServerApp(config=server_config, strategy=strategy)
+
+ fl.simulation.run_simulation(
+ server_app=server_app,
+ client_app=client_app,
+ num_supernodes=cfg.num_clients,
+ backend_config=utils.config_sim_resources(cfg),
+ )
+
+ utils.plot_metrics(round2gm_accs, round2feddebug_accs, cfg, save_path)
+
+ log(INFO, "Training Complete for Experiment: %s", exp_key)
+
+
+@hydra.main(config_path="conf", config_name="base", version_base=None)
+def main(cfg) -> None:
+ """Run the baseline."""
+ start_time = time.time()
+ run_simulation(cfg)
+ log(INFO, "Total Time Taken: %s seconds", time.time() - start_time)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/baselines/feddebug/feddebug/models.py b/baselines/feddebug/feddebug/models.py
new file mode 100644
index 000000000000..1d5509f2620f
--- /dev/null
+++ b/baselines/feddebug/feddebug/models.py
@@ -0,0 +1,146 @@
+"""Define our models, and training and eval functions.
+
+If your model is 100% off-the-shelf (e.g. directly from torchvision without requiring
+modifications) you might be better off instantiating your model directly from the Hydra
+config. In this way, swapping your model for another one can be done without changing
+the python code at all
+"""
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision
+
+
+class LeNet(nn.Module):
+ """LeNet model."""
+
+ def __init__(self, config):
+ super().__init__()
+ self.conv1 = nn.Conv2d(config["channels"], 6, 5)
+ self.pool = nn.MaxPool2d(2, 2)
+ self.conv2 = nn.Conv2d(6, 16, 5)
+ self.fc1 = nn.Linear(16 * 5 * 5, 120)
+ self.fc2 = nn.Linear(120, 84)
+ self.fc3 = nn.Linear(84, config["num_classes"])
+
+ def forward(self, x):
+ """Forward pass."""
+ x = self.pool(F.relu(self.conv1(x)))
+ x = self.pool(F.relu(self.conv2(x)))
+ x = x.view(-1, 16 * 5 * 5)
+ x = F.relu(self.fc1(x))
+ x = F.relu(self.fc2(x))
+ x = self.fc3(x)
+ return x
+
+
+def _get_inputs_labels_from_batch(batch):
+ if "image" in batch:
+ return batch["image"], batch["label"]
+ x, y = batch
+ return x, y
+
+
+def initialize_model(name, cfg):
+ """Initialize the model with the given name."""
+ model_functions = {
+ "resnet18": lambda: torchvision.models.resnet18(weights="IMAGENET1K_V1"),
+ "resnet34": lambda: torchvision.models.resnet34(weights="IMAGENET1K_V1"),
+ "resnet50": lambda: torchvision.models.resnet50(weights="IMAGENET1K_V1"),
+ "resnet101": lambda: torchvision.models.resnet101(weights="IMAGENET1K_V1"),
+ "resnet152": lambda: torchvision.models.resnet152(weights="IMAGENET1K_V1"),
+ "densenet121": lambda: torchvision.models.densenet121(weights="IMAGENET1K_V1"),
+ "vgg16": lambda: torchvision.models.vgg16(weights="IMAGENET1K_V1"),
+ "lenet": lambda: LeNet(
+ {"channels": cfg.channels, "num_classes": cfg.num_classes}
+ ),
+ }
+ model = model_functions[name]()
+ # Modify model for grayscale input if necessary
+ if cfg.channels == 1:
+ if name.startswith("resnet"):
+ model.conv1 = torch.nn.Conv2d(
+ 1, 64, kernel_size=7, stride=2, padding=3, bias=False
+ )
+ elif name == "densenet121":
+ model.features[0] = torch.nn.Conv2d(
+ 1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False
+ )
+ elif name == "vgg16":
+ model.features[0] = torch.nn.Conv2d(
+ 1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)
+ )
+
+ # Modify final layer to match the number of classes
+ if name.startswith("resnet"):
+ num_ftrs = model.fc.in_features
+ model.fc = torch.nn.Linear(num_ftrs, cfg.num_classes)
+ elif name == "densenet121":
+ num_ftrs = model.classifier.in_features
+ model.classifier = torch.nn.Linear(num_ftrs, cfg.num_classes)
+ elif name == "vgg16":
+ num_ftrs = model.classifier[-1].in_features
+ model.classifier[-1] = torch.nn.Linear(num_ftrs, cfg.num_classes)
+
+ return model.cpu()
+
+
+def _train(tconfig):
+ """Train the network on the training set."""
+ trainloader = tconfig["train_data"]
+ net = tconfig["model"]
+ criterion = torch.nn.CrossEntropyLoss()
+ optimizer = torch.optim.Adam(net.parameters(), lr=tconfig["lr"])
+ net = net.to(tconfig["device"]).train()
+ epoch_loss = 0
+ epoch_acc = 0
+ for _epoch in range(tconfig["epochs"]):
+ correct, total, epoch_loss = 0, 0, 0.0
+ for batch in trainloader:
+ images, labels = _get_inputs_labels_from_batch(batch)
+ images, labels = images.to(tconfig["device"]), labels.to(tconfig["device"])
+ optimizer.zero_grad()
+ outputs = net(images)
+ loss = criterion(net(images), labels)
+ loss.backward()
+ optimizer.step()
+ # Metrics
+ epoch_loss += loss.item()
+ total += labels.size(0)
+ correct += (torch.max(outputs.data, 1)[1] == labels).sum().item()
+ images = images.cpu()
+ labels = labels.cpu()
+ # break
+ epoch_loss /= total
+ epoch_acc = correct / total
+ net = net.cpu()
+ return {"train_loss": epoch_loss, "train_accuracy": epoch_acc}
+
+
+def test(net, testloader, device):
+ """Evaluate the network on the entire test set."""
+ criterion = torch.nn.CrossEntropyLoss()
+ correct, total, loss = 0, 0, 0.0
+ net = net.to(device).eval()
+ with torch.no_grad():
+ for batch in testloader:
+ images, labels = _get_inputs_labels_from_batch(batch)
+ images, labels = images.to(device), labels.to(device)
+ outputs = net(images)
+ loss += criterion(outputs, labels).item()
+ _, predicted = torch.max(outputs.data, 1)
+ total += labels.size(0)
+ correct += (predicted == labels).sum().item()
+ images = images.cpu()
+ labels = labels.cpu()
+ loss /= len(testloader.dataset)
+ accuracy = correct / total
+ net = net.cpu()
+ return {"loss": loss, "accuracy": accuracy}
+
+
+def train_neural_network(tconfig):
+ """Train the neural network."""
+ train_dict = _train(tconfig)
+ return train_dict
diff --git a/baselines/feddebug/feddebug/neuron_activation.py b/baselines/feddebug/feddebug/neuron_activation.py
new file mode 100644
index 000000000000..d7dfae49ed6f
--- /dev/null
+++ b/baselines/feddebug/feddebug/neuron_activation.py
@@ -0,0 +1,59 @@
+"""The file contains the code to get the activations of all neurons."""
+
+import torch
+import torch.nn.functional as F
+
+
+class NeuronActivation:
+ """Class to get the activations of all neurons in the model."""
+
+ def __init__(self):
+ self.hooks_storage = []
+
+ def _get_all_layers_in_neural_network(self, net):
+ layers = []
+ for layer in net.children():
+ if len(list(layer.children())) == 0 and isinstance(
+ layer, (torch.nn.Conv2d, torch.nn.Linear)
+ ):
+ layers.append(layer)
+ if len(list(layer.children())) > 0:
+ temp_layers = self._get_all_layers_in_neural_network(layer)
+ layers = layers + temp_layers
+ return layers
+
+ def _get_input_and_output_of_layer(self, layer, input_t, output_t):
+ assert (
+ len(input_t) == 1
+ ), f"Hook, {layer.__class__.__name__} Expected 1 input, got {len(input_t)}"
+ self.hooks_storage.append(output_t.detach())
+
+ def _insert_hooks(self, layers):
+ all_hooks = []
+ for layer in layers:
+ hook = layer.register_forward_hook(self._get_input_and_output_of_layer)
+ all_hooks.append(hook)
+ return all_hooks
+
+ def get_neurons_activations(self, model, img):
+ """Return the activations of model for the given input."""
+ layer2output = []
+ layers = self._get_all_layers_in_neural_network(model)
+ hooks = self._insert_hooks(layers)
+ model(img) # forward pass and everything is stored in hooks_storage
+ for l_id in range(len(layers)):
+ activations = F.relu(self.hooks_storage[l_id]).cpu()
+ layer2output.append(activations)
+ _ = [h.remove() for h in hooks] # remove the hooks
+ self.hooks_storage = []
+ neurons = (
+ torch.cat([out.flatten() for out in layer2output]).flatten().detach().cpu()
+ )
+ return neurons
+
+
+def get_neurons_activations(model, img):
+ """Return the activations of all neurons in the model for the given input image."""
+ model = model.eval()
+ neurons = NeuronActivation()
+ return neurons.get_neurons_activations(model, img)
diff --git a/baselines/feddebug/feddebug/server.py b/baselines/feddebug/feddebug/server.py
new file mode 100644
index 000000000000..2fd7d42cde5a
--- /dev/null
+++ b/baselines/feddebug/feddebug/server.py
@@ -0,0 +1,5 @@
+"""Create global evaluation function.
+
+Optionally, also define a new Server class (please note this is not needed in most
+settings).
+"""
diff --git a/baselines/feddebug/feddebug/strategy.py b/baselines/feddebug/feddebug/strategy.py
new file mode 100644
index 000000000000..3b7533cc9b8a
--- /dev/null
+++ b/baselines/feddebug/feddebug/strategy.py
@@ -0,0 +1,80 @@
+"""Optionally define a custom strategy.
+
+Needed only when the strategy is not yet implemented in Flower or because you want to
+extend or modify the functionality of an existing strategy.
+"""
+
+from collections import Counter
+
+import flwr as fl
+
+from feddebug import utils
+from feddebug.differential_testing import differential_testing_fl_clients
+
+
+class FedAvgWithFedDebug(fl.server.strategy.FedAvg):
+ """FedAvg with Differential Testing."""
+
+ def __init__(
+ self,
+ num_bugs,
+ num_inputs,
+ input_shape,
+ na_t,
+ device,
+ fast,
+ callback_create_model_fn,
+ callback_fed_debug_evaluate_fn,
+ *args,
+ **kwargs,
+ ):
+ """Initialize."""
+ super().__init__(*args, **kwargs)
+ self.input_shape = input_shape
+ self.num_bugs = num_bugs
+ self.num_inputs = num_inputs
+ self.na_t = na_t
+ self.device = device
+ self.fast = fast
+ self.create_model_fn = callback_create_model_fn
+ self.callback_fed_debug_evaluate_fn = callback_fed_debug_evaluate_fn
+
+ def aggregate_fit(self, server_round, results, failures):
+ """Aggregate clients updates."""
+ potential_mal_clients = self._run_differential_testing_helper(results)
+ aggregated_parameters, aggregated_metrics = super().aggregate_fit(
+ server_round, results, failures
+ )
+ aggregated_metrics["potential_malicious_clients"] = potential_mal_clients
+ self.callback_fed_debug_evaluate_fn(server_round, potential_mal_clients)
+ return aggregated_parameters, aggregated_metrics
+
+ def _get_model_from_parameters(self, parameters):
+ """Convert parameters to state_dict."""
+ ndarr = fl.common.parameters_to_ndarrays(parameters)
+ temp_net = self.create_model_fn()
+ utils.set_parameters(temp_net, ndarr)
+ return temp_net
+
+ def _run_differential_testing_helper(self, results):
+ client2model = {
+ fit_res.metrics["cid"]: self._get_model_from_parameters(fit_res.parameters)
+ for _, fit_res in results
+ }
+ predicted_faulty_clients_on_each_input = differential_testing_fl_clients(
+ client2model,
+ self.num_bugs,
+ self.num_inputs,
+ self.input_shape,
+ self.na_t,
+ self.fast,
+ self.device,
+ )
+ mal_clients_dict = Counter(
+ [
+ f"{e}"
+ for temp_set in predicted_faulty_clients_on_each_input
+ for e in temp_set
+ ]
+ )
+ return dict(mal_clients_dict)
diff --git a/baselines/feddebug/feddebug/utils.py b/baselines/feddebug/feddebug/utils.py
new file mode 100644
index 000000000000..4e1cfdb8cbd8
--- /dev/null
+++ b/baselines/feddebug/feddebug/utils.py
@@ -0,0 +1,121 @@
+"""Define any utility function.
+
+They are not directly relevant to the other (more FL specific) python modules. For
+example, you may define here things like: loading a model from a checkpoint, saving
+results, plotting.
+"""
+
+import random
+from logging import INFO
+
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+from flwr.common.logger import log
+from torchvision import transforms
+
+
+def seed_everything(seed=786):
+ """Seed everything."""
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+
+
+def calculate_localization_accuracy(true_faulty_clients, predicted_faulty_clients):
+ """Calculate the fault localization accuracy."""
+ true_preds = 0
+ total = 0
+ for client, predicted_faulty_count in predicted_faulty_clients.items():
+ if client in true_faulty_clients:
+ true_preds += predicted_faulty_count
+
+ total += predicted_faulty_count
+
+ accuracy = (true_preds / total) * 100
+ return accuracy
+
+
+def create_transform():
+ """Create the transform for the dataset."""
+ tfms = transforms.Compose([transforms.Normalize((0.5,), (0.5,))])
+ return tfms
+
+
+def set_exp_key(cfg):
+ """Set the experiment key."""
+ key = (
+ f"{cfg.model}-{cfg.dataset.name}-"
+ f"faulty_clients[{cfg.malicious_clients_ids}]-"
+ f"noise_rate{cfg.noise_rate}-"
+ f"TClients{cfg.num_clients}-"
+ f"-clientsPerR{cfg.clients_per_round})"
+ f"-{cfg.distribution}"
+ f"-batch{cfg.client.batch_size}-epochs{cfg.client.epochs}-"
+ f"lr{cfg.client.lr}"
+ )
+ return key
+
+
+def config_sim_resources(cfg):
+ """Configure the resources for the simulation."""
+ client_resources = {"num_cpus": cfg.client_resources.num_cpus}
+ if cfg.device == "cuda":
+ client_resources["num_gpus"] = cfg.client_resources.num_gpus
+
+ init_args = {"num_cpus": cfg.total_cpus, "num_gpus": cfg.total_gpus}
+ backend_config = {
+ "client_resources": client_resources,
+ "init_args": init_args,
+ }
+ return backend_config
+
+
+def get_parameters(model):
+ """Return model parameters as a list of NumPy ndarrays."""
+ model = model.cpu()
+ return [val.cpu().detach().clone().numpy() for _, val in model.state_dict().items()]
+
+
+def set_parameters(net, parameters):
+ """Set model parameters from a list of NumPy ndarrays."""
+ net = net.cpu()
+ params_dict = zip(net.state_dict().keys(), parameters)
+ new_state_dict = {k: torch.from_numpy(v) for k, v in params_dict}
+ net.load_state_dict(new_state_dict, strict=True)
+
+
+def plot_metrics(gm_accs, feddebug_accs, cfg, save_path):
+ """Plot the metrics with legend and save the plot."""
+ fig, axis = plt.subplots(
+ figsize=(3.5, 2.5)
+ ) # Increase figure size for better readability
+
+ # Convert accuracy to percentages
+ gm_accs = [x * 100 for x in gm_accs][1:]
+
+ # Plot lines with distinct styles for better differentiation
+ axis.plot(gm_accs, label="Global Model", linestyle="-", linewidth=2)
+ axis.plot(feddebug_accs, label="FedDebug", linestyle="--", linewidth=2)
+
+ # Set labels with font settings
+ axis.set_xlabel("Training Round", fontsize=12)
+ axis.set_ylabel("Accuracy (%)", fontsize=12)
+
+ # Set title with font settings
+ title = f"{cfg.distribution}-{cfg.model}-{cfg.dataset.name}"
+ axis.set_title(title, fontsize=12)
+
+ # Set legend with better positioning and font size
+ axis.legend(fontsize=12, loc="lower right", frameon=False)
+ # change the font family to serif and font.serif to Times
+
+ # Tight layout to avoid clipping
+ fig.tight_layout()
+
+ # Save the figure with a higher resolution for publication quality
+ graph_path = save_path / f"{title}.png"
+
+ plt.savefig(graph_path, dpi=300, bbox_inches="tight")
+ plt.close()
+ log(INFO, "Saved plot at %s", graph_path)
diff --git a/baselines/feddebug/pyproject.toml b/baselines/feddebug/pyproject.toml
new file mode 100644
index 000000000000..bbc8f55777ea
--- /dev/null
+++ b/baselines/feddebug/pyproject.toml
@@ -0,0 +1,140 @@
+[build-system]
+requires = ["poetry-core>=1.4.0"]
+build-backend = "poetry.masonry.api"
+
+[tool.poetry]
+name = "feddebug" # <----- Ensure it matches the name of your baseline directory containing all the source code
+version = "1.0.0"
+description = "Flower Baselines"
+license = "Apache-2.0"
+authors = ["The Flower Authors "]
+readme = "README.md"
+homepage = "https://flower.ai"
+repository = "https://github.com/adap/flower"
+documentation = "https://flower.ai"
+classifiers = [
+ "Development Status :: 3 - Alpha",
+ "Intended Audience :: Developers",
+ "Intended Audience :: Science/Research",
+ "License :: OSI Approved :: Apache Software License",
+ "Operating System :: MacOS :: MacOS X",
+ "Operating System :: POSIX :: Linux",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3 :: Only",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Topic :: Scientific/Engineering",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+ "Topic :: Scientific/Engineering :: Mathematics",
+ "Topic :: Software Development",
+ "Topic :: Software Development :: Libraries",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+ "Typing :: Typed",
+]
+
+[tool.poetry.dependencies]
+python = ">=3.8.15, <3.12.0" # don't change this
+flwr = { extras = ["simulation"], version = "1.9.0" }
+hydra-core = "1.3.2" # don't change this
+torch = "2.2.2"
+torchvision = "0.17.2"
+flwr-datasets = "0.3.0"
+
+[tool.poetry.dev-dependencies]
+isort = "==5.13.2"
+black = "==24.2.0"
+docformatter = "==1.7.5"
+mypy = "==1.4.1"
+pylint = "==2.8.2"
+flake8 = "==3.9.2"
+pytest = "==6.2.4"
+pytest-watch = "==4.2.0"
+ruff = "==0.0.272"
+types-requests = "==2.27.7"
+
+[tool.isort]
+line_length = 88
+indent = " "
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+
+[tool.black]
+line-length = 88
+target-version = ["py38", "py39", "py310", "py311"]
+
+[tool.pytest.ini_options]
+minversion = "6.2"
+addopts = "-qq"
+testpaths = [
+ "flwr_baselines",
+]
+
+[tool.mypy]
+ignore_missing_imports = true
+strict = false
+plugins = "numpy.typing.mypy_plugin"
+
+[tool.pylint."MESSAGES CONTROL"]
+disable = "bad-continuation,duplicate-code,too-few-public-methods,useless-import-alias,too-many-locals,too-many-arguments,too-many-instance-attributes"
+good-names = "i,j,k,_,x,y,X,Y"
+signature-mutators = "hydra.main.main"
+
+[tool.pylint.typecheck]
+generated-members = "numpy.*, torch.*, tensorflow.*"
+
+[[tool.mypy.overrides]]
+module = [
+ "importlib.metadata.*",
+ "importlib_metadata.*",
+]
+follow_imports = "skip"
+follow_imports_for_stubs = true
+disallow_untyped_calls = false
+
+[[tool.mypy.overrides]]
+module = "torch.*"
+follow_imports = "skip"
+follow_imports_for_stubs = true
+
+[tool.docformatter]
+wrap-summaries = 88
+wrap-descriptions = 88
+
+[tool.ruff]
+target-version = "py38"
+line-length = 88
+select = ["D", "E", "F", "W", "B", "ISC", "C4"]
+fixable = ["D", "E", "F", "W", "B", "ISC", "C4"]
+ignore = ["B024", "B027"]
+exclude = [
+ ".bzr",
+ ".direnv",
+ ".eggs",
+ ".git",
+ ".hg",
+ ".mypy_cache",
+ ".nox",
+ ".pants.d",
+ ".pytype",
+ ".ruff_cache",
+ ".svn",
+ ".tox",
+ ".venv",
+ "__pypackages__",
+ "_build",
+ "buck-out",
+ "build",
+ "dist",
+ "node_modules",
+ "venv",
+ "proto",
+]
+
+[tool.ruff.pydocstyle]
+convention = "numpy"
diff --git a/benchmarks/flowertune-llm/evaluation/finance/README.md b/benchmarks/flowertune-llm/evaluation/finance/README.md
index b5595433a238..15d2410b8ca4 100644
--- a/benchmarks/flowertune-llm/evaluation/finance/README.md
+++ b/benchmarks/flowertune-llm/evaluation/finance/README.md
@@ -27,6 +27,7 @@ huggingface-cli login
```bash
python eval.py \
+--base-model-name-path=your-base-model-name \ # e.g., mistralai/Mistral-7B-v0.3
--peft-path=/path/to/fine-tuned-peft-model-dir/ \ # e.g., ./peft_1
--run-name=fl \ # specified name for this run
--batch-size=32 \
diff --git a/benchmarks/flowertune-llm/evaluation/finance/benchmarks.py b/benchmarks/flowertune-llm/evaluation/finance/benchmarks.py
index 2b1a174e571f..f2dad1e056b8 100644
--- a/benchmarks/flowertune-llm/evaluation/finance/benchmarks.py
+++ b/benchmarks/flowertune-llm/evaluation/finance/benchmarks.py
@@ -122,7 +122,10 @@ def inference(dataset, model, tokenizer, batch_size):
**tokens, max_length=512, eos_token_id=tokenizer.eos_token_id
)
res_sentences = [tokenizer.decode(i, skip_special_tokens=True) for i in res]
- out_text = [o.split("Answer: ")[1] for o in res_sentences]
+ out_text = [
+ o.split("Answer: ")[1] if len(o.split("Answer: ")) > 1 else "None"
+ for o in res_sentences
+ ]
out_text_list += out_text
torch.cuda.empty_cache()
diff --git a/benchmarks/flowertune-llm/evaluation/general-nlp/README.md b/benchmarks/flowertune-llm/evaluation/general-nlp/README.md
index c3fd71da6ea2..5acd75285dd3 100644
--- a/benchmarks/flowertune-llm/evaluation/general-nlp/README.md
+++ b/benchmarks/flowertune-llm/evaluation/general-nlp/README.md
@@ -27,6 +27,7 @@ huggingface-cli login
```bash
python eval.py \
+--base-model-name-path=your-base-model-name \ # e.g., mistralai/Mistral-7B-v0.3
--peft-path=/path/to/fine-tuned-peft-model-dir/ \ # e.g., ./peft_1
--run-name=fl \ # specified name for this run
--batch-size=16 \
diff --git a/benchmarks/flowertune-llm/evaluation/medical/README.md b/benchmarks/flowertune-llm/evaluation/medical/README.md
index 628489ce8de6..6a519e8a7c54 100644
--- a/benchmarks/flowertune-llm/evaluation/medical/README.md
+++ b/benchmarks/flowertune-llm/evaluation/medical/README.md
@@ -27,6 +27,7 @@ huggingface-cli login
```bash
python eval.py \
+--base-model-name-path=your-base-model-name \ # e.g., mistralai/Mistral-7B-v0.3
--peft-path=/path/to/fine-tuned-peft-model-dir/ \ # e.g., ./peft_1
--run-name=fl \ # specified name for this run
--batch-size=16 \
diff --git a/datasets/README.md b/datasets/README.md
index 25db77233558..0d35d2e31b6a 100644
--- a/datasets/README.md
+++ b/datasets/README.md
@@ -6,7 +6,7 @@

[](https://flower.ai/join-slack)
-Flower Datasets (`flwr-datasets`) is a library to quickly and easily create datasets for federated learning, federated evaluation, and federated analytics. It was created by the `Flower Labs` team that also created Flower: A Friendly Federated Learning Framework.
+Flower Datasets (`flwr-datasets`) is a library to quickly and easily create datasets for federated learning, federated evaluation, and federated analytics. It was created by the `Flower Labs` team that also created Flower: A Friendly Federated AI Framework.
> [!TIP]
diff --git a/datasets/doc/source/conf.py b/datasets/doc/source/conf.py
index dcba63dd221c..92d59d7df370 100644
--- a/datasets/doc/source/conf.py
+++ b/datasets/doc/source/conf.py
@@ -38,7 +38,7 @@
author = "The Flower Authors"
# The full version, including alpha/beta/rc tags
-release = "0.3.0"
+release = "0.4.0"
# -- General configuration ---------------------------------------------------
diff --git a/datasets/doc/source/contributor-how-to-contribute-dataset.rst b/datasets/doc/source/contributor-how-to-contribute-dataset.rst
new file mode 100644
index 000000000000..07a6ba6378f7
--- /dev/null
+++ b/datasets/doc/source/contributor-how-to-contribute-dataset.rst
@@ -0,0 +1,56 @@
+How to contribute a dataset
+===========================
+
+To make a dataset available in Flower Dataset (`flwr-datasets`), you need to add the dataset to `HuggingFace Hub `_ .
+
+This guide will explain the best practices we found when adding datasets ourselves and point to the HFs guides. To see the datasets added by Flower, visit https://huggingface.co/flwrlabs.
+
+Dataset contribution process
+----------------------------
+The contribution contains three steps: first, on your development machine transform your dataset into a ``datasets.Dataset`` object, the preferred format for datasets in HF Hub; second, upload the dataset to HuggingFace Hub and detail it its readme how can be used with Flower Dataset; third, share your dataset with us and we will add it to the `recommended FL dataset list `_
+
+Creating a dataset locally
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+You can create a local dataset directly using the `datasets` library or load it in any custom way and transform it to the `datasets.Dataset` from other Python objects.
+To complete this step, we recommend reading our :doc:`how-to-use-with-local-data` guide or/and the `Create a dataset `_ guide from HF.
+
+.. tip::
+ We recommend that you do not upload custom scripts to HuggingFace Hub; instead, create the dataset locally and upload the data, which will speed up the processing time each time the data set is downloaded.
+
+Contribution to the HuggingFace Hub
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Each dataset in the HF Hub is a Git repository with a specific structure and readme file, and HuggingFace provides an API to push the dataset and, alternatively, a user interface directly in the website to populate the information in the readme file.
+
+Contributions to the HuggingFace Hub come down to:
+
+1. creating an HF repository for the dataset.
+2. uploading the dataset.
+3. filling in the information in the readme file.
+
+To complete this step, follow this HF's guide `Share dataset to the Hub `_.
+
+Note that the push of the dataset is straightforward, and here's what it could look like:
+
+.. code-block:: python
+
+ from datasets import Dataset
+
+ # Example dataset
+ data = {
+ 'column1': [1, 2, 3],
+ 'column2': ['a', 'b', 'c']
+ }
+
+ # Create a Dataset object
+ dataset = Dataset.from_dict(data)
+
+ # Push the dataset to the HuggingFace Hub
+ dataset.push_to_hub("you-hf-username/your-ds-name")
+
+To make the dataset easily accessible in FL we recommend adding the "Use in FL" section. Here's an example of how it is done in `one of our repos `_ for the cinic10 dataset.
+
+Increasing visibility of the dataset
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you want the dataset listed in our `recommended FL dataset list `_ , please send a PR or ping us in `Slack `_ #contributions channel.
+
+That's it! You have successfully contributed a dataset to the HuggingFace Hub and made it available for FL community. Thank you for your contribution!
\ No newline at end of file
diff --git a/datasets/doc/source/how-to-install-flwr-datasets.rst b/datasets/doc/source/how-to-install-flwr-datasets.rst
index 2068fc11da85..3f79daceb753 100644
--- a/datasets/doc/source/how-to-install-flwr-datasets.rst
+++ b/datasets/doc/source/how-to-install-flwr-datasets.rst
@@ -42,5 +42,5 @@ If everything worked, it should print the version of Flower Datasets to the comm
.. code-block:: none
- 0.3.0
+ 0.4.0
diff --git a/datasets/doc/source/index.rst b/datasets/doc/source/index.rst
index d6b51fc84ad6..55e4ea963453 100644
--- a/datasets/doc/source/index.rst
+++ b/datasets/doc/source/index.rst
@@ -1,7 +1,7 @@
Flower Datasets
===============
-Flower Datasets (``flwr-datasets``) is a library that enables the quick and easy creation of datasets for federated learning/analytics/evaluation. It enables heterogeneity (non-iidness) simulation and division of datasets with the preexisting notion of IDs. The library was created by the ``Flower Labs`` team that also created `Flower `_ : A Friendly Federated Learning Framework.
+Flower Datasets (``flwr-datasets``) is a library that enables the quick and easy creation of datasets for federated learning/analytics/evaluation. It enables heterogeneity (non-iidness) simulation and division of datasets with the preexisting notion of IDs. The library was created by the ``Flower Labs`` team that also created `Flower `_ : A Friendly Federated AI Framework.
Try out an interactive demo to generate code and visualize heterogeneous divisions at the :ref:`bottom of this page`.
@@ -63,20 +63,28 @@ Information-oriented API reference and other reference material.
:maxdepth: 1
:caption: Reference docs
+ recommended-fl-datasets
ref-telemetry
+.. toctree::
+ :maxdepth: 1
+ :caption: Contributor tutorials
+
+ contributor-how-to-contribute-dataset
+
+
Main features
-------------
Flower Datasets library supports:
- **Downloading datasets** - choose the dataset from Hugging Face's ``dataset`` (`link `_)(*)
-- **Partitioning datasets** - choose one of the implemented partitioning scheme or create your own.
+- **Partitioning datasets** - choose one of the implemented partitioning schemes or create your own.
- **Creating centralized datasets** - leave parts of the dataset unpartitioned (e.g. for centralized evaluation)
- **Visualization of the partitioned datasets** - visualize the label distribution of the partitioned dataset (and compare the results on different parameters of the same partitioning schemes, different datasets, different partitioning schemes, or any mix of them)
.. note::
- (*) Once the dataset is available on HuggingFace Hub it can be **immediately** used in ``Flower Datasets`` (no approval from the Flower team needed, no custom code needed).
+ (*) Once the dataset is available on HuggingFace Hub, it can be **immediately** used in ``Flower Datasets`` without requiring approval from the Flower team or the need for custom code.
.. image:: ./_static/readme/comparison_of_partitioning_schemes.png
@@ -93,7 +101,7 @@ Thanks to using Hugging Face's ``datasets`` used under the hood, Flower Datasets
- Jax
- Arrow
-Here are a few of the ``Partitioner`` s that are available: (for a full list see `link `_ )
+Here are a few of the ``Partitioners`` that are available: (for a full list see `link `_ )
* Partitioner (the abstract base class) ``Partitioner``
* IID partitioning ``IidPartitioner(num_partitions)``
@@ -119,7 +127,7 @@ What makes Flower Datasets stand out from other libraries?
* Access to the largest online repository of datasets:
- * The library functionality is independent of the dataset, so you can use any dataset available on `🤗Hugging Face Datasets `_, which means that others can immediately benefit from the dataset you added.
+ * The library functionality is independent of the dataset, so you can use any dataset available on `🤗Hugging Face Datasets `_. This means that others can immediately benefit from the dataset you added.
* Out-of-the-box reproducibility across different projects.
diff --git a/datasets/doc/source/recommended-fl-datasets.rst b/datasets/doc/source/recommended-fl-datasets.rst
new file mode 100644
index 000000000000..92479bd0542a
--- /dev/null
+++ b/datasets/doc/source/recommended-fl-datasets.rst
@@ -0,0 +1,167 @@
+Recommended FL Datasets
+=======================
+
+This page lists the recommended datasets for federated learning research, which can be
+used with Flower Datasets ``flwr-datasets``. To learn about the library, see the
+`quickstart tutorial `_ . To
+see the full FL example with Flower and Flower Datasets open the `quickstart-pytorch
+`_.
+
+.. note::
+
+ All datasets from `HuggingFace Hub `_ can be used with our library. This page presents just a set of datasets we collected that you might find useful.
+
+For more information about any dataset, visit its page by clicking the dataset name. For more information how to use the
+
+Image Datasets
+--------------
+
+.. list-table:: Image Datasets
+ :widths: 40 40 20
+ :header-rows: 1
+
+ * - Name
+ - Size
+ - Image Shape
+ * - `ylecun/mnist `_
+ - train 60k;
+ test 10k
+ - 28x28
+ * - `uoft-cs/cifar10 `_
+ - train 50k;
+ test 10k
+ - 32x32x3
+ * - `uoft-cs/cifar100 `_
+ - train 50k;
+ test 10k
+ - 32x32x3
+ * - `zalando-datasets/fashion_mnist `_
+ - train 60k;
+ test 10k
+ - 28x28
+ * - `flwrlabs/femnist `_
+ - train 814k
+ - 28x28
+ * - `zh-plus/tiny-imagenet `_
+ - train 100k;
+ valid 10k
+ - 64x64x3
+ * - `flwrlabs/usps `_
+ - train 7.3k;
+ test 2k
+ - 16x16
+ * - `flwrlabs/pacs `_
+ - train 10k
+ - 227x227
+ * - `flwrlabs/cinic10 `_
+ - train 90k;
+ valid 90k;
+ test 90k
+ - 32x32x3
+ * - `flwrlabs/caltech101 `_
+ - train 8.7k
+ - varies
+ * - `flwrlabs/office-home `_
+ - train 15.6k
+ - varies
+ * - `flwrlabs/fed-isic2019 `_
+ - train 18.6k;
+ test 4.7k
+ - varies
+ * - `ufldl-stanford/svhn `_
+ - train 73.3k;
+ test 26k;
+ extra 531k
+ - 32x32x3
+ * - `sasha/dog-food `_
+ - train 2.1k;
+ test 0.9k
+ - varies
+ * - `Mike0307/MNIST-M `_
+ - train 59k;
+ test 9k
+ - 32x32
+
+Audio Datasets
+--------------
+
+.. list-table:: Audio Datasets
+ :widths: 35 30 15
+ :header-rows: 1
+
+ * - Name
+ - Size
+ - Subset
+ * - `google/speech_commands `_
+ - train 64.7k
+ - v0.01
+ * - `google/speech_commands `_
+ - train 105.8k
+ - v0.02
+ * - `flwrlabs/ambient-acoustic-context `_
+ - train 70.3k
+ -
+ * - `fixie-ai/common_voice_17_0 `_
+ - varies
+ - 14 versions
+ * - `fixie-ai/librispeech_asr `_
+ - varies
+ - clean/other
+
+Tabular Datasets
+----------------
+
+.. list-table:: Tabular Datasets
+ :widths: 35 30
+ :header-rows: 1
+
+ * - Name
+ - Size
+ * - `scikit-learn/adult-census-income `_
+ - train 32.6k
+ * - `jlh/uci-mushrooms `_
+ - train 8.1k
+ * - `scikit-learn/iris `_
+ - train 150
+
+Text Datasets
+-------------
+
+.. list-table:: Text Datasets
+ :widths: 40 30 30
+ :header-rows: 1
+
+ * - Name
+ - Size
+ - Category
+ * - `sentiment140 `_
+ - train 1.6M;
+ test 0.5k
+ - Sentiment
+ * - `google-research-datasets/mbpp `_
+ - full 974; sanitized 427
+ - General
+ * - `openai/openai_humaneval `_
+ - test 164
+ - General
+ * - `lukaemon/mmlu `_
+ - varies
+ - General
+ * - `takala/financial_phrasebank `_
+ - train 4.8k
+ - Financial
+ * - `pauri32/fiqa-2018 `_
+ - train 0.9k; validation 0.1k; test 0.2k
+ - Financial
+ * - `zeroshot/twitter-financial-news-sentiment `_
+ - train 9.5k; validation 2.4k
+ - Financial
+ * - `bigbio/pubmed_qa `_
+ - train 2M; validation 11k
+ - Medical
+ * - `openlifescienceai/medmcqa `_
+ - train 183k; validation 4.3k; test 6.2k
+ - Medical
+ * - `bigbio/med_qa `_
+ - train 10.1k; test 1.3k; validation 1.3k
+ - Medical
diff --git a/datasets/e2e/pytorch/pyproject.toml b/datasets/e2e/pytorch/pyproject.toml
index 009ad2d74235..d42409ca1195 100644
--- a/datasets/e2e/pytorch/pyproject.toml
+++ b/datasets/e2e/pytorch/pyproject.toml
@@ -9,8 +9,8 @@ description = "Flower Datasets with PyTorch"
authors = ["The Flower Authors "]
[tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
flwr-datasets = { path = "./../../", extras = ["vision"] }
-torch = "^1.12.0"
-torchvision = "^0.14.1"
+torch = ">=1.12.0,<3.0.0"
+torchvision = ">=0.19.0,<1.0.0"
parameterized = "==0.9.0"
diff --git a/datasets/e2e/scikit-learn/pyproject.toml b/datasets/e2e/scikit-learn/pyproject.toml
index 48356e4a945f..ca7fb45d82be 100644
--- a/datasets/e2e/scikit-learn/pyproject.toml
+++ b/datasets/e2e/scikit-learn/pyproject.toml
@@ -9,7 +9,7 @@ description = "Flower Datasets with scikit-learn"
authors = ["The Flower Authors "]
[tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
flwr-datasets = { path = "./../../", extras = ["vision"] }
scikit-learn = "^1.2.0"
parameterized = "==0.9.0"
diff --git a/datasets/e2e/tensorflow/pyproject.toml b/datasets/e2e/tensorflow/pyproject.toml
index dbb6720219b2..fbfc8eb89451 100644
--- a/datasets/e2e/tensorflow/pyproject.toml
+++ b/datasets/e2e/tensorflow/pyproject.toml
@@ -9,7 +9,7 @@ description = "Flower Datasets with TensorFlow"
authors = ["The Flower Authors "]
[tool.poetry.dependencies]
-python = ">=3.8,<3.11"
+python = ">=3.9,<3.11"
flwr-datasets = { path = "./../../", extras = ["vision"] }
tensorflow-cpu = "^2.9.1, !=2.11.1"
tensorflow-io-gcs-filesystem = "<0.35.0"
diff --git a/datasets/flwr_datasets/__init__.py b/datasets/flwr_datasets/__init__.py
index bd68fa43c606..dfae804046bc 100644
--- a/datasets/flwr_datasets/__init__.py
+++ b/datasets/flwr_datasets/__init__.py
@@ -15,7 +15,7 @@
"""Flower Datasets main package."""
-from flwr_datasets import partitioner, preprocessor
+from flwr_datasets import metrics, partitioner, preprocessor
from flwr_datasets import utils as utils
from flwr_datasets import visualization
from flwr_datasets.common.version import package_version as _package_version
diff --git a/datasets/flwr_datasets/common/typing.py b/datasets/flwr_datasets/common/typing.py
index d6d37b468494..6b76e7b22eea 100644
--- a/datasets/flwr_datasets/common/typing.py
+++ b/datasets/flwr_datasets/common/typing.py
@@ -22,5 +22,5 @@
NDArray = npt.NDArray[Any]
NDArrayInt = npt.NDArray[np.int_]
-NDArrayFloat = npt.NDArray[np.float_]
+NDArrayFloat = npt.NDArray[np.float64]
NDArrays = list[NDArray]
diff --git a/datasets/flwr_datasets/mock_utils_test.py b/datasets/flwr_datasets/mock_utils_test.py
index 0976166648eb..acfa4b16e4ee 100644
--- a/datasets/flwr_datasets/mock_utils_test.py
+++ b/datasets/flwr_datasets/mock_utils_test.py
@@ -409,7 +409,11 @@ def _load_mocked_dataset_by_partial_download(
The dataset with the requested samples.
"""
dataset = datasets.load_dataset(
- dataset_name, name=subset_name, split=split_name, streaming=True
+ dataset_name,
+ name=subset_name,
+ split=split_name,
+ streaming=True,
+ trust_remote_code=True,
)
dataset_list = []
# It's a list of dict such that each dict represent a single sample of the dataset
diff --git a/datasets/flwr_datasets/partitioner/dirichlet_partitioner_test.py b/datasets/flwr_datasets/partitioner/dirichlet_partitioner_test.py
index ed38e8ee2a41..693e0d6a5aa6 100644
--- a/datasets/flwr_datasets/partitioner/dirichlet_partitioner_test.py
+++ b/datasets/flwr_datasets/partitioner/dirichlet_partitioner_test.py
@@ -29,7 +29,7 @@
def _dummy_setup(
num_partitions: int,
- alpha: Union[float, NDArray[np.float_]],
+ alpha: Union[float, NDArray[np.float64]],
num_rows: int,
partition_by: str,
self_balancing: bool = True,
diff --git a/datasets/flwr_datasets/partitioner/distribution_partitioner.py b/datasets/flwr_datasets/partitioner/distribution_partitioner.py
index 86be62b36070..e4182f587cad 100644
--- a/datasets/flwr_datasets/partitioner/distribution_partitioner.py
+++ b/datasets/flwr_datasets/partitioner/distribution_partitioner.py
@@ -36,21 +36,22 @@ class DistributionPartitioner(Partitioner): # pylint: disable=R0902
in a deterministic pathological manner. The 1st dimension is the number of unique
labels and the 2nd-dimension is the number of buckets into which the samples
associated with each label will be divided. That is, given a distribution array of
- shape,
- `num_unique_labels_per_partition` x `num_partitions`
- ( `num_unique_labels`, ---------------------------------------------------- ),
- `num_unique_labels`
- the label_id at the i'th row is assigned to the partition_id based on the following
- approach.
-
- First, for an i'th row, generate a list of `id`s according to the formula:
- id = alpha + beta
- where,
- alpha = (i - num_unique_labels_per_partition + 1) \
- + (j % num_unique_labels_per_partition),
- alpha = alpha + (alpha >= 0 ? 0 : num_unique_labels),
- beta = num_unique_labels * (j // num_unique_labels_per_partition)
- and j in {0, 1, 2, ..., `num_columns`}. Then, sort the list of `id`s in ascending
+ shape,::
+
+ `num_unique_labels_per_partition` x `num_partitions`
+ ( `num_unique_labels`, ---------------------------------------------------- ),
+ `num_unique_labels`
+ the label_id at the i'th row is assigned to the partition_id based on the
+ following approach.
+
+ First, for an i'th row, generate a list of `id`s according to the formula:
+ id = alpha + beta
+ where,
+ alpha = (i - num_unique_labels_per_partition + 1) +
+ + (j % num_unique_labels_per_partition),
+ alpha = alpha + (alpha >= 0 ? 0 : num_unique_labels),
+ beta = num_unique_labels * (j // num_unique_labels_per_partition)
+ and j in {0, 1, 2, ..., `num_columns`}. Then, sort the list of `id` s in ascending
order. The j'th index in this sorted list corresponds to the partition_id that the
i'th unique label (and the underlying distribution array value) will be assigned to.
So, for a dataset with 10 unique labels and a configuration with 20 partitions and
diff --git a/datasets/flwr_datasets/visualization/bar_plot.py b/datasets/flwr_datasets/visualization/bar_plot.py
index 2b09fb189c7a..0f6936976fc0 100644
--- a/datasets/flwr_datasets/visualization/bar_plot.py
+++ b/datasets/flwr_datasets/visualization/bar_plot.py
@@ -22,6 +22,7 @@
from matplotlib import colors as mcolors
from matplotlib import pyplot as plt
from matplotlib.axes import Axes
+from matplotlib.figure import Figure
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
@@ -82,10 +83,11 @@ def _plot_bar(
if "stacked" not in plot_kwargs:
plot_kwargs["stacked"] = True
- axis = dataframe.plot(
+ axis_df: Axes = dataframe.plot(
ax=axis,
**plot_kwargs,
)
+ assert axis_df is not None, "axis is None after plotting using DataFrame.plot()"
if legend:
if legend_kwargs is None:
@@ -104,20 +106,22 @@ def _plot_bar(
shift = min(0.05 + max_len_label_str / 100, 0.15)
legend_kwargs["bbox_to_anchor"] = (1.0 + shift, 0.5)
- handles, legend_labels = axis.get_legend_handles_labels()
- _ = axis.figure.legend(
+ handles, legend_labels = axis_df.get_legend_handles_labels()
+ figure = axis_df.figure
+ assert isinstance(figure, Figure), "figure extraction from axes is not a Figure"
+ _ = figure.legend(
handles=handles[::-1], labels=legend_labels[::-1], **legend_kwargs
)
# Heuristic to make the partition id on xticks non-overlapping
if partition_id_axis == "x":
- xticklabels = axis.get_xticklabels()
+ xticklabels = axis_df.get_xticklabels()
if len(xticklabels) > 20:
# Make every other xtick label not visible
for i, label in enumerate(xticklabels):
if i % 2 == 1:
label.set_visible(False)
- return axis
+ return axis_df
def _initialize_figsize(
diff --git a/datasets/flwr_datasets/visualization/comparison_label_distribution.py b/datasets/flwr_datasets/visualization/comparison_label_distribution.py
index 17b9a9aec251..c741ddee219e 100644
--- a/datasets/flwr_datasets/visualization/comparison_label_distribution.py
+++ b/datasets/flwr_datasets/visualization/comparison_label_distribution.py
@@ -30,6 +30,7 @@
# pylint: disable=too-many-arguments,too-many-locals
+# mypy: disable-error-code="call-overload"
def plot_comparison_label_distribution(
partitioner_list: list[Partitioner],
label_name: Union[str, list[str]],
@@ -153,7 +154,11 @@ def plot_comparison_label_distribution(
figsize = _initialize_comparison_figsize(figsize, num_partitioners)
axes_sharing = _initialize_axis_sharing(size_unit, plot_type, partition_id_axis)
fig, axes = plt.subplots(
- 1, num_partitioners, layout="constrained", figsize=figsize, **axes_sharing
+ nrows=1,
+ ncols=num_partitioners,
+ figsize=figsize,
+ layout="constrained",
+ **axes_sharing,
)
if titles is None:
diff --git a/datasets/flwr_datasets/visualization/label_distribution.py b/datasets/flwr_datasets/visualization/label_distribution.py
index b1183c225b86..550a4ecae725 100644
--- a/datasets/flwr_datasets/visualization/label_distribution.py
+++ b/datasets/flwr_datasets/visualization/label_distribution.py
@@ -245,5 +245,7 @@ def plot_label_distributions(
plot_kwargs,
legend_kwargs,
)
- assert axis is not None
- return axis.figure, axis, dataframe
+ assert axis is not None, "axis is None after plotting"
+ figure = axis.figure
+ assert isinstance(figure, Figure), "figure extraction from axes is not a Figure"
+ return figure, axis, dataframe
diff --git a/datasets/pyproject.toml b/datasets/pyproject.toml
index 73523af2039e..2d699c5e901b 100644
--- a/datasets/pyproject.toml
+++ b/datasets/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "flwr-datasets"
-version = "0.3.0"
+version = "0.4.0"
description = "Flower Datasets"
license = "Apache-2.0"
authors = ["The Flower Authors "]
@@ -51,14 +51,13 @@ exclude = [
]
[tool.poetry.dependencies]
-python = "^3.8"
-numpy = "^1.21.0"
-datasets = ">=2.14.6 <2.20.0"
+python = "^3.9"
+numpy = ">=1.26.0,<3.0.0"
+datasets = ">=2.14.6 <=3.1.0"
pillow = { version = ">=6.2.1", optional = true }
soundfile = { version = ">=0.12.1", optional = true }
librosa = { version = ">=0.10.0.post2", optional = true }
tqdm ="^4.66.1"
-pyarrow = "==16.1.0"
matplotlib = "^3.7.5"
seaborn = "^0.13.0"
diff --git a/dev/build-docker-image-matrix.py b/dev/build-docker-image-matrix.py
index 52c96e3cca7a..9d255ac5471f 100644
--- a/dev/build-docker-image-matrix.py
+++ b/dev/build-docker-image-matrix.py
@@ -1,13 +1,36 @@
"""
-Usage: python dev/build-docker-image-matrix.py --flwr-version
+Usage: python dev/build-docker-image-matrix.py --flwr-version
+
+Images are built in three workflows: stable, nightly, and unstable (main).
+Each builds for `amd64` and `arm64`.
+
+1. **Ubuntu Images**:
+ - Used for images where dependencies might be installed by users.
+ - Ubuntu uses `glibc`, compatible with most ML frameworks.
+
+2. **Alpine Images**:
+ - Used only for minimal images (e.g., SuperLink) where no extra dependencies are expected.
+ - Limited use due to dependency (in particular ML frameworks) compilation complexity with `musl`.
+
+Workflow Details:
+- **Stable Release**: Triggered on new releases. Builds full matrix (all Python versions, Ubuntu and Alpine).
+- **Nightly Release**: Daily trigger. Builds full matrix (latest Python, Ubuntu only).
+- **Unstable**: Triggered on main branch commits. Builds simplified matrix (latest Python, Ubuntu only).
"""
+import sys
import argparse
import json
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, field
from enum import Enum
from typing import Any, Callable, Dict, List, Optional
+# when we switch to Python 3.11 in the ci, we need to change the DistroName to:
+# class DistroName(StrEnum):
+# ALPINE = "alpine"
+# UBUNTU = "ubuntu"
+assert sys.version_info < (3, 11), "Script requires Python 3.9 or lower."
+
class DistroName(str, Enum):
ALPINE = "alpine"
@@ -31,36 +54,91 @@ class Distro:
@dataclass
-class BaseImage:
+class Variant:
distro: Distro
- python_version: str
- namespace_repository: str
- file_dir: str
- tag: str
- flwr_version: str
+ extras: Optional[Any] = None
-def new_base_image(
- flwr_version: str, python_version: str, distro: Distro
-) -> Dict[str, Any]:
- return BaseImage(
- distro,
- python_version,
- "flwr/base",
- f"{DOCKERFILE_ROOT}/base/{distro.name.value}",
- f"{flwr_version}-py{python_version}-{distro.name.value}{distro.version}",
- flwr_version,
+@dataclass
+class CpuVariant:
+ pass
+
+
+@dataclass
+class CudaVariant:
+ version: str
+
+
+CUDA_VERSIONS_CONFIG = [
+ ("11.2.2", "20.04"),
+ ("11.8.0", "22.04"),
+ ("12.1.0", "22.04"),
+ ("12.3.2", "22.04"),
+]
+LATEST_SUPPORTED_CUDA_VERSION = Variant(
+ Distro(DistroName.UBUNTU, "22.04"),
+ CudaVariant(version="12.4.1"),
+)
+
+# ubuntu base image
+UBUNTU_VARIANT = Variant(
+ Distro(DistroName.UBUNTU, "24.04"),
+ CpuVariant(),
+)
+
+
+# alpine base image
+ALPINE_VARIANT = Variant(
+ Distro(DistroName.ALPINE, "3.19"),
+ CpuVariant(),
+)
+
+
+# ubuntu cuda base images
+CUDA_VARIANTS = [
+ Variant(
+ Distro(DistroName.UBUNTU, ubuntu_version),
+ CudaVariant(version=cuda_version),
)
+ for (cuda_version, ubuntu_version) in CUDA_VERSIONS_CONFIG
+] + [LATEST_SUPPORTED_CUDA_VERSION]
-def generate_base_images(
- flwr_version: str, python_versions: List[str], distros: List[Dict[str, str]]
-) -> List[Dict[str, Any]]:
- return [
- new_base_image(flwr_version, python_version, distro)
- for distro in distros
- for python_version in python_versions
- ]
+def remove_patch_version(version: str) -> str:
+ return ".".join(version.split(".")[0:2])
+
+
+@dataclass
+class BaseImageBuilder:
+ file_dir_fn: Callable[[Any], str]
+ tags_fn: Callable[[Any], list[str]]
+ build_args_fn: Callable[[Any], str]
+ build_args: Any
+ tags: list[str] = field(init=False)
+ file_dir: str = field(init=False)
+ tags_encoded: str = field(init=False)
+ build_args_encoded: str = field(init=False)
+
+
+@dataclass
+class BaseImage(BaseImageBuilder):
+ namespace_repository: str = "flwr/base"
+
+ @property
+ def file_dir(self) -> str:
+ return self.file_dir_fn(self.build_args)
+
+ @property
+ def tags(self) -> str:
+ return self.tags_fn(self.build_args)
+
+ @property
+ def tags_encoded(self) -> str:
+ return "\n".join(self.tags)
+
+ @property
+ def build_args_encoded(self) -> str:
+ return self.build_args_fn(self.build_args)
@dataclass
@@ -68,7 +146,7 @@ class BinaryImage:
namespace_repository: str
file_dir: str
base_image: str
- tags: List[str]
+ tags_encoded: str
def new_binary_image(
@@ -83,7 +161,7 @@ def new_binary_image(
return BinaryImage(
f"flwr/{name}",
f"{DOCKERFILE_ROOT}/{name}",
- base_image.tag,
+ base_image.tags[0],
"\n".join(tags),
)
@@ -103,47 +181,95 @@ def generate_binary_images(
def tag_latest_alpine_with_flwr_version(image: BaseImage) -> List[str]:
if (
- image.distro.name == DistroName.ALPINE
- and image.python_version == LATEST_SUPPORTED_PYTHON_VERSION
+ image.build_args.variant.distro.name == DistroName.ALPINE
+ and image.build_args.python_version == LATEST_SUPPORTED_PYTHON_VERSION
):
- return [image.tag, image.flwr_version]
+ return image.tags + [image.build_args.flwr_version]
else:
- return [image.tag]
+ return image.tags
def tag_latest_ubuntu_with_flwr_version(image: BaseImage) -> List[str]:
if (
- image.distro.name == DistroName.UBUNTU
- and image.python_version == LATEST_SUPPORTED_PYTHON_VERSION
+ image.build_args.variant.distro.name == DistroName.UBUNTU
+ and image.build_args.python_version == LATEST_SUPPORTED_PYTHON_VERSION
+ and isinstance(image.build_args.variant.extras, CpuVariant)
):
- return [image.tag, image.flwr_version]
+ return image.tags + [image.build_args.flwr_version]
else:
- return [image.tag]
+ return image.tags
+
+
+#
+# Build matrix for stable releases
+#
+def build_stable_matrix(flwr_version: str) -> List[BaseImage]:
+ @dataclass
+ class StableBaseImageBuildArgs:
+ variant: Variant
+ python_version: str
+ flwr_version: str
+
+ cpu_build_args = """PYTHON_VERSION={python_version}
+FLWR_VERSION={flwr_version}
+DISTRO={distro_name}
+DISTRO_VERSION={distro_version}
+"""
+ cpu_build_args_variants = [
+ StableBaseImageBuildArgs(UBUNTU_VARIANT, python_version, flwr_version)
+ for python_version in SUPPORTED_PYTHON_VERSIONS
+ ] + [
+ StableBaseImageBuildArgs(
+ ALPINE_VARIANT, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version
+ )
+ ]
-if __name__ == "__main__":
- arg_parser = argparse.ArgumentParser(
- description="Generate Github Docker workflow matrix"
- )
- arg_parser.add_argument("--flwr-version", type=str, required=True)
- args = arg_parser.parse_args()
+ cpu_base_images = [
+ BaseImage(
+ file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}",
+ tags_fn=lambda args: [
+ f"{args.flwr_version}-py{args.python_version}-{args.variant.distro.name.value}{args.variant.distro.version}"
+ ],
+ build_args_fn=lambda args: cpu_build_args.format(
+ python_version=args.python_version,
+ flwr_version=args.flwr_version,
+ distro_name=args.variant.distro.name,
+ distro_version=args.variant.distro.version,
+ ),
+ build_args=build_args_variant,
+ )
+ for build_args_variant in cpu_build_args_variants
+ ]
- flwr_version = args.flwr_version
+ cuda_build_args_variants = [
+ StableBaseImageBuildArgs(variant, python_version, flwr_version)
+ for variant in CUDA_VARIANTS
+ for python_version in SUPPORTED_PYTHON_VERSIONS
+ ]
- # ubuntu base images for each supported python version
- ubuntu_base_images = generate_base_images(
- flwr_version,
- SUPPORTED_PYTHON_VERSIONS,
- [Distro(DistroName.UBUNTU, "24.04")],
- )
- # alpine base images for the latest supported python version
- alpine_base_images = generate_base_images(
- flwr_version,
- [LATEST_SUPPORTED_PYTHON_VERSION],
- [Distro(DistroName.ALPINE, "3.19")],
- )
+ cuda_build_args = cpu_build_args + """CUDA_VERSION={cuda_version}"""
+
+ cuda_base_image = [
+ BaseImage(
+ file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}-cuda",
+ tags_fn=lambda args: [
+ f"{args.flwr_version}-py{args.python_version}-cu{remove_patch_version(args.variant.extras.version)}-{args.variant.distro.name.value}{args.variant.distro.version}",
+ ],
+ build_args_fn=lambda args: cuda_build_args.format(
+ python_version=args.python_version,
+ flwr_version=args.flwr_version,
+ distro_name=args.variant.distro.name,
+ distro_version=args.variant.distro.version,
+ cuda_version=args.variant.extras.version,
+ ),
+ build_args=build_args_variant,
+ )
+ for build_args_variant in cuda_build_args_variants
+ ]
- base_images = ubuntu_base_images + alpine_base_images
+ # base_images = cpu_base_images + cuda_base_image
+ base_images = cpu_base_images
binary_images = (
# ubuntu and alpine images for the latest supported python version
@@ -151,17 +277,22 @@ def tag_latest_ubuntu_with_flwr_version(image: BaseImage) -> List[str]:
"superlink",
base_images,
tag_latest_alpine_with_flwr_version,
- lambda image: image.python_version == LATEST_SUPPORTED_PYTHON_VERSION,
+ lambda image: image.build_args.python_version
+ == LATEST_SUPPORTED_PYTHON_VERSION
+ and isinstance(image.build_args.variant.extras, CpuVariant),
)
# ubuntu images for each supported python version
+ generate_binary_images(
"supernode",
base_images,
tag_latest_alpine_with_flwr_version,
- lambda image: image.distro.name == DistroName.UBUNTU
+ lambda image: (
+ image.build_args.variant.distro.name == DistroName.UBUNTU
+ and isinstance(image.build_args.variant.extras, CpuVariant)
+ )
or (
- image.distro.name == DistroName.ALPINE
- and image.python_version == LATEST_SUPPORTED_PYTHON_VERSION
+ image.build_args.variant.distro.name == DistroName.ALPINE
+ and image.build_args.python_version == LATEST_SUPPORTED_PYTHON_VERSION
),
)
# ubuntu images for each supported python version
@@ -169,28 +300,216 @@ def tag_latest_ubuntu_with_flwr_version(image: BaseImage) -> List[str]:
"serverapp",
base_images,
tag_latest_ubuntu_with_flwr_version,
- lambda image: image.distro.name == DistroName.UBUNTU,
+ lambda image: image.build_args.variant.distro.name == DistroName.UBUNTU,
)
# ubuntu images for each supported python version
+ generate_binary_images(
- "superexec",
+ "clientapp",
base_images,
tag_latest_ubuntu_with_flwr_version,
- lambda image: image.distro.name == DistroName.UBUNTU,
+ lambda image: image.build_args.variant.distro.name == DistroName.UBUNTU,
+ )
+ )
+
+ return base_images, binary_images
+
+
+#
+# Build matrix for unstable releases
+#
+def build_unstable_matrix(flwr_version_ref: str) -> List[BaseImage]:
+ @dataclass
+ class UnstableBaseImageBuildArgs:
+ variant: Variant
+ python_version: str
+ flwr_version_ref: str
+
+ cpu_ubuntu_build_args_variant = UnstableBaseImageBuildArgs(
+ UBUNTU_VARIANT, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version_ref
+ )
+
+ cpu_build_args = """PYTHON_VERSION={python_version}
+FLWR_VERSION_REF={flwr_version_ref}
+DISTRO={distro_name}
+DISTRO_VERSION={distro_version}
+"""
+
+ cpu_base_image = BaseImage(
+ file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}",
+ tags_fn=lambda _: ["unstable"],
+ build_args_fn=lambda args: cpu_build_args.format(
+ python_version=args.python_version,
+ flwr_version_ref=args.flwr_version_ref,
+ distro_name=args.variant.distro.name,
+ distro_version=args.variant.distro.version,
+ ),
+ build_args=cpu_ubuntu_build_args_variant,
+ )
+
+ cuda_build_args_variant = UnstableBaseImageBuildArgs(
+ LATEST_SUPPORTED_CUDA_VERSION, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version_ref
+ )
+
+ cuda_build_args = cpu_build_args + """CUDA_VERSION={cuda_version}"""
+
+ cuda_base_image = BaseImage(
+ file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}-cuda",
+ tags_fn=lambda _: ["unstable-cuda"],
+ build_args_fn=lambda args: cuda_build_args.format(
+ python_version=args.python_version,
+ flwr_version_ref=args.flwr_version_ref,
+ distro_name=args.variant.distro.name,
+ distro_version=args.variant.distro.version,
+ cuda_version=args.variant.extras.version,
+ ),
+ build_args=cuda_build_args_variant,
+ )
+
+ # base_images = [cpu_base_image, cuda_base_image]
+ base_images = [cpu_base_image]
+
+ binary_images = (
+ generate_binary_images(
+ "superlink",
+ base_images,
+ lambda image: image.tags,
+ lambda image: isinstance(image.build_args.variant.extras, CpuVariant),
)
- # ubuntu images for each supported python version
+ generate_binary_images(
- "clientapp",
+ "supernode",
base_images,
- tag_latest_ubuntu_with_flwr_version,
- lambda image: image.distro.name == DistroName.UBUNTU,
+ lambda image: image.tags,
+ lambda image: isinstance(image.build_args.variant.extras, CpuVariant),
)
+ + generate_binary_images("serverapp", base_images, lambda image: image.tags)
+ + generate_binary_images("clientapp", base_images, lambda image: image.tags)
+ )
+
+ return base_images, binary_images
+
+
+#
+# Build matrix for nightly releases
+#
+def build_nightly_matrix(flwr_version: str, flwr_package: str) -> List[BaseImage]:
+ @dataclass
+ class NightlyBaseImageBuildArgs:
+ variant: Variant
+ python_version: str
+ flwr_version: str
+ flwr_package: str
+
+ cpu_ubuntu_build_args_variant = NightlyBaseImageBuildArgs(
+ UBUNTU_VARIANT, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version, flwr_package
)
+ cpu_build_args = """PYTHON_VERSION={python_version}
+FLWR_VERSION={flwr_version}
+FLWR_PACKAGE={flwr_package}
+DISTRO={distro_name}
+DISTRO_VERSION={distro_version}
+"""
+
+ cpu_base_image = BaseImage(
+ file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}",
+ tags_fn=lambda args: [args.flwr_version, "nightly"],
+ build_args_fn=lambda args: cpu_build_args.format(
+ python_version=args.python_version,
+ flwr_version=args.flwr_version,
+ flwr_package=args.flwr_package,
+ distro_name=args.variant.distro.name,
+ distro_version=args.variant.distro.version,
+ ),
+ build_args=cpu_ubuntu_build_args_variant,
+ )
+
+ cuda_build_args_variant = NightlyBaseImageBuildArgs(
+ LATEST_SUPPORTED_CUDA_VERSION,
+ LATEST_SUPPORTED_PYTHON_VERSION,
+ flwr_version,
+ flwr_package,
+ )
+
+ cuda_build_args = cpu_build_args + """CUDA_VERSION={cuda_version}"""
+
+ cuda_base_image = BaseImage(
+ file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}-cuda",
+ tags_fn=lambda args: [f"{args.flwr_version}-cuda", "nightly-cuda"],
+ build_args_fn=lambda args: cuda_build_args.format(
+ python_version=args.python_version,
+ flwr_version=args.flwr_version,
+ flwr_package=args.flwr_package,
+ distro_name=args.variant.distro.name,
+ distro_version=args.variant.distro.version,
+ cuda_version=args.variant.extras.version,
+ ),
+ build_args=cuda_build_args_variant,
+ )
+
+ # base_images = [cpu_base_image, cuda_base_image]
+ base_images = [cpu_base_image]
+
+ binary_images = (
+ generate_binary_images(
+ "superlink",
+ base_images,
+ lambda image: image.tags,
+ lambda image: isinstance(image.build_args.variant.extras, CpuVariant),
+ )
+ + generate_binary_images(
+ "supernode",
+ base_images,
+ lambda image: image.tags,
+ lambda image: isinstance(image.build_args.variant.extras, CpuVariant),
+ )
+ + generate_binary_images("serverapp", base_images, lambda image: image.tags)
+ + generate_binary_images("clientapp", base_images, lambda image: image.tags)
+ )
+
+ return base_images, binary_images
+
+
+if __name__ == "__main__":
+ arg_parser = argparse.ArgumentParser(
+ description="Generate Github Docker workflow matrix"
+ )
+ arg_parser.add_argument("--flwr-version", type=str, required=True)
+ arg_parser.add_argument("--flwr-package", type=str, default="flwr")
+ arg_parser.add_argument(
+ "--matrix", choices=["stable", "nightly", "unstable"], default="stable"
+ )
+
+ args = arg_parser.parse_args()
+
+ flwr_version = args.flwr_version
+ flwr_package = args.flwr_package
+ matrix = args.matrix
+
+ if matrix == "stable":
+ base_images, binary_images = build_stable_matrix(flwr_version)
+ elif matrix == "nightly":
+ base_images, binary_images = build_nightly_matrix(flwr_version, flwr_package)
+ else:
+ base_images, binary_images = build_unstable_matrix(flwr_version)
+
print(
json.dumps(
{
- "base": {"images": list(map(lambda image: asdict(image), base_images))},
+ "base": {
+ "images": list(
+ map(
+ lambda image: asdict(
+ image,
+ dict_factory=lambda x: {
+ k: v
+ for (k, v) in x
+ if v is not None and callable(v) is False
+ },
+ ),
+ base_images,
+ )
+ )
+ },
"binary": {
"images": list(map(lambda image: asdict(image), binary_images))
},
diff --git a/dev/build-example-docs.py b/dev/build-example-docs.py
index 772a26272fd7..05656967bbbd 100644
--- a/dev/build-example-docs.py
+++ b/dev/build-example-docs.py
@@ -28,7 +28,7 @@
-----------------------------
Welcome to Flower Examples' documentation. `Flower `_ is
-a friendly federated learning framework.
+a friendly federated AI framework.
Join the Flower Community
-------------------------
diff --git a/dev/prepare-release-changelog.sh b/dev/prepare-release-changelog.sh
deleted file mode 100755
index 3f2a2ae325e9..000000000000
--- a/dev/prepare-release-changelog.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-set -e
-cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../
-
-# Get the current date in the format YYYY-MM-DD
-current_date=$(date +"%Y-%m-%d")
-
-tags=$(git tag --sort=-v:refname)
-new_version=$1
-old_version=$(echo "$tags" | sed -n '1p')
-
-shortlog=$(git shortlog "$old_version"..main -s | grep -vEi '(\(|\[)bot(\)|\])' | awk '{name = substr($0, index($0, $2)); printf "%s`%s`", sep, name; sep=", "} END {print ""}')
-
-token=""
-thanks="\n### Thanks to our contributors\n\nWe would like to give our special thanks to all the contributors who made the new version of Flower possible (in \`git shortlog\` order):\n\n$shortlog $token"
-
-# Check if the token exists in the markdown file
-if ! grep -q "$token" doc/source/ref-changelog.md; then
- # If the token does not exist in the markdown file, append the new content after the version
- awk -v version="$new_version" -v date="$current_date" -v text="$thanks" \
- '{ if ($0 ~ "## Unreleased") print "## " version " (" date ")\n" text; else print $0 }' doc/source/ref-changelog.md > temp.md && mv temp.md doc/source/ref-changelog.md
-else
- # If the token exists, replace the line containing the token with the new shortlog
- awk -v token="$token" -v newlog="$shortlog $token" '{ if ($0 ~ token) print newlog; else print $0 }' doc/source/ref-changelog.md > temp.md && mv temp.md doc/source/ref-changelog.md
-fi
diff --git a/dev/swift-docs-resources/footer.html b/dev/swift-docs-resources/footer.html
index 6a3a5492c83b..cb4dcbf44c1c 100644
--- a/dev/swift-docs-resources/footer.html
+++ b/dev/swift-docs-resources/footer.html
@@ -1 +1 @@
-