diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 542b8a16e..eae6d83ac 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -2,15 +2,7 @@ name: Build and publish to Docker Hub on: release: # job will automatically run after a new "release" is create on github. - types: [created] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - inputs: - dry_run: - description: 'If true, will not push the built images to docker hub.' - required: false - default: 'false' + types: [published] jobs: # this job will build, test and (potentially) push the docker images to docker hub @@ -29,6 +21,12 @@ jobs: # - Pushes images (built at BUILD PHASE) to docker hub. docker_build_and_publish: runs-on: ubuntu-latest + env: + github_token: ${{ secrets.TOKEN_GITHUB }} + permissions: + id-token: write + contents: write # 'write' access to repository contents + pull-requests: write # 'write' access to pull requests steps: # BUILD PHASE - name: Checkout @@ -43,25 +41,19 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Get version tag from github release - if: github.event_name == 'release' && github.event.action == 'created' - run: | - echo "opal_version_tag=${{ github.event.release.tag_name }}" >> $GITHUB_ENV - - - name: Get version tag from git history - if: ${{ !(github.event_name == 'release' && github.event.action == 'created') }} + - name: Docker Compose install run: | - echo "opal_version_tag=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV + curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose - name: Echo version tag run: | - echo "The version tag that will be published to docker hub is: ${{ env.opal_version_tag }}" + echo "The version tag that will be published to docker hub is: ${{ github.event.release.tag_name }}" - name: Build client for testing id: build_client @@ -76,19 +68,6 @@ jobs: tags: | permitio/opal-client:test - - name: Build client-standalone for testing - id: build_client_standalone - uses: docker/build-push-action@v4 - with: - file: docker/Dockerfile - push: false - target: client-standalone - cache-from: type=registry,ref=permitio/opal-client-standalone:latest - cache-to: type=inline - load: true - tags: | - permitio/opal-client-standalone:test - - name: Build server for testing id: build_server uses: docker/build-push-action@v4 @@ -122,7 +101,6 @@ jobs: # pushes the *same* docker images that were previously tested as part of e2e sanity test. # each image is pushed with the versioned tag first, if it succeeds the image is pushed with the latest tag as well. - name: Build & Push client - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client uses: docker/build-push-action@v4 with: @@ -134,10 +112,9 @@ jobs: cache-to: type=inline tags: | permitio/opal-client:latest - permitio/opal-client:${{ env.opal_version_tag }} + permitio/opal-client:${{ github.event.release.tag_name }} - name: Build client-standalone - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_client_standalone uses: docker/build-push-action@v4 with: @@ -149,10 +126,9 @@ jobs: cache-to: type=inline tags: | permitio/opal-client-standalone:latest - permitio/opal-client-standalone:${{ env.opal_version_tag }} + permitio/opal-client-standalone:${{ github.event.release.tag_name }} - name: Build server - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true') }} id: build_push_server uses: docker/build-push-action@v4 with: @@ -164,4 +140,92 @@ jobs: cache-to: type=inline tags: | permitio/opal-server:latest - permitio/opal-server:${{ env.opal_version_tag }} + permitio/opal-server:${{ github.event.release.tag_name }} + + - name: Build & Push client cedar + id: build_push_client_cedar + uses: docker/build-push-action@v4 + with: + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + target: client-cedar + cache-from: type=registry,ref=permitio/opal-client-cedar:latest + cache-to: type=inline + tags: | + permitio/opal-client-cedar:latest + permitio/opal-client-cedar:${{ github.event.release.tag_name }} + + - name: Python setup + uses: actions/setup-python@v5 + with: + python-version: '3.11.8' + + # This is the root file representing the package for all the sub-packages. + - name: Bump version - packaging__.py + run: | + version_tag=${{ github.event.release.tag_name }} + version_tag=${version_tag#v} # Remove the leading 'v' + version_tuple=$(echo $version_tag | sed 's/\./, /g') + sed -i "s/VERSION = (.*/VERSION = (${version_tuple})/" packages/__packaging__.py + cat packages/__packaging__.py + + - name: Cleanup setup.py and Build every sub-packages + run: | + pip install wheel + cd packages/opal-common/ ; rm -rf *.egg-info build/ dist/ + python setup.py sdist bdist_wheel + cd ../.. + cd packages/opal-client/ ; rm -rf *.egg-info build/ dist/ + python setup.py sdist bdist_wheel + cd ../.. + cd packages/opal-server/ ; rm -rf *.egg-info build/ dist/ + python setup.py sdist bdist_wheel + cd ../.. + + # Upload package distributions to the release - All assets in one step + - name: Upload assets to release + uses: shogo82148/actions-upload-release-asset@v1.7.5 + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: | + packages/opal-common/dist/* + packages/opal-client/dist/* + packages/opal-server/dist/* + + # Publish package distributions to PyPI + - name: Publish package distributions to PyPI - Opal-Common + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + packages-dir: packages/opal-common/dist/ + # For Test only ! + # password: ${{ secrets.TEST_PYPI_TOKEN }} + # repository-url: https://test.pypi.org/legacy/ + env: + name: pypi + url: https://pypi.org/p/opal-common/ + + - name: Publish package distributions to PyPI - Opal-Client + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + packages-dir: packages/opal-client/dist/ + # For Test only ! + # password: ${{ secrets.TEST_PYPI_TOKEN }} + # repository-url: https://test.pypi.org/legacy/ + env: + name: pypi + url: https://pypi.org/p/opal-client/ + + - name: Publish package distributions to PyPI - Opal-Server + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + packages-dir: packages/opal-server/dist/ + # For Test only ! + # password: ${{ secrets.TEST_PYPI_TOKEN }} + # repository-url: https://test.pypi.org/legacy/ + env: + name: pypi + url: https://pypi.org/p/opal-server/ diff --git a/.github/workflows/sync_opal_plus.yml b/.github/workflows/sync_opal_plus.yml new file mode 100644 index 000000000..ab92edaa6 --- /dev/null +++ b/.github/workflows/sync_opal_plus.yml @@ -0,0 +1,65 @@ +name: Sync branch to OPAL Plus + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + sync: + name: Sync branch to OPAL Plus + if: github.repository == 'permitio/opal' + runs-on: ubuntu-latest + steps: + - name: Set up Git configuration + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + - name: Get Token + id: get_workflow_token + uses: peter-murray/workflow-application-token-action@v1 + with: + application_id: ${{ secrets.APPLICATION_ID }} + application_private_key: ${{ secrets.APPLICATION_PRIVATE_KEY }} + + - name: Checkout permitio/opal repository + uses: actions/checkout@v4 + with: + repository: permitio/opal + ref: ${{ github.ref_name }} + path: opal + fetch-depth: 0 + + - name: Checkout permitio/opal-plus repository + uses: actions/checkout@v4 + with: + repository: permitio/opal-plus + path: opal-plus + token: ${{ steps.get_workflow_token.outputs.token }} + + - name: Create public-${{ github.ref_name }} branch in opal repository + working-directory: opal + run: | + git checkout -b public-${{ github.ref_name }} + + - name: Rebase opal-plus/public-${{ github.ref_name }} onto opal/${{ github.ref_name }} + working-directory: opal-plus + run: | + git remote add opal ../opal + git fetch opal + git checkout public-${{ github.ref_name }} + git rebase opal/${{ github.ref_name }} + + - name: Push changes to opal-plus/public-${{ github.ref_name }} branch + working-directory: opal-plus + run: | + git push origin public-${{ github.ref_name }} + + - name: Create Pull Request for opal-plus + working-directory: opal-plus + run: | + gh pr create --repo permitio/opal-plus --assignee "$GITHUB_ACTOR" --reviewer "$GITHUB_ACTOR" --base master --head public-${{ github.ref_name }} --title "Sync changes from public OPAL repository" --body "This PR synchronizes changes from the public OPAL repository to the private OPAL Plus repository." + env: + GITHUB_TOKEN: ${{ steps.get_workflow_token.outputs.token }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 610cdd05d..8638cba36 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,6 +66,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - name: Docker Compose install + run: | + curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + - name: Build client id: build_client uses: docker/build-push-action@v2 @@ -105,7 +110,8 @@ jobs: - name: Output container logs run: docker-compose -f docker/docker-compose-test.yml logs - - name: check if opal-client was brought up + - name: check if opal-client was brought up successfully run: | docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Connected to PubSub server" docker-compose -f docker/docker-compose-test.yml logs opal_client | grep "Got policy bundle" + docker-compose -f docker/docker-compose-test.yml logs opal_client | grep 'PUT /v1/data/static -> 204' diff --git a/cedar-agent b/cedar-agent index 1838635f1..687efc59e 160000 --- a/cedar-agent +++ b/cedar-agent @@ -1 +1 @@ -Subproject commit 1838635f16ba6db60d16c2ca28cb257e970bdff0 +Subproject commit 687efc59ecc732d1b98fc7789ab803abfc45b94c diff --git a/docker/Dockerfile b/docker/Dockerfile index bccdf3d2c..35f1144a6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # BUILD STAGE --------------------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM python:3.10-bookworm as BuildStage +FROM python:3.10-bookworm AS build-stage # from now on, work in the /app directory WORKDIR /app/ # Layer dependency install (for caching) @@ -15,19 +15,17 @@ RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -r ./ # CEDAR AGENT BUILD STAGE --------------------------- # split this stage to save time and reduce image size # --------------------------------------------------- -FROM rust:1.69.0 as cedar-builder -COPY cedar-agent /tmp/cedar-agent/ -ARG cargo_flags="-r" -RUN cd /tmp/cedar-agent && \ - cargo build ${cargo_flags} && \ - cp /tmp/cedar-agent/target/*/cedar-agent / +FROM rust:1.79 AS cedar-builder +COPY ./cedar-agent /tmp/cedar-agent +WORKDIR /tmp/cedar-agent +RUN CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse cargo build --release # COMMON IMAGE -------------------------------------- # --------------------------------------------------- -FROM python:3.10-slim-bookworm as common +FROM python:3.10-slim-bookworm AS common -# copy libraries from build stage (This won't copy redundant libraries we used in BuildStage) -COPY --from=BuildStage /usr/local /usr/local +# copy libraries from build stage (This won't copy redundant libraries we used in build-stage) +COPY --from=build-stage /usr/local /usr/local # Add non-root user (with home dir at /opal) RUN useradd -m -b / -s /bin/bash opal @@ -61,7 +59,7 @@ CMD ["./start.sh"] # STANDALONE IMAGE ---------------------------------- # --------------------------------------------------- -FROM common as client-standalone +FROM common AS client-standalone # uvicorn config ------------------------------------ # install the opal-client package RUN cd ./packages/opal-client && python setup.py install @@ -88,7 +86,7 @@ VOLUME /opal/backup # IMAGE to extract OPA from official image ---------- # --------------------------------------------------- -FROM alpine:latest as opa-extractor +FROM alpine:latest AS opa-extractor USER root RUN apk update && apk add skopeo tar @@ -106,7 +104,7 @@ RUN skopeo copy "docker://${opa_image}:${opa_tag}" docker-archive:./image.tar && # OPA CLIENT IMAGE ---------------------------------- # Using standalone image as base -------------------- # --------------------------------------------------- -FROM client-standalone as client +FROM client-standalone AS client # Temporarily move back to root for additional setup USER root @@ -123,13 +121,13 @@ USER opal # CEDAR CLIENT IMAGE -------------------------------- # Using standalone image as base -------------------- # --------------------------------------------------- -FROM client-standalone as client-cedar +FROM client-standalone AS client-cedar # Temporarily move back to root for additional setup USER root # Copy cedar from its build stage -COPY --from=cedar-builder /cedar-agent /bin/cedar-agent +COPY --from=cedar-builder /tmp/cedar-agent/target/*/cedar-agent /bin/cedar-agent # enable inline Cedar agent ENV OPAL_POLICY_STORE_TYPE=CEDAR @@ -142,9 +140,10 @@ USER opal # SERVER IMAGE -------------------------------------- # --------------------------------------------------- -FROM common as server +FROM common AS server RUN apt-get update && apt-get install -y openssh-client git && apt-get clean +RUN git config --global core.symlinks false # Mitigate CVE-2024-32002 USER opal diff --git a/docker/docker-compose-api-policy-source-example.yml b/docker/docker-compose-api-policy-source-example.yml index 466689a5a..219381598 100644 --- a/docker/docker-compose-api-policy-source-example.yml +++ b/docker/docker-compose-api-policy-source-example.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. @@ -37,7 +36,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true ports: # exposes opal server on the host machine, you can access the server at: http://localhost:7002 diff --git a/docker/docker-compose-example-cedar.yml b/docker/docker-compose-example-cedar.yml index a3170e575..38e5509a6 100644 --- a/docker/docker-compose-example-cedar.yml +++ b/docker/docker-compose-example-cedar.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-example.yml b/docker/docker-compose-example.yml index 13855734b..36c52db58 100644 --- a/docker/docker-compose-example.yml +++ b/docker/docker-compose-example.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-git-webhook.yml b/docker/docker-compose-git-webhook.yml index c5a394270..388ced755 100644 --- a/docker/docker-compose-git-webhook.yml +++ b/docker/docker-compose-git-webhook.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-scopes-example.yml b/docker/docker-compose-scopes-example.yml index 789ebea5a..9a3c1f162 100644 --- a/docker/docker-compose-scopes-example.yml +++ b/docker/docker-compose-scopes-example.yml @@ -1,4 +1,3 @@ -version: "3.8" services: redis: image: redis diff --git a/docker/docker-compose-with-callbacks.yml b/docker/docker-compose-with-callbacks.yml index 7ce3eacce..ca75903e6 100644 --- a/docker/docker-compose-with-callbacks.yml +++ b/docker/docker-compose-with-callbacks.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. @@ -32,7 +31,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true ports: # exposes opal server on the host machine, you can access the server at: http://localhost:7002 diff --git a/docker/docker-compose-with-kafka-example.yml b/docker/docker-compose-with-kafka-example.yml index 99b4129c7..1289e0592 100644 --- a/docker/docker-compose-with-kafka-example.yml +++ b/docker/docker-compose-with-kafka-example.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # Based on: https://developer.confluent.io/quickstart/kafka-docker/ @@ -70,7 +69,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true ports: # exposes opal server on the host machine, you can access the server at: http://localhost:7002 diff --git a/docker/docker-compose-with-oauth-initial.yml b/docker/docker-compose-with-oauth-initial.yml index 98b647e88..6a121e719 100644 --- a/docker/docker-compose-with-oauth-initial.yml +++ b/docker/docker-compose-with-oauth-initial.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. diff --git a/docker/docker-compose-with-oauth-jwt-token.yml b/docker/docker-compose-with-oauth-jwt-token.yml new file mode 100644 index 000000000..b62197241 --- /dev/null +++ b/docker/docker-compose-with-oauth-jwt-token.yml @@ -0,0 +1,93 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # JWT validation + - OPAL_OAUTH2_OPENID_CONFIGURATION_URL=https://example/.well-known/openid-configuration + - OPAL_OAUTH2_EXACT_MATCH_CLAIMS=aud=some_audience,iss=some_issuer + - OPAL_OAUTH2_REQUIRED_CLAIMS=sub,iat,exp + - OPAL_OAUTH2_JWT_ALGORITHM=RS256 + - OPAL_OAUTH2_JWT_AUDIENCE=some_audience + - OPAL_OAUTH2_JWT_ISSUER=https://example/issuer + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" diff --git a/docker/docker-compose-with-oauth-opaque-token.yml b/docker/docker-compose-with-oauth-opaque-token.yml new file mode 100644 index 000000000..7641cd0e8 --- /dev/null +++ b/docker/docker-compose-with-oauth-opaque-token.yml @@ -0,0 +1,83 @@ +services: + # When scaling the opal-server to multiple nodes and/or multiple workers, we use + # a *broadcast* channel to sync between all the instances of opal-server. + # Under the hood, this channel is implemented by encode/broadcaster (see link below). + # At the moment, the broadcast channel can be either: postgresdb, redis or kafka. + # The format of the broadcaster URI string (the one we pass to opal server as `OPAL_BROADCAST_URI`) is specified here: + # https://github.com/encode/broadcaster#available-backends + broadcast_channel: + image: postgres:alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + opal_server: + # by default we run opal-server from latest official image + image: permitio/opal-server:latest + environment: + # the broadcast backbone uri used by opal server workers (see comments above for: broadcast_channel) + - OPAL_BROADCAST_URI=postgres://postgres:postgres@broadcast_channel:5432/postgres + # number of uvicorn workers to run inside the opal-server container + - UVICORN_NUM_WORKERS=4 + # the git repo hosting our policy + # - if this repo is not public, you can pass an ssh key via `OPAL_POLICY_REPO_SSH_KEY`) + # - the repo we pass in this example is *public* and acts as an example repo with dummy rego policy + # - for more info, see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_URL=https://github.com/permitio/opal-example-policy-repo + # in this example we will use a polling interval of 30 seconds to check for new policy updates (git commits affecting the rego policy). + # however, it is better to utilize a git *webhook* to trigger the server to check for changes only when the repo has new commits. + # for more info see: https://docs.opal.ac/tutorials/track_a_git_repo + - OPAL_POLICY_REPO_POLLING_INTERVAL=30 + # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). + # the data sources represents from where the opal clients should get a "complete picture" of the data they need. + # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal_server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_LOG_FORMAT_INCLUDE_PID=true + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + ports: + # exposes opal server on the host machine, you can access the server at: http://localhost:7002 + - "7002:7002" + depends_on: + - broadcast_channel + opal_client: + # by default we run opal-client from latest official image + image: permitio/opal-client:latest + environment: + - OPAL_SERVER_URL=http://opal_server:7002 + - OPAL_LOG_FORMAT_INCLUDE_PID=true + - OPAL_INLINE_OPA_LOG_FORMAT=http + # to protect resources with OAuth2 Opaque token provided by dedicated server + - OPAL_AUTH_TYPE=oauth2 + # client credentials + - OPAL_OAUTH2_CLIENT_ID=some_client_id + - OPAL_OAUTH2_CLIENT_SECRET=some_client_secret + # URL to generate new OAuth 2.0 Client Credentials Grant token + - OPAL_OAUTH2_TOKEN_URL=https://example/oauth2/token + # introspect URL for Opaque token validation + - OPAL_OAUTH2_INTROSPECT_URL=https://example/oauth2/introspect + # Enable Authorization / Authentication in OPA + - 'OPAL_INLINE_OPA_CONFIG={"authentication":"token", "authorization":"basic", "files": ["authz.rego"]}' + volumes: + # The goal is to create an initial authorization rego that allows OPAL to write the first policy from the POLICY_REPO_URL. + # This is achieved through policy overwrite based on the "id" attribute. + # When the authz.rego file is placed in the root directory of OPA, it is given the id 'authz.rego'. + # Similarly, if there is another authz.rego file in the root of POLICY_REPO_URL, it will also be given the id 'authz.rego'. + # Therefore, if the authz.rego file from the POLICY_REPO_URL exists, it will overwrite the initial authz.rego file. + - ./docker_files/policy_test/authz.rego:/opal/authz.rego + ports: + # exposes opal client on the host machine, you can access the client at: http://localhost:7766 + - "7766:7000" + # exposes the OPA agent (being run by OPAL) on the host machine + # you can access the OPA api that you know and love at: http://localhost:8181 + # OPA api docs are at: https://www.openpolicyagent.org/docs/latest/rest-api/ + - "8181:8181" + depends_on: + - opal_server + # this command is not necessary when deploying OPAL for real, it is simply a trick for dev environments + # to make sure that opal-server is already up before starting the client. + command: sh -c "exec ./wait-for.sh opal_server:7002 --timeout=20 -- ./start.sh" diff --git a/docker/docker-compose-with-rate-limiting.yml b/docker/docker-compose-with-rate-limiting.yml index a0fd3bbcc..6f10caf5e 100644 --- a/docker/docker-compose-with-rate-limiting.yml +++ b/docker/docker-compose-with-rate-limiting.yml @@ -1,5 +1,4 @@ # This docker compose example shows how to configure OPAL's rate limiting feature -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. @@ -31,7 +30,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # Turns on rate limiting in the server # supported formats documented here: https://limits.readthedocs.io/en/stable/quickstart.html#rate-limit-string-notation diff --git a/docker/docker-compose-with-security.yml b/docker/docker-compose-with-security.yml index 8a80ad2a6..2c27a711f 100644 --- a/docker/docker-compose-with-security.yml +++ b/docker/docker-compose-with-security.yml @@ -1,6 +1,5 @@ # this docker compose file is relying on external environment variables! # run it by running the script: ./run-example-with-security.sh -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. @@ -46,7 +45,7 @@ services: # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. # please notice - since we fetch data entries from the OPAL server itself, we need to authenticate to that endpoint # with the client's token (JWT). - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","config":{"headers":{"Authorization":"Bearer ${OPAL_CLIENT_TOKEN}"}},"topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # -------------------------------------------------------------------------------- # the jwt audience and jwt issuer are not typically necessary in real setups diff --git a/docker/docker-compose-with-statistics.yml b/docker/docker-compose-with-statistics.yml index 0c15d5a95..eb26daef4 100644 --- a/docker/docker-compose-with-statistics.yml +++ b/docker/docker-compose-with-statistics.yml @@ -1,4 +1,3 @@ -version: "3.8" services: # When scaling the opal-server to multiple nodes and/or multiple workers, we use # a *broadcast* channel to sync between all the instances of opal-server. @@ -32,7 +31,7 @@ services: # configures from where the opal client should initially fetch data (when it first goes up, after disconnection, etc). # the data sources represents from where the opal clients should get a "complete picture" of the data they need. # after the initial sources are fetched, the client will subscribe only to update notifications sent by the server. - - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://host.docker.internal:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} + - OPAL_DATA_CONFIG_SOURCES={"config":{"entries":[{"url":"http://opal-server:7002/policy-data","topics":["policy_data"],"dst_path":"/static"}]}} - OPAL_LOG_FORMAT_INCLUDE_PID=true # turning on statistics collection on the server side - OPAL_STATISTICS_ENABLED=true diff --git a/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx b/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx index 1d11235e3..f404a6c3b 100644 --- a/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx +++ b/documentation/docs/getting-started/quickstart/docker-compose-config/overview.mdx @@ -11,7 +11,6 @@ This example is running three containers that we have mentioned at the beginning Here is an overview of the whole `docker-compose.yml` file, but don't worry, we will be referring to each section separately. ```yml showLineNumbers -version: "3.8" services: broadcast_channel: image: postgres:alpine diff --git a/documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx b/documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx index f0c632f58..24e9a3461 100644 --- a/documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx +++ b/documentation/docs/getting-started/quickstart/opal-playground/updating-the-policy.mdx @@ -35,7 +35,6 @@ opal_server: You can also simply change the tracked repo in the example `docker-compose.yml` file by editing these variables: ```yml {7,9,11} showLineNumbers -version: "3.8" services: ... opal_server: diff --git a/documentation/docs/opal-plus/deploy.mdx b/documentation/docs/opal-plus/deploy.mdx new file mode 100644 index 000000000..7448b37f7 --- /dev/null +++ b/documentation/docs/opal-plus/deploy.mdx @@ -0,0 +1,34 @@ +--- +sidebar_position: 2 +title: Deploy OPAL+ +--- + +With OPAL+, you get access to private Docker images that include additional features and capabilities. +To apply for Permit OPAL+, [fill in the form available here](https://hello.permit.io/opal-plus) + +In order to access the OPAL+ Docker images, you need to have Docker Hub credentials with an access token. +Those should be received from your Customer Success manager. +Reach out to us [on Slack](https://bit.ly/permit-slack) if you need assistance. + +## Accessing the OPAL+ Docker Images + +To access the OPAL+ Docker images, you need to log in to Docker Hub with your credentials. +You can do this by running the [docker login](https://docs.docker.com/reference/cli/docker/login/) command: + +```bash +docker login -u -p +``` + +If you are using Kubernetes, check out the [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) on how to pull images from a private registry. + +After logging in, you can pull the OPAL+ Docker images using the following commands: + +```bash +docker pull permitio/opal-plus:latest +``` + +## Running the OPAL+ Docker Images + +Running the OPAL+ Docker images is similar to running the open-source OPAL images. + +Check out the [OPAL Docker documentation](/getting-started/running-opal/run-docker-containers) for more information. diff --git a/documentation/docs/opal-plus/features.mdx b/documentation/docs/opal-plus/features.mdx new file mode 100644 index 000000000..bc3a0249f --- /dev/null +++ b/documentation/docs/opal-plus/features.mdx @@ -0,0 +1,80 @@ +--- +sidebar_position: 3 +title: Advanced Features +--- + +OPAL+ has a number of advanced features extending the capabilities of the open-source OPAL. +These features are available to OPAL+ users only. + +To apply for Permit OPAL+, [fill in the form available here](https://hello.permit.io/opal-plus) + + +## Support and SLA + +OPAL+ provides dedicated support and a custom SLA to help you get the most out of your OPAL+ deployment. +Reach out to us [on Slack](https://bit.ly/permit-slack) for more information. + +## Licensing Capabilities + +OPAL+ provides additional licensing capabilities to help you manage your OPAL+ deployment. +Reach out to us [on Slack](https://bit.ly/permit-slack) for more information. + +## Custom Data Fetcher Providers + +OPAL+ provides custom data fetcher providers to help you fetch data from your private data sources. + +## Logging and Monitoring + +OPAL+ provides advanced logging and monitoring capabilities to help you track and debug your OPAL+ deployment. + +#### Connect to logging system + +On production, we advise you to connect OPAL+ to your logging system to collect and store the logs. +Configure the [OPAL_LOG_SERIALIZE](/getting-started/configuration) environment variable to `true` to serialize logs in JSON format. + +#### Monitor OPAL Servers and Clients + +OPAL+ provides monitoring endpoints to help you track the health of your OPAL+ servers and clients. +Configure the [OPAL_STATISTICS_ENABLED=true](/getting-started/configuration) environment variable to enable the statistics APIs. + +You can then monitor the state of your OPAL+ cluster by calling the `/stats` API route on the server. +```bash +curl http://opal-server:8181/stats -H "Authorization: Bearer " +# { "uptime": "2024-07-14T14:55:02.710Z", "version": "0.7.8", "client_count": 1, "server_count": 1 } +``` + +You can also get detailed information about the OPAL+ clients and servers by calling the `/statistics` API route on the server. +```bash +curl http://opal-server:8181/statistics -H "Authorization: Bear " +``` +```json +{ + "uptime": "2024-07-14T14:54:09.809Z", + "version": "0.7.8", + "clients": { + "opal-client-1": [ + { + "rpc_id": "7ba198b1329d439faaa79dd7447401dc", + "client_id": "693ac1b4d060416eaad50c2bf04121b1", + "topics": [ + "string" + ] + } + ], + "opal-client-2": [ + { + "rpc_id": "d343d92292794630994a8a077bcb413a", + "client_id": "4d71d88ba16f49e1a0ae89f16c5a55d5", + "topics": [ + "string" + ] + } + ] + }, + "servers": [ + "774b376fbead49b79f6a9fd42cef2cfd" + ] +} +``` + +For more information on monitoring OPAL, see the [Monitoring OPAL](/tutorials/monitoring_opal) tutorial. diff --git a/documentation/docs/OPAL_PLUS.mdx b/documentation/docs/opal-plus/introduction.mdx similarity index 84% rename from documentation/docs/OPAL_PLUS.mdx rename to documentation/docs/opal-plus/introduction.mdx index 8ca739539..08b609d8a 100644 --- a/documentation/docs/OPAL_PLUS.mdx +++ b/documentation/docs/opal-plus/introduction.mdx @@ -1,8 +1,9 @@ --- sidebar_position: 1 -title: Permit OPAL+ (Extended OPAL License) +title: Introduction --- +
=8" @@ -6394,9 +6396,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -10380,11 +10382,11 @@ ] }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", + "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { @@ -14776,9 +14778,9 @@ } }, "node_modules/webpack-dev-server/node_modules/ws": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz", - "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "engines": { "node": ">=10.0.0" }, @@ -15029,9 +15031,9 @@ } }, "node_modules/ws": { - "version": "7.5.9", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", - "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", "engines": { "node": ">=8.3.0" }, diff --git a/documentation/package.json b/documentation/package.json index 32320080d..89ac97334 100644 --- a/documentation/package.json +++ b/documentation/package.json @@ -23,7 +23,9 @@ "prism-react-renderer": "^2.1.0", "react": "^18.0.0", "react-dom": "^18.0.0", - "sass": "^1.71.1" + "sass": "^1.71.1", + "axios": "^1.6.4", + "micromatch": "^4.0.6" }, "devDependencies": { "@docusaurus/module-type-aliases": "3.0.0" diff --git a/documentation/sidebars.js b/documentation/sidebars.js index 394dd70ff..b6192be4d 100644 --- a/documentation/sidebars.js +++ b/documentation/sidebars.js @@ -283,14 +283,20 @@ const sidebars = { label: "Fetch Providers", }, { - type: "doc", - id: "FAQ", - label: "FAQ", + type: "category", + label: "💎 OPAL+ (Extended License)", + collapsed: true, + items: [ + { + type: "autogenerated", + dirName: "opal-plus", + }, + ], }, { type: "doc", - id: "OPAL_PLUS", - label: "OPAL + (Extended OPAL License)", + id: "FAQ", + label: "FAQ", }, ], }; diff --git a/packages/__packaging__.py b/packages/__packaging__.py index 727191f80..d56cb9982 100644 --- a/packages/__packaging__.py +++ b/packages/__packaging__.py @@ -6,9 +6,10 @@ Project homepage: https://github.com/permitio/opal """ + import os -VERSION = (0, 7, 7) +VERSION = (0, 0, 0) # Placeholder, to be set by CI/CD VERSION_STRING = ".".join(map(str, VERSION)) __version__ = VERSION_STRING diff --git a/packages/opal-client/opal_client/__init__.py b/packages/opal-client/opal_client/__init__.py index c2810c5f7..a1eb3e09d 100644 --- a/packages/opal-client/opal_client/__init__.py +++ b/packages/opal-client/opal_client/__init__.py @@ -1 +1 @@ -from .client import OpalClient +from opal_client.client import OpalClient diff --git a/packages/opal-client/opal_client/callbacks/api.py b/packages/opal-client/opal_client/callbacks/api.py index 49cb0853a..b1e22d7f1 100644 --- a/packages/opal-client/opal_client/callbacks/api.py +++ b/packages/opal-client/opal_client/callbacks/api.py @@ -3,8 +3,8 @@ from fastapi import APIRouter, Depends, HTTPException, Response, status from opal_client.callbacks.register import CallbacksRegister from opal_client.config import opal_client_config +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -13,7 +13,7 @@ from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR -def init_callbacks_api(authenticator: JWTAuthenticator, register: CallbacksRegister): +def init_callbacks_api(authenticator: Authenticator, register: CallbacksRegister): async def require_listener_token(claims: JWTClaims = Depends(authenticator)): try: require_peer_type( diff --git a/packages/opal-client/opal_client/callbacks/reporter.py b/packages/opal-client/opal_client/callbacks/reporter.py index 264f45b51..c9f2987b6 100644 --- a/packages/opal-client/opal_client/callbacks/reporter.py +++ b/packages/opal-client/opal_client/callbacks/reporter.py @@ -5,7 +5,7 @@ from opal_client.callbacks.register import CallbackConfig, CallbacksRegister from opal_client.data.fetcher import DataFetcher from opal_common.fetcher.providers.http_fetch_provider import HttpFetcherConfig -from opal_common.http import is_http_error_response +from opal_common.http_utils import is_http_error_response from opal_common.logger import logger from opal_common.schemas.data import DataUpdateReport diff --git a/packages/opal-client/opal_client/client.py b/packages/opal-client/opal_client/client.py index be8e5ca49..9b828cced 100644 --- a/packages/opal-client/opal_client/client.py +++ b/packages/opal-client/opal_client/client.py @@ -29,8 +29,7 @@ from opal_client.policy_store.policy_store_client_factory import ( PolicyStoreClientFactory, ) -from opal_common.authentication.deps import JWTAuthenticator -from opal_common.authentication.verifier import JWTVerifier +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger from opal_common.middleware import configure_middleware @@ -49,7 +48,7 @@ def __init__( inline_opa_options: OpaServerOptions = None, inline_cedar_enabled: bool = None, inline_cedar_options: CedarServerOptions = None, - verifier: Optional[JWTVerifier] = None, + authenticator: Optional[ClientAuthenticator] = None, store_backup_path: Optional[str] = None, store_backup_interval: Optional[int] = None, offline_mode_enabled: bool = False, @@ -64,6 +63,10 @@ def __init__( data_updater (DataUpdater, optional): Defaults to None. policy_updater (PolicyUpdater, optional): Defaults to None. """ + if authenticator is not None: + self.authenticator = authenticator + else: + self.authenticator = ClientAuthenticator() self._shard_id = shard_id # defaults policy_store_type: PolicyStoreTypes = ( @@ -119,6 +122,7 @@ def __init__( policy_store=self.policy_store, callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, + authenticator=self.authenticator, ) else: self.policy_updater = None @@ -140,6 +144,7 @@ def __init__( callbacks_register=self._callbacks_register, opal_client_id=opal_client_identifier, shard_id=self._shard_id, + authenticator=self.authenticator, ) else: self.data_updater = None @@ -162,19 +167,6 @@ def __init__( "OPAL client is configured to trust self-signed certificates" ) - if verifier is not None: - self.verifier = verifier - else: - self.verifier = JWTVerifier( - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if not self.verifier.enabled: - logger.info( - "API authentication disabled (public encryption key was not provided)" - ) self.store_backup_path = ( store_backup_path or opal_client_config.STORE_BACKUP_PATH ) @@ -250,13 +242,11 @@ def _init_fast_api_app(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.verifier) - # Init api routers with required dependencies policy_router = init_policy_router(policy_updater=self.policy_updater) data_router = init_data_router(data_updater=self.data_updater) - policy_store_router = init_policy_store_router(authenticator) - callbacks_router = init_callbacks_api(authenticator, self._callbacks_register) + policy_store_router = init_policy_store_router(self.authenticator) + callbacks_router = init_callbacks_api(self.authenticator, self._callbacks_register) # mount the api routes on the app object app.include_router(policy_router, tags=["Policy Updater"]) diff --git a/packages/opal-client/opal_client/config.py b/packages/opal-client/opal_client/config.py index 3f9f72125..58d7ae2c8 100644 --- a/packages/opal-client/opal_client/config.py +++ b/packages/opal-client/opal_client/config.py @@ -115,6 +115,12 @@ class OpalClientConfig(Confi): description="path to the file containing the ca certificate(s) used for tls authentication with the policy store", ) + EXCLUDE_POLICY_STORE_SECRETS = confi.bool( + "EXCLUDE_POLICY_STORE_SECRETS", + False, + description="If set, policy store secrets will be excluded from the /policy-store/config route", + ) + # create an instance of a policy store upon load def load_policy_store(): from opal_client.policy_store.policy_store_client_factory import ( diff --git a/packages/opal-client/opal_client/data/updater.py b/packages/opal-client/opal_client/data/updater.py index d2c81c9ed..444219e34 100644 --- a/packages/opal-client/opal_client/data/updater.py +++ b/packages/opal-client/opal_client/data/updater.py @@ -24,9 +24,10 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool, repeated_call +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.fetcher.events import FetcherConfig -from opal_common.http import is_http_error_response +from opal_common.http_utils import is_http_error_response from opal_common.schemas.data import ( DataEntryReport, DataSourceConfig, @@ -54,6 +55,7 @@ def __init__( callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, shard_id: Optional[str] = None, + authenticator: Optional[ClientAuthenticator] = None, ): """Keeps policy-stores (e.g. OPA) up to date with relevant data Obtains data configuration on startup from OPAL-server Uses Pub/Sub to @@ -110,17 +112,18 @@ def __init__( self._callbacks_register, ) self._token = token + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None self._shard_id = shard_id self._server_url = pubsub_url self._data_sources_config_url = data_sources_config_url self._opal_client_id = opal_client_id - self._extra_headers = [] + self._extra_headers = {} if self._token is not None: - self._extra_headers.append(get_authorization_header(self._token)) + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] if self._shard_id is not None: - self._extra_headers.append(("X-Shard-ID", self._shard_id)) - if len(self._extra_headers) == 0: - self._extra_headers = None + self._extra_headers['X-Shard-ID'] = self._shard_id self._stopping = False # custom SSL context (for self-signed certificates) self._custom_ssl_context = get_custom_ssl_context() @@ -132,6 +135,10 @@ def __init__( self._updates_storing_queue = TakeANumberQueue(logger) self._tasks = TasksPool() self._polling_update_tasks = [] + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() async def __aenter__(self): await self.start() @@ -177,8 +184,14 @@ async def get_policy_data_config(self, url: str = None) -> DataSourceConfig: if url is None: url = self._data_sources_config_url logger.info("Getting data-sources configuration from '{source}'", source=url) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + try: - async with ClientSession(headers=self._extra_headers) as session: + async with ClientSession(headers=headers) as session: response = await session.get(url, **self._ssl_context_kwargs) if response.status == 200: return DataSourceConfig.parse_obj(await response.json()) @@ -274,12 +287,19 @@ async def _subscriber(self): """Coroutine meant to be spunoff with create_task to listen in the background for data events and pass them to the data_fetcher.""" logger.info("Subscribing to topics: {topics}", topics=self._data_topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( - self._data_topics, - self._update_policy_data_callback, + topics=self._data_topics, + callback=self._update_policy_data_callback, methods_class=TenantAwareRpcEventClientMethods, on_connect=[self.on_connect], - extra_headers=self._extra_headers, + on_disconnect=[self.on_disconnect], + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/main.py b/packages/opal-client/opal_client/main.py index 611cdd741..65f3bb665 100644 --- a/packages/opal-client/opal_client/main.py +++ b/packages/opal-client/opal_client/main.py @@ -1,4 +1,4 @@ -from .client import OpalClient +from opal_client.client import OpalClient client = OpalClient() # expose app for Uvicorn diff --git a/packages/opal-client/opal_client/policy/fetcher.py b/packages/opal-client/opal_client/policy/fetcher.py index a435370b1..5ae9d93b6 100644 --- a/packages/opal-client/opal_client/policy/fetcher.py +++ b/packages/opal-client/opal_client/policy/fetcher.py @@ -4,6 +4,7 @@ from fastapi import HTTPException, status from opal_client.config import opal_client_config from opal_client.logger import logger +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.schemas.policy import PolicyBundle from opal_common.security.sslcontext import get_custom_ssl_context from opal_common.utils import ( @@ -28,15 +29,26 @@ def force_valid_bundle(bundle) -> PolicyBundle: class PolicyFetcher: """fetches policy from backend.""" - def __init__(self, backend_url=None, token=None): + def __init__( + self, + backend_url=None, + token=None, + authenticator: Optional[ClientAuthenticator] = None, + ): """ Args: backend_url (str): Defaults to opal_client_config.SERVER_URL. token ([type], optional): [description]. Defaults to opal_client_config.CLIENT_TOKEN. """ + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() self._token = token or opal_client_config.CLIENT_TOKEN self._backend_url = backend_url or opal_client_config.SERVER_URL - self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) + self._auth_headers = {} + if self._token != "THIS_IS_A_DEV_SECRET": + self._auth_headers = tuple_to_dict(get_authorization_header(self._token)) self._retry_config = ( opal_client_config.POLICY_UPDATER_CONN_RETRY.toTenacityConfig() @@ -82,10 +94,15 @@ async def _fetch_policy_bundle( May throw, in which case we retry again. """ + headers = {} + if self._auth_headers is not None: + headers = self._auth_headers.copy() + await self._authenticator.authenticate(headers) + params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash - async with aiohttp.ClientSession() as session: + async with aiohttp.ClientSession(headers=headers) as session: logger.info( "Fetching policy bundle from {url}", url=self._policy_endpoint_url, @@ -95,7 +112,6 @@ async def _fetch_policy_bundle( self._policy_endpoint_url, headers={ "content-type": "text/plain", - **self._auth_headers, }, params=params, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy/updater.py b/packages/opal-client/opal_client/policy/updater.py index 57d93099f..d505c52f5 100644 --- a/packages/opal-client/opal_client/policy/updater.py +++ b/packages/opal-client/opal_client/policy/updater.py @@ -16,6 +16,7 @@ DEFAULT_POLICY_STORE_GETTER, ) from opal_common.async_utils import TakeANumberQueue, TasksPool +from opal_common.authentication.authenticator import ClientAuthenticator from opal_common.config import opal_common_config from opal_common.schemas.data import DataUpdateReport from opal_common.schemas.policy import PolicyBundle, PolicyUpdateMessage @@ -43,6 +44,7 @@ def __init__( data_fetcher: Optional[DataFetcher] = None, callbacks_register: Optional[CallbacksRegister] = None, opal_client_id: str = None, + authenticator: Optional[ClientAuthenticator] = None, ): """inits the policy updater. @@ -64,15 +66,21 @@ def __init__( self._opal_client_id = opal_client_id self._scope_id = opal_client_config.SCOPE_ID + if authenticator is not None: + self._authenticator = authenticator + else: + self._authenticator = ClientAuthenticator() # The policy store we'll save policy modules into (i.e: OPA) self._policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() # pub/sub server url and authentication data self._server_url = pubsub_url self._token = token - if self._token is None: - self._extra_headers = None - else: - self._extra_headers = [get_authorization_header(self._token)] + if self._token == "THIS_IS_A_DEV_SECRET": + self._token = None + self._extra_headers = {} + if self._token is not None: + auth_token = get_authorization_header(self._token) + self._extra_headers[auth_token[0]] = auth_token[1] # Pub/Sub topics we subscribe to for policy updates if self._scope_id == "default": self._topics = pubsub_topics_from_directories( @@ -87,7 +95,7 @@ def __init__( self._policy_update_task = None self._stopping = False # policy fetcher - fetches policy bundles - self._policy_fetcher = PolicyFetcher() + self._policy_fetcher = PolicyFetcher(authenticator=self._authenticator) # callbacks on policy changes self._data_fetcher = data_fetcher or DataFetcher() self._callbacks_register = callbacks_register or CallbacksRegister() @@ -240,12 +248,18 @@ async def _subscriber(self): update_policy() callback (which will fetch the relevant policy bundle from the server and update the policy store).""" logger.info("Subscribing to topics: {topics}", topics=self._topics) + + headers = {} + if self._extra_headers is not None: + headers = self._extra_headers.copy() + await self._authenticator.authenticate(headers) + self._client = PubSubClient( topics=self._topics, callback=self._update_policy_callback, on_connect=[self._on_connect], on_disconnect=[self._on_disconnect], - extra_headers=self._extra_headers, + extra_headers=headers, keep_alive=opal_client_config.KEEP_ALIVE_INTERVAL, server_uri=self._server_url, **self._ssl_context_kwargs, diff --git a/packages/opal-client/opal_client/policy_store/api.py b/packages/opal-client/opal_client/policy_store/api.py index 3c7428644..97113f109 100644 --- a/packages/opal-client/opal_client/policy_store/api.py +++ b/packages/opal-client/opal_client/policy_store/api.py @@ -1,21 +1,23 @@ from fastapi import APIRouter, Depends from opal_client.config import opal_client_config from opal_client.policy_store.schemas import PolicyStoreAuth, PolicyStoreDetails +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.authz import require_peer_type -from opal_common.authentication.deps import JWTAuthenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger from opal_common.schemas.security import PeerType -def init_policy_store_router(authenticator: JWTAuthenticator): +def init_policy_store_router(authenticator: Authenticator): router = APIRouter() @router.get( "/policy-store/config", response_model=PolicyStoreDetails, response_model_exclude_none=True, + # Deprecating this route + deprecated=True, ) async def get_policy_store_details(claims: JWTClaims = Depends(authenticator)): try: @@ -26,14 +28,20 @@ async def get_policy_store_details(claims: JWTClaims = Depends(authenticator)): logger.error(f"Unauthorized to publish update: {repr(e)}") raise + token = None + oauth_client_secret = None + if not opal_client_config.EXCLUDE_POLICY_STORE_SECRETS: + token = opal_client_config.POLICY_STORE_AUTH_TOKEN + oauth_client_secret = ( + opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET + ) return PolicyStoreDetails( url=opal_client_config.POLICY_STORE_URL, - token=opal_client_config.POLICY_STORE_AUTH_TOKEN or None, + token=token or None, auth_type=opal_client_config.POLICY_STORE_AUTH_TYPE or PolicyStoreAuth.NONE, oauth_client_id=opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_ID or None, - oauth_client_secret=opal_client_config.POLICY_STORE_AUTH_OAUTH_CLIENT_SECRET - or None, + oauth_client_secret=oauth_client_secret or None, oauth_server=opal_client_config.POLICY_STORE_AUTH_OAUTH_SERVER or None, ) diff --git a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py index 4aa27f0d0..549dd8435 100644 --- a/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py +++ b/packages/opal-client/opal_client/policy_store/mock_policy_store_client.py @@ -3,13 +3,15 @@ from typing import Any, Dict, List, Optional import jsonpatch +from opal_client.policy_store.base_policy_store_client import ( + BasePolicyStoreClient, + JsonableValue, +) from opal_client.utils import exclude_none_fields from opal_common.schemas.policy import PolicyBundle from opal_common.schemas.store import JSONPatchAction, StoreTransaction from pydantic import BaseModel -from .base_policy_store_client import BasePolicyStoreClient, JsonableValue - class MockPolicyStoreClient(BasePolicyStoreClient): """A naive mock policy and policy-data store for tests.""" diff --git a/packages/opal-client/opal_client/policy_store/opa_client.py b/packages/opal-client/opal_client/policy_store/opa_client.py index 1d8ec0211..54bc94dac 100644 --- a/packages/opal-client/opal_client/policy_store/opa_client.py +++ b/packages/opal-client/opal_client/policy_store/opa_client.py @@ -20,7 +20,7 @@ from opal_client.policy_store.schemas import PolicyStoreAuth from opal_client.utils import exclude_none_fields, proxy_response from opal_common.engine.parsing import get_rego_package -from opal_common.git.bundle_utils import BundleUtils +from opal_common.git_utils.bundle_utils import BundleUtils from opal_common.paths import PathUtils from opal_common.schemas.policy import DataModule, PolicyBundle, RegoModule from opal_common.schemas.store import JSONPatchAction, StoreTransaction, TransactionType diff --git a/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py b/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py index 3e6522277..a3372c56f 100644 --- a/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py +++ b/packages/opal-client/opal_client/tests/server_to_client_intergation_test.py @@ -31,7 +31,7 @@ from opal_server.server import OpalServer # Server settings -PORT = int(os.environ.get("PORT") or "9123") +PORT = int(os.environ.get("PORT") or "9124") UPDATES_URL = f"ws://localhost:{PORT}/ws" DATA_ROUTE = "/fetchable_data" DATA_URL = f"http://localhost:{PORT}{DATA_ROUTE}" diff --git a/packages/opal-common/opal_common/async_utils.py b/packages/opal-common/opal_common/async_utils.py index e7f6029fd..a2df90c69 100644 --- a/packages/opal-common/opal_common/async_utils.py +++ b/packages/opal-common/opal_common/async_utils.py @@ -22,9 +22,12 @@ async def run_sync( """Shorthand for running a sync function in an executor within an async context. - For example: def sync_function_that_takes_time_to_run(arg1, - arg2): time.sleep(5) async def async_function(): - await run_sync(sync_function_that_takes_time_to_run, 1, arg2=5) + For example: + def sync_function_that_takes_time_to_run(arg1, arg2): + time.sleep(5) + + async def async_function(): + await run_sync(sync_function_that_takes_time_to_run, 1, arg2=5) """ return await asyncio.get_event_loop().run_in_executor( None, partial(func, *args, **kwargs) diff --git a/packages/opal-common/opal_common/authentication/authenticator.py b/packages/opal-common/opal_common/authentication/authenticator.py new file mode 100644 index 000000000..938ac001f --- /dev/null +++ b/packages/opal-common/opal_common/authentication/authenticator.py @@ -0,0 +1,50 @@ +from abc import abstractmethod +from typing import Optional + +from fastapi import Header +from opal_common.config import opal_common_config +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from .oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator + +class Authenticator: + @property + def enabled(self): + return self._delegate().enabled + + async def authenticate(self, headers): + if hasattr(self._delegate(), "authenticate") and callable(getattr(self._delegate(), "authenticate")): + await self._delegate().authenticate(headers) + + @abstractmethod + def _delegate(self) -> dict: + pass + +class _ClientAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will authenticate API requests.") + else: + self.__delegate = JWTAuthenticator(self.__verifier()) + + def __verifier(self) -> JWTVerifier: + verifier = JWTVerifier( + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if not verifier.enabled: + logger.info("API authentication disabled (public encryption key was not provided)") + + return verifier + + def _delegate(self) -> dict: + return self.__delegate + +class ClientAuthenticator(_ClientAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) diff --git a/packages/opal-common/opal_common/authentication/authz.py b/packages/opal-common/opal_common/authentication/authz.py index 742304bf5..822497e64 100644 --- a/packages/opal-common/opal_common/authentication/authz.py +++ b/packages/opal-common/opal_common/authentication/authz.py @@ -1,4 +1,4 @@ -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.schemas.data import DataUpdate @@ -6,7 +6,7 @@ def require_peer_type( - authenticator: JWTAuthenticator, claims: JWTClaims, required_type: PeerType + authenticator: Authenticator, claims: JWTClaims, required_type: PeerType ): if not authenticator.enabled: return @@ -28,7 +28,7 @@ def require_peer_type( def restrict_optional_topics_to_publish( - authenticator: JWTAuthenticator, claims: JWTClaims, update: DataUpdate + authenticator: Authenticator, claims: JWTClaims, update: DataUpdate ): if not authenticator.enabled: return diff --git a/packages/opal-common/opal_common/authentication/casting.py b/packages/opal-common/opal_common/authentication/casting.py index 9713ed2d5..d14a04fc7 100644 --- a/packages/opal-common/opal_common/authentication/casting.py +++ b/packages/opal-common/opal_common/authentication/casting.py @@ -5,7 +5,6 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey -from opal_common.logging.decorators import log_exception logger = logging.getLogger("opal.authentication") diff --git a/packages/opal-common/opal_common/authentication/jwk.py b/packages/opal-common/opal_common/authentication/jwk.py new file mode 100644 index 000000000..182b5cdb9 --- /dev/null +++ b/packages/opal-common/opal_common/authentication/jwk.py @@ -0,0 +1,45 @@ +import jwt +import httpx + +from cachetools import TTLCache +from opal_common.authentication.verifier import Unauthorized + +class JWKManager: + def __init__(self, openid_configuration_url, jwt_algorithm, cache_maxsize, cache_ttl): + self._openid_configuration_url = openid_configuration_url + self._jwt_algorithm = jwt_algorithm + self._cache = TTLCache(maxsize=cache_maxsize, ttl=cache_ttl) + + def public_key(self, token): + header = jwt.get_unverified_header(token) + kid = header['kid'] + + public_key = self._cache.get(kid) + if public_key is None: + public_key = self._fetch_public_key(token) + self._cache[kid] = public_key + + return public_key + + def _fetch_public_key(self, token: str): + try: + return self._jwks_client().get_signing_key_from_jwt(token).key + except Exception: + raise Unauthorized(description="unknown JWT error") + + def _jwks_client(self): + oidc_config = self._openid_configuration() + signing_algorithms = oidc_config["id_token_signing_alg_values_supported"] + if self._jwt_algorithm.name not in signing_algorithms: + raise Unauthorized(description="unknown JWT algorithm") + if "jwks_uri" not in oidc_config: + raise Unauthorized(description="missing 'jwks_uri' property") + return jwt.PyJWKClient(oidc_config["jwks_uri"]) + + def _openid_configuration(self): + response = httpx.get(self._openid_configuration_url) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + return response.json() diff --git a/packages/opal-common/opal_common/authentication/oauth2.py b/packages/opal-common/opal_common/authentication/oauth2.py new file mode 100644 index 000000000..aad738b66 --- /dev/null +++ b/packages/opal-common/opal_common/authentication/oauth2.py @@ -0,0 +1,156 @@ +import asyncio +import httpx +import time + +from cachetools import cached, TTLCache +from fastapi import Header +from httpx import AsyncClient, BasicAuth +from opal_common.authentication.deps import get_token_from_header +from opal_common.authentication.jwk import JWKManager +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.config import opal_common_config +from typing import Optional + +class _OAuth2Authenticator: + async def authenticate(self, headers): + if "Authorization" not in headers: + token = await self.token() + headers['Authorization'] = f"Bearer {token}" + + +class OAuth2ClientCredentialsAuthenticator(_OAuth2Authenticator): + def __init__(self) -> None: + self._client_id = opal_common_config.OAUTH2_CLIENT_ID + self._client_secret = opal_common_config.OAUTH2_CLIENT_SECRET + self._token_url = opal_common_config.OAUTH2_TOKEN_URL + self._introspect_url = opal_common_config.OAUTH2_INTROSPECT_URL + self._jwt_algorithm = opal_common_config.OAUTH2_JWT_ALGORITHM + self._jwt_audience = opal_common_config.OAUTH2_JWT_AUDIENCE + self._jwt_issuer = opal_common_config.OAUTH2_JWT_ISSUER + self._jwk_manager = JWKManager( + opal_common_config.OAUTH2_OPENID_CONFIGURATION_URL, + opal_common_config.OAUTH2_JWT_ALGORITHM, + opal_common_config.OAUTH2_JWK_CACHE_MAXSIZE, + opal_common_config.OAUTH2_JWK_CACHE_TTL, + ) + + cfg = opal_common_config.OAUTH2_EXACT_MATCH_CLAIMS + if cfg is None: + self._exact_match_claims = {} + else: + self._exact_match_claims = dict(map(lambda x: x.split("="), cfg.split(","))) + + cfg = opal_common_config.OAUTH2_REQUIRED_CLAIMS + if cfg is None: + self._required_claims = [] + else: + self._required_claims = cfg.split(",") + + @property + def enabled(self): + return True + + async def token(self): + auth = BasicAuth(self._client_id, self._client_secret) + data = {"grant_type": "client_credentials"} + + async with AsyncClient() as client: + response = await client.post(self._token_url, auth=auth, data=data) + return (response.json())['access_token'] + + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + token = get_token_from_header(authorization) + return self.verify(token) + + def verify(self, token: str) -> {}: + if self._introspect_url is not None: + claims = self._verify_opaque(token) + else: + claims = self._verify_jwt(token) + + self._verify_exact_match_claims(claims) + self._verify_required_claims(claims) + + return claims + + def _verify_opaque(self, token: str) -> {}: + response = httpx.post(self._introspect_url, data={'token': token}) + + if response.status_code != httpx.codes.OK: + raise Unauthorized(description=f"invalid status code {response.status_code}") + + claims = response.json() + active = claims.get("active", False) + if not active: + raise Unauthorized(description="inactive token") + + return claims or {} + + def _verify_jwt(self, token: str) -> {}: + public_key = self._jwk_manager.public_key(token) + + verifier = JWTVerifier( + public_key=public_key, + algorithm=self._jwt_algorithm, + audience=self._jwt_audience, + issuer=self._jwt_issuer, + ) + claims = verifier.verify(token) + + return claims or {} + + def _verify_exact_match_claims(self, claims): + for key, value in self._exact_match_claims.items(): + if key not in claims: + raise Unauthorized(description=f"missing required '{key}' claim") + elif claims[key] != value: + raise Unauthorized(description=f"invalid '{key}' claim value") + + def _verify_required_claims(self, claims): + for claim in self._required_claims: + if claim not in claims: + raise Unauthorized(description=f"missing required '{claim}' claim") + + +class CachedOAuth2Authenticator(_OAuth2Authenticator): + lock = asyncio.Lock() + + def __init__(self, delegate: OAuth2ClientCredentialsAuthenticator) -> None: + self._token = None + self._exp = None + self._exp_margin = opal_common_config.OAUTH2_EXP_MARGIN + self._delegate = delegate + + @property + def enabled(self): + return True + + def _expired(self): + if self._token is None: + return True + + now = int(time.time()) + return now > self._exp - self._exp_margin + + async def token(self): + if not self._expired(): + return self._token + + async with CachedOAuth2Authenticator.lock: + if not self._expired(): + return self._token + + token = await self._delegate.token() + claims = self._delegate.verify(token) + + self._token = token + self._exp = claims['exp'] + + return self._token + + @cached(cache=TTLCache( + maxsize=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE, + ttl=opal_common_config.OAUTH2_TOKEN_VERIFY_CACHE_TTL + )) + def __call__(self, authorization: Optional[str] = Header(None)) -> {}: + return self._delegate(authorization) diff --git a/packages/opal-common/opal_common/cli/typer_app.py b/packages/opal-common/opal_common/cli/typer_app.py index 45de0d594..47d38dd39 100644 --- a/packages/opal-common/opal_common/cli/typer_app.py +++ b/packages/opal-common/opal_common/cli/typer_app.py @@ -1,6 +1,5 @@ import typer - -from .commands import all_commands +from opal_common.cli.commands import all_commands def get_typer_app(): diff --git a/packages/opal-common/opal_common/confi/__init__.py b/packages/opal-common/opal_common/confi/__init__.py index ccd3d49ff..114be01c7 100644 --- a/packages/opal-common/opal_common/confi/__init__.py +++ b/packages/opal-common/opal_common/confi/__init__.py @@ -1 +1 @@ -from .confi import * +from opal_common.confi.confi import * diff --git a/packages/opal-common/opal_common/confi/cli.py b/packages/opal-common/opal_common/confi/cli.py index 00e3097ee..0ab88e55a 100644 --- a/packages/opal-common/opal_common/confi/cli.py +++ b/packages/opal-common/opal_common/confi/cli.py @@ -2,10 +2,9 @@ import click import typer +from opal_common.confi.types import ConfiEntry from typer.main import Typer -from .types import ConfiEntry - def create_click_cli(confi_entries: Dict[str, ConfiEntry], callback: Callable): cli = callback diff --git a/packages/opal-common/opal_common/confi/confi.py b/packages/opal-common/opal_common/confi/confi.py index f391c26a2..cbaa9a587 100644 --- a/packages/opal-common/opal_common/confi/confi.py +++ b/packages/opal-common/opal_common/confi/confi.py @@ -15,13 +15,12 @@ from decouple import Csv, UndefinedValueError, config, text_type, undefined from opal_common.authentication.casting import cast_private_key, cast_public_key from opal_common.authentication.types import EncryptionKeyFormat, PrivateKey, PublicKey -from opal_common.logging.decorators import log_exception +from opal_common.confi.cli import get_cli_object_for_config_objects +from opal_common.confi.types import ConfiDelay, ConfiEntry, no_cast +from opal_common.logging_utils.decorators import log_exception from pydantic import BaseModel, ValidationError from typer import Typer -from .cli import get_cli_object_for_config_objects -from .types import ConfiDelay, ConfiEntry, no_cast - class Placeholder(object): """Placeholder instead of default value for decouple.""" diff --git a/packages/opal-common/opal_common/config.py b/packages/opal-common/opal_common/config.py index 7666d47e4..64c0ea093 100644 --- a/packages/opal-common/opal_common/config.py +++ b/packages/opal-common/opal_common/config.py @@ -2,8 +2,7 @@ from sys import prefix from opal_common.authentication.types import EncryptionKeyFormat, JWTAlgorithm - -from .confi import Confi, confi +from opal_common.confi import Confi, confi _LOG_FORMAT_WITHOUT_PID = "{time} | {name: <40}|{level:^6} | {message}\n{exception}" _LOG_FORMAT_WITH_PID = "{time} | {process} | {name: <40}|{level:^6} | {message}\n{exception}" @@ -160,6 +159,28 @@ class OpalCommonConfig(Confi): [".rego"], description="List of extensions to serve as policy modules", ) + AUTH_TYPE = confi.str("AUTH_TYPE", None, description="Authentication type.") + OAUTH2_CLIENT_ID = confi.str("OAUTH2_CLIENT_ID", None, description="OAuth2 Client ID.") + OAUTH2_CLIENT_SECRET = confi.str("OAUTH2_CLIENT_SECRET", None, description="OAuth2 Client Secret.") + OAUTH2_TOKEN_URL = confi.str("OAUTH2_TOKEN_URL", None, description="OAuth2 Token URL.") + OAUTH2_INTROSPECT_URL = confi.str("OAUTH2_INTROSPECT_URL", None, description="OAuth2 introspect URL.") + OAUTH2_OPENID_CONFIGURATION_URL = confi.str("OAUTH2_OPENID_CONFIGURATION_URL", None, description="OAuth2 OpenID configuration URL.") + OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_MAXSIZE", 100, description="OAuth2 token validation cache maxsize.") + OAUTH2_TOKEN_VERIFY_CACHE_TTL = confi.int("OAUTH2_TOKEN_VERIFY_CACHE_TTL", 5 * 60, description="OAuth2 token validation cache TTL.") + + OAUTH2_EXP_MARGIN = confi.int("OAUTH2_EXP_MARGIN", 5 * 60, description="OAuth2 expiration margin.") + OAUTH2_EXACT_MATCH_CLAIMS = confi.str("OAUTH2_EXACT_MATCH_CLAIMS", None, description="OAuth2 exact match claims.") + OAUTH2_REQUIRED_CLAIMS = confi.str("OAUTH2_REQUIRED_CLAIMS", None, description="Comma separated list of required claims.") + OAUTH2_JWT_ALGORITHM = confi.enum( + "OAUTH2_JWT_ALGORITHM", + JWTAlgorithm, + getattr(JWTAlgorithm, "RS256"), + description="jwt algorithm, possible values: see: https://pyjwt.readthedocs.io/en/stable/algorithms.html", + ) + OAUTH2_JWT_AUDIENCE = confi.str("OAUTH2_JWT_AUDIENCE", None, description="OAuth2 required audience") + OAUTH2_JWT_ISSUER = confi.str("OAUTH2_JWT_ISSUER", None, description="OAuth2 required issuer") + OAUTH2_JWK_CACHE_MAXSIZE = confi.int("OAUTH2_JWK_CACHE_MAXSIZE", 100, description="OAuth2 JWKS cache maxsize.") + OAUTH2_JWK_CACHE_TTL = confi.int("OAUTH2_JWK_CACHE_TTL", 7 * 24 * 60 * 60, description="OAuth2 JWKS cache TTL.") ENABLE_METRICS = confi.bool("ENABLE_METRICS", False) diff --git a/packages/opal-common/opal_common/engine/__init__.py b/packages/opal-common/opal_common/engine/__init__.py index bbc306e62..33d1be247 100644 --- a/packages/opal-common/opal_common/engine/__init__.py +++ b/packages/opal-common/opal_common/engine/__init__.py @@ -1,2 +1,2 @@ -from .parsing import get_rego_package -from .paths import is_data_module, is_policy_module +from opal_common.engine.parsing import get_rego_package +from opal_common.engine.paths import is_data_module, is_policy_module diff --git a/packages/opal-common/opal_common/fetcher/__init__.py b/packages/opal-common/opal_common/fetcher/__init__.py index 84232e236..70a1f643c 100644 --- a/packages/opal-common/opal_common/fetcher/__init__.py +++ b/packages/opal-common/opal_common/fetcher/__init__.py @@ -1,3 +1,3 @@ -from .engine.fetching_engine import FetchingEngine -from .events import FetcherConfig, FetchEvent -from .fetcher_register import FetcherRegister +from opal_common.fetcher.engine.fetching_engine import FetchingEngine +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister diff --git a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py index a30f033d2..19a636a35 100644 --- a/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/base_fetching_engine.py @@ -1,8 +1,8 @@ from typing import Coroutine -from ..events import FetcherConfig, FetchEvent -from ..fetcher_register import FetcherRegister -from .core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister class BaseFetchingEngine: diff --git a/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py b/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py index b083e779e..3da152f14 100644 --- a/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py +++ b/packages/opal-common/opal_common/fetcher/engine/core_callbacks.py @@ -1,4 +1,4 @@ -from ..events import FetchEvent +from opal_common.fetcher.events import FetchEvent # Callback signatures diff --git a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py index eb816ecf2..6db97b338 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetch_worker.py @@ -1,10 +1,10 @@ import asyncio from typing import Coroutine -from ..events import FetchEvent -from ..fetcher_register import FetcherRegister -from ..logger import get_logger -from .base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.logger import get_logger logger = get_logger("fetch_worker") diff --git a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py index fd50a9f14..b439d4b8d 100644 --- a/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py +++ b/packages/opal-common/opal_common/fetcher/engine/fetching_engine.py @@ -2,13 +2,13 @@ import uuid from typing import Coroutine, Dict, List, Union -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..fetcher_register import FetcherRegister -from ..logger import get_logger -from .base_fetching_engine import BaseFetchingEngine -from .core_callbacks import OnFetchFailureCallback -from .fetch_worker import fetch_worker +from opal_common.fetcher.engine.base_fetching_engine import BaseFetchingEngine +from opal_common.fetcher.engine.core_callbacks import OnFetchFailureCallback +from opal_common.fetcher.engine.fetch_worker import fetch_worker +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.fetcher_register import FetcherRegister +from opal_common.fetcher.logger import get_logger logger = get_logger("engine") diff --git a/packages/opal-common/opal_common/fetcher/fetch_provider.py b/packages/opal-common/opal_common/fetcher/fetch_provider.py index 62ad97532..c05008fcd 100644 --- a/packages/opal-common/opal_common/fetcher/fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/fetch_provider.py @@ -1,8 +1,7 @@ +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.logger import get_logger from tenacity import retry, stop, wait -from .events import FetchEvent -from .logger import get_logger - logger = get_logger("opal.providers") diff --git a/packages/opal-common/opal_common/fetcher/fetcher_register.py b/packages/opal-common/opal_common/fetcher/fetcher_register.py index 5bf925160..9abf1322c 100644 --- a/packages/opal-common/opal_common/fetcher/fetcher_register.py +++ b/packages/opal-common/opal_common/fetcher/fetcher_register.py @@ -1,11 +1,10 @@ from typing import Dict, Optional, Type +from opal_common.config import opal_common_config +from opal_common.fetcher.events import FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider from opal_common.fetcher.logger import get_logger - -from ..config import opal_common_config -from .events import FetchEvent -from .fetch_provider import BaseFetchProvider -from .providers.http_fetch_provider import HttpFetchProvider +from opal_common.fetcher.providers.http_fetch_provider import HttpFetchProvider logger = get_logger("opal.fetcher_register") @@ -30,7 +29,7 @@ def __init__(self, config: Optional[Dict[str, BaseFetchProvider]] = None) -> Non if config is not None: self._config = config else: - from ..emport import emport_objects_by_class + from opal_common.emport import emport_objects_by_class # load fetchers fetchers = [] diff --git a/packages/opal-common/opal_common/fetcher/providers/__init__.py b/packages/opal-common/opal_common/fetcher/providers/__init__.py index 8e1f6bf77..ff1078ced 100644 --- a/packages/opal-common/opal_common/fetcher/providers/__init__.py +++ b/packages/opal-common/opal_common/fetcher/providers/__init__.py @@ -1,3 +1,3 @@ -from ...emport import dynamic_all +from opal_common.emport import dynamic_all __all__ = dynamic_all(__file__) diff --git a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py index 61432b751..4b574a8ea 100644 --- a/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/fastapi_rpc_fetch_provider.py @@ -2,10 +2,9 @@ from fastapi_websocket_rpc.rpc_methods import RpcMethodsBase from fastapi_websocket_rpc.websocket_rpc_client import WebSocketRpcClient - -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger logger = get_logger("rpc_fetch_provider") diff --git a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py index 7261b538b..cb493cf62 100644 --- a/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py +++ b/packages/opal-common/opal_common/fetcher/providers/http_fetch_provider.py @@ -1,19 +1,19 @@ """Simple HTTP get data fetcher using requests supports.""" from enum import Enum -from typing import Any, Union, cast +from typing import Any, Optional, Union, cast import httpx from aiohttp import ClientResponse, ClientSession from opal_common.config import opal_common_config +from opal_common.fetcher.events import FetcherConfig, FetchEvent +from opal_common.fetcher.fetch_provider import BaseFetchProvider +from opal_common.fetcher.logger import get_logger +from opal_common.http_utils import is_http_error_response +from opal_common.security.sslcontext import get_custom_ssl_context +from opal_common.authentication.authenticator import ClientAuthenticator from pydantic import validator -from ...http import is_http_error_response -from ...security.sslcontext import get_custom_ssl_context -from ..events import FetcherConfig, FetchEvent -from ..fetch_provider import BaseFetchProvider -from ..logger import get_logger - logger = get_logger("http_fetch_provider") @@ -53,6 +53,8 @@ class HttpFetchEvent(FetchEvent): class HttpFetchProvider(BaseFetchProvider): + _authenticator: Optional[dict] = None + def __init__(self, event: HttpFetchEvent) -> None: self._event: HttpFetchEvent if event.config is None: @@ -65,6 +67,9 @@ def __init__(self, event: HttpFetchEvent) -> None: if self._custom_ssl_context is not None else {} ) + if HttpFetchProvider._authenticator is None: + HttpFetchProvider._authenticator = ClientAuthenticator() + self._authenticator = HttpFetchProvider._authenticator def parse_event(self, event: FetchEvent) -> HttpFetchEvent: return HttpFetchEvent(**event.dict(exclude={"config"}), config=event.config) @@ -72,7 +77,10 @@ def parse_event(self, event: FetchEvent) -> HttpFetchEvent: async def __aenter__(self): headers = {} if self._event.config.headers is not None: - headers = self._event.config.headers + headers = self._event.config.headers.copy() + + await self._authenticator.authenticate(headers) + if opal_common_config.HTTP_FETCHER_PROVIDER_CLIENT == "httpx": self._session = httpx.AsyncClient(headers=headers) else: diff --git a/packages/opal-common/opal_common/git/__init__.py b/packages/opal-common/opal_common/git_utils/__init__.py similarity index 100% rename from packages/opal-common/opal_common/git/__init__.py rename to packages/opal-common/opal_common/git_utils/__init__.py diff --git a/packages/opal-common/opal_common/git/branch_tracker.py b/packages/opal-common/opal_common/git_utils/branch_tracker.py similarity index 97% rename from packages/opal-common/opal_common/git/branch_tracker.py rename to packages/opal-common/opal_common/git_utils/branch_tracker.py index 19bba8770..28f692e15 100644 --- a/packages/opal-common/opal_common/git/branch_tracker.py +++ b/packages/opal-common/opal_common/git_utils/branch_tracker.py @@ -3,8 +3,8 @@ from git import GitCommandError, Head, Remote, Repo from git.objects.commit import Commit -from opal_common.git.env import provide_git_ssh_environment -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.env import provide_git_ssh_environment +from opal_common.git_utils.exceptions import GitFailed from opal_common.logger import logger from tenacity import retry, stop_after_attempt, wait_fixed diff --git a/packages/opal-common/opal_common/git/bundle_maker.py b/packages/opal-common/opal_common/git_utils/bundle_maker.py similarity index 99% rename from packages/opal-common/opal_common/git/bundle_maker.py rename to packages/opal-common/opal_common/git_utils/bundle_maker.py index 0b006b6a7..f9d621a89 100644 --- a/packages/opal-common/opal_common/git/bundle_maker.py +++ b/packages/opal-common/opal_common/git_utils/bundle_maker.py @@ -6,7 +6,7 @@ from git import Repo from git.objects import Commit from opal_common.engine import get_rego_package, is_data_module, is_policy_module -from opal_common.git.commit_viewer import ( +from opal_common.git_utils.commit_viewer import ( CommitViewer, VersionedDirectory, VersionedFile, @@ -14,7 +14,7 @@ has_extension, is_under_directories, ) -from opal_common.git.diff_viewer import ( +from opal_common.git_utils.diff_viewer import ( DiffViewer, diffed_file_has_extension, diffed_file_is_under_directories, diff --git a/packages/opal-common/opal_common/git/bundle_utils.py b/packages/opal-common/opal_common/git_utils/bundle_utils.py similarity index 100% rename from packages/opal-common/opal_common/git/bundle_utils.py rename to packages/opal-common/opal_common/git_utils/bundle_utils.py diff --git a/packages/opal-common/opal_common/git/commit_viewer.py b/packages/opal-common/opal_common/git_utils/commit_viewer.py similarity index 100% rename from packages/opal-common/opal_common/git/commit_viewer.py rename to packages/opal-common/opal_common/git_utils/commit_viewer.py diff --git a/packages/opal-common/opal_common/git/diff_viewer.py b/packages/opal-common/opal_common/git_utils/diff_viewer.py similarity index 99% rename from packages/opal-common/opal_common/git/diff_viewer.py rename to packages/opal-common/opal_common/git_utils/diff_viewer.py index af5720f5c..ec6dff9d0 100644 --- a/packages/opal-common/opal_common/git/diff_viewer.py +++ b/packages/opal-common/opal_common/git_utils/diff_viewer.py @@ -4,7 +4,7 @@ from git import Repo from git.diff import Diff, DiffIndex from git.objects.commit import Commit -from opal_common.git.commit_viewer import VersionedFile +from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.paths import PathUtils DiffFilter = Callable[[Diff], bool] diff --git a/packages/opal-common/opal_common/git/env.py b/packages/opal-common/opal_common/git_utils/env.py similarity index 100% rename from packages/opal-common/opal_common/git/env.py rename to packages/opal-common/opal_common/git_utils/env.py diff --git a/packages/opal-common/opal_common/git/exceptions.py b/packages/opal-common/opal_common/git_utils/exceptions.py similarity index 100% rename from packages/opal-common/opal_common/git/exceptions.py rename to packages/opal-common/opal_common/git_utils/exceptions.py diff --git a/packages/opal-common/opal_common/git/repo_cloner.py b/packages/opal-common/opal_common/git_utils/repo_cloner.py similarity index 98% rename from packages/opal-common/opal_common/git/repo_cloner.py rename to packages/opal-common/opal_common/git_utils/repo_cloner.py index 43bda1ba2..76ebb4949 100644 --- a/packages/opal-common/opal_common/git/repo_cloner.py +++ b/packages/opal-common/opal_common/git_utils/repo_cloner.py @@ -8,8 +8,8 @@ from git import GitCommandError, GitError, Repo from opal_common.config import opal_common_config -from opal_common.git.env import provide_git_ssh_environment -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.env import provide_git_ssh_environment +from opal_common.git_utils.exceptions import GitFailed from opal_common.logger import logger from opal_common.utils import get_filepaths_with_glob from tenacity import RetryError, retry, stop, wait diff --git a/packages/opal-common/opal_common/git/tar_file_to_local_git_extractor.py b/packages/opal-common/opal_common/git_utils/tar_file_to_local_git_extractor.py similarity index 100% rename from packages/opal-common/opal_common/git/tar_file_to_local_git_extractor.py rename to packages/opal-common/opal_common/git_utils/tar_file_to_local_git_extractor.py diff --git a/packages/opal-common/opal_common/git/tests/branch_tracker_test.py b/packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py similarity index 95% rename from packages/opal-common/opal_common/git/tests/branch_tracker_test.py rename to packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py index bc17d6b09..751231a0d 100644 --- a/packages/opal-common/opal_common/git/tests/branch_tracker_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/branch_tracker_test.py @@ -18,8 +18,8 @@ from git import Repo from git.objects.commit import Commit -from opal_common.git.branch_tracker import BranchTracker -from opal_common.git.exceptions import GitFailed +from opal_common.git_utils.branch_tracker import BranchTracker +from opal_common.git_utils.exceptions import GitFailed def test_pull_with_no_changes(local_repo_clone: Repo): diff --git a/packages/opal-common/opal_common/git/tests/bundle_maker_test.py b/packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py similarity index 99% rename from packages/opal-common/opal_common/git/tests/bundle_maker_test.py rename to packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py index 63624f6eb..5e77ad0e5 100644 --- a/packages/opal-common/opal_common/git/tests/bundle_maker_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/bundle_maker_test.py @@ -19,8 +19,8 @@ from git import Repo from git.objects import Commit -from opal_common.git.bundle_maker import BundleMaker -from opal_common.git.commit_viewer import CommitViewer +from opal_common.git_utils.bundle_maker import BundleMaker +from opal_common.git_utils.commit_viewer import CommitViewer from opal_common.schemas.policy import PolicyBundle, RegoModule OPA_FILE_EXTENSIONS = (".rego", ".json") diff --git a/packages/opal-common/opal_common/git/tests/commit_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py similarity index 98% rename from packages/opal-common/opal_common/git/tests/commit_viewer_test.py rename to packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py index 91d19fcf1..1f9ca522a 100644 --- a/packages/opal-common/opal_common/git/tests/commit_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/commit_viewer_test.py @@ -19,7 +19,7 @@ from git import Repo from git.objects import Commit -from opal_common.git.commit_viewer import CommitViewer, VersionedNode +from opal_common.git_utils.commit_viewer import CommitViewer, VersionedNode def node_paths(nodes: List[VersionedNode]) -> List[Path]: diff --git a/packages/opal-common/opal_common/git/tests/conftest.py b/packages/opal-common/opal_common/git_utils/tests/conftest.py similarity index 99% rename from packages/opal-common/opal_common/git/tests/conftest.py rename to packages/opal-common/opal_common/git_utils/tests/conftest.py index c60099a5c..e3e5e0e14 100644 --- a/packages/opal-common/opal_common/git/tests/conftest.py +++ b/packages/opal-common/opal_common/git_utils/tests/conftest.py @@ -98,7 +98,7 @@ def local_repo(tmp_path, helpers: Helpers) -> Repo: """ root: Path = tmp_path / "myrepo" root.mkdir() - repo = Repo.init(root) + repo = Repo.init(root, initial_branch="master") # create file to delete later helpers.create_new_file_commit(repo, root / "deleted.rego") diff --git a/packages/opal-common/opal_common/git/tests/diff_viewer_test.py b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py similarity index 97% rename from packages/opal-common/opal_common/git/tests/diff_viewer_test.py rename to packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py index 6dc77ec4b..bcfbb93be 100644 --- a/packages/opal-common/opal_common/git/tests/diff_viewer_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/diff_viewer_test.py @@ -20,8 +20,11 @@ from git import Diff, Repo from git.objects import Commit -from opal_common.git.commit_viewer import VersionedFile -from opal_common.git.diff_viewer import DiffViewer, diffed_file_is_under_directories +from opal_common.git_utils.commit_viewer import VersionedFile +from opal_common.git_utils.diff_viewer import ( + DiffViewer, + diffed_file_is_under_directories, +) def diff_paths(diffs: List[Diff]) -> List[Path]: diff --git a/packages/opal-common/opal_common/git/tests/repo_cloner_test.py b/packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py similarity index 96% rename from packages/opal-common/opal_common/git/tests/repo_cloner_test.py rename to packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py index ffe6a02fa..567f3707b 100644 --- a/packages/opal-common/opal_common/git/tests/repo_cloner_test.py +++ b/packages/opal-common/opal_common/git_utils/tests/repo_cloner_test.py @@ -18,8 +18,8 @@ from git import Repo from opal_common.confi import Confi -from opal_common.git.exceptions import GitFailed -from opal_common.git.repo_cloner import RepoCloner +from opal_common.git_utils.exceptions import GitFailed +from opal_common.git_utils.repo_cloner import RepoCloner VALID_REPO_REMOTE_URL_HTTPS = "https://github.com/permitio/fastapi_websocket_pubsub.git" diff --git a/packages/opal-common/opal_common/git/tests/repo_watcher_test.py b/packages/opal-common/opal_common/git_utils/tests/repo_watcher_test.py similarity index 100% rename from packages/opal-common/opal_common/git/tests/repo_watcher_test.py rename to packages/opal-common/opal_common/git_utils/tests/repo_watcher_test.py diff --git a/packages/opal-common/opal_common/http.py b/packages/opal-common/opal_common/http_utils.py similarity index 100% rename from packages/opal-common/opal_common/http.py rename to packages/opal-common/opal_common/http_utils.py diff --git a/packages/opal-common/opal_common/logger.py b/packages/opal-common/opal_common/logger.py index 5f1229a80..8e826abd6 100644 --- a/packages/opal-common/opal_common/logger.py +++ b/packages/opal-common/opal_common/logger.py @@ -2,13 +2,12 @@ import sys from loguru import logger - -from .config import opal_common_config -from .logging.filter import ModuleFilter -from .logging.formatter import Formatter -from .logging.intercept import InterceptHandler -from .logging.thirdparty import hijack_uvicorn_logs -from .monitoring.apm import fix_ddtrace_logging +from opal_common.config import opal_common_config +from opal_common.logging_utils.filter import ModuleFilter +from opal_common.logging_utils.formatter import Formatter +from opal_common.logging_utils.intercept import InterceptHandler +from opal_common.logging_utils.thirdparty import hijack_uvicorn_logs +from opal_common.monitoring.apm import fix_ddtrace_logging def configure_logs(): diff --git a/packages/opal-common/opal_common/logging/__init__.py b/packages/opal-common/opal_common/logging_utils/__init__.py similarity index 100% rename from packages/opal-common/opal_common/logging/__init__.py rename to packages/opal-common/opal_common/logging_utils/__init__.py diff --git a/packages/opal-common/opal_common/logging/decorators.py b/packages/opal-common/opal_common/logging_utils/decorators.py similarity index 100% rename from packages/opal-common/opal_common/logging/decorators.py rename to packages/opal-common/opal_common/logging_utils/decorators.py diff --git a/packages/opal-common/opal_common/logging/filter.py b/packages/opal-common/opal_common/logging_utils/filter.py similarity index 100% rename from packages/opal-common/opal_common/logging/filter.py rename to packages/opal-common/opal_common/logging_utils/filter.py diff --git a/packages/opal-common/opal_common/logging/formatter.py b/packages/opal-common/opal_common/logging_utils/formatter.py similarity index 100% rename from packages/opal-common/opal_common/logging/formatter.py rename to packages/opal-common/opal_common/logging_utils/formatter.py diff --git a/packages/opal-common/opal_common/logging/intercept.py b/packages/opal-common/opal_common/logging_utils/intercept.py similarity index 100% rename from packages/opal-common/opal_common/logging/intercept.py rename to packages/opal-common/opal_common/logging_utils/intercept.py diff --git a/packages/opal-common/opal_common/logging/thirdparty.py b/packages/opal-common/opal_common/logging_utils/thirdparty.py similarity index 100% rename from packages/opal-common/opal_common/logging/thirdparty.py rename to packages/opal-common/opal_common/logging_utils/thirdparty.py diff --git a/packages/opal-common/opal_common/sources/api_policy_source.py b/packages/opal-common/opal_common/sources/api_policy_source.py index 9b6487907..7adc9ad70 100644 --- a/packages/opal-common/opal_common/sources/api_policy_source.py +++ b/packages/opal-common/opal_common/sources/api_policy_source.py @@ -6,7 +6,9 @@ import aiohttp from fastapi import status from fastapi.exceptions import HTTPException -from opal_common.git.tar_file_to_local_git_extractor import TarFileToLocalGitExtractor +from opal_common.git_utils.tar_file_to_local_git_extractor import ( + TarFileToLocalGitExtractor, +) from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource from opal_common.utils import ( diff --git a/packages/opal-common/opal_common/sources/git_policy_source.py b/packages/opal-common/opal_common/sources/git_policy_source.py index bffe8517d..8252cd4ce 100644 --- a/packages/opal-common/opal_common/sources/git_policy_source.py +++ b/packages/opal-common/opal_common/sources/git_policy_source.py @@ -1,9 +1,9 @@ from typing import Optional from git import Repo -from opal_common.git.branch_tracker import BranchTracker -from opal_common.git.exceptions import GitFailed -from opal_common.git.repo_cloner import RepoCloner +from opal_common.git_utils.branch_tracker import BranchTracker +from opal_common.git_utils.exceptions import GitFailed +from opal_common.git_utils.repo_cloner import RepoCloner from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource diff --git a/packages/opal-common/opal_common/urls.py b/packages/opal-common/opal_common/urls.py index ba68da20d..404877f7b 100644 --- a/packages/opal-common/opal_common/urls.py +++ b/packages/opal-common/opal_common/urls.py @@ -8,8 +8,8 @@ def set_url_query_param(url: str, param_name: str, param_value: str): >> set_url_query_param('https://api.permit.io/opal/data/config', 'token', 'secret') 'https://api.permit.io/opal/data/config?token=secret' - >> set_url_query_param('https://api.permit.io/opal/data/config&some=var', 'token', 'secret') - 'https://api.permit.io/opal/data/config&some=var?token=secret' + >> set_url_query_param('https://api.permit.io/opal/data/config?some=var', 'token', 'secret') + 'https://api.permit.io/opal/data/config?some=var&token=secret' """ parsed_url: ParseResult = urlparse(url) diff --git a/packages/opal-server/opal_server/authentication/__init__.py b/packages/opal-server/opal_server/authentication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/opal-server/opal_server/authentication/authenticator.py b/packages/opal-server/opal_server/authentication/authenticator.py new file mode 100644 index 000000000..6d8773a1e --- /dev/null +++ b/packages/opal-server/opal_server/authentication/authenticator.py @@ -0,0 +1,55 @@ +from typing import Optional + +from fastapi import Header +from fastapi.exceptions import HTTPException +from opal_common.config import opal_common_config +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.oauth2 import CachedOAuth2Authenticator, OAuth2ClientCredentialsAuthenticator +from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.types import JWTClaims +from opal_common.authentication.verifier import JWTVerifier, Unauthorized +from opal_common.logger import logger +from opal_server.config import opal_server_config + +class _ServerAuthenticator(Authenticator): + def __init__(self): + if opal_common_config.AUTH_TYPE == "oauth2": + self.__delegate = CachedOAuth2Authenticator(OAuth2ClientCredentialsAuthenticator()) + logger.info("OPAL is running in secure mode - will verify API requests with OAuth2 tokens.") + else: + self.__delegate = JWTAuthenticator(self.__signer()) + + def __signer(self) -> JWTSigner: + signer = JWTSigner( + private_key=opal_server_config.AUTH_PRIVATE_KEY, + public_key=opal_common_config.AUTH_PUBLIC_KEY, + algorithm=opal_common_config.AUTH_JWT_ALGORITHM, + audience=opal_common_config.AUTH_JWT_AUDIENCE, + issuer=opal_common_config.AUTH_JWT_ISSUER, + ) + if signer.enabled: + logger.info("OPAL is running in secure mode - will verify API requests with JWT tokens.") + else: + logger.info("OPAL was not provided with JWT encryption keys, cannot verify api requests!") + return signer + + def _delegate(self) -> dict: + return self.__delegate + + def signer(self) -> Optional[JWTSigner]: + if hasattr(self._delegate(), "verifier"): + return self._delegate().verifier + else: + return None + +class ServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + return self._delegate()(authorization) + +class WebsocketServerAuthenticator(_ServerAuthenticator): + def __call__(self, authorization: Optional[str] = Header(None)) -> JWTClaims: + try: + return self._delegate()(authorization) + except (Unauthorized, HTTPException): + return None diff --git a/packages/opal-server/opal_server/data/api.py b/packages/opal-server/opal_server/data/api.py index da5d043a9..45d953b41 100644 --- a/packages/opal-server/opal_server/data/api.py +++ b/packages/opal-server/opal_server/data/api.py @@ -6,7 +6,8 @@ require_peer_type, restrict_optional_topics_to_publish, ) -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.authenticator import Authenticator +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -25,7 +26,7 @@ def init_data_updates_router( data_update_publisher: DataUpdatePublisher, data_sources_config: ServerDataSourceConfig, - authenticator: JWTAuthenticator, + authenticator: Authenticator, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 0887c9160..36932ee30 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -12,7 +12,7 @@ from ddtrace import tracer from git import Repo from opal_common.async_utils import run_sync -from opal_common.git.bundle_maker import BundleMaker +from opal_common.git_utils.bundle_maker import BundleMaker from opal_common.logger import logger from opal_common.schemas.policy import PolicyBundle from opal_common.schemas.policy_source import ( @@ -139,14 +139,7 @@ def __init__( ) async def _get_repo_lock(self): - # # This implementation works across multiple processes/threads, but is not fair (next acquiree is random) - # locks_dir = self._base_dir / ".locks" - # await aiofiles.os.makedirs(str(locks_dir), exist_ok=True) - - # return NamedLock( - # locks_dir / GitPolicyFetcher.source_id(self._source), attempt_interval=0.1 - # ) - + # Previous file based implementation worked across multiple processes/threads, but wasn't fair (next acquiree is random) # This implementation works only within the same process/thread, but is fair (next acquiree is the earliest to enter the lock) src_id = GitPolicyFetcher.source_id(self._source) lock = GitPolicyFetcher.repo_locks[src_id] = GitPolicyFetcher.repo_locks.get( diff --git a/packages/opal-server/opal_server/main.py b/packages/opal-server/opal_server/main.py index 908c4561d..7e61e2a66 100644 --- a/packages/opal-server/opal_server/main.py +++ b/packages/opal-server/opal_server/main.py @@ -1,5 +1,5 @@ def create_app(*args, **kwargs): - from .server import OpalServer + from opal_server.server import OpalServer server = OpalServer(*args, **kwargs) return server.app diff --git a/packages/opal-server/opal_server/policy/bundles/api.py b/packages/opal-server/opal_server/policy/bundles/api.py index 223e72001..ae1da68ef 100644 --- a/packages/opal-server/opal_server/policy/bundles/api.py +++ b/packages/opal-server/opal_server/policy/bundles/api.py @@ -4,11 +4,11 @@ import fastapi.responses from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, status -from git import Repo +from git.repo import Repo from opal_common.confi.confi import load_conf_if_none -from opal_common.git.bundle_maker import BundleMaker -from opal_common.git.commit_viewer import CommitViewer -from opal_common.git.repo_cloner import RepoClonePathFinder +from opal_common.git_utils.bundle_maker import BundleMaker +from opal_common.git_utils.commit_viewer import CommitViewer +from opal_common.git_utils.repo_cloner import RepoClonePathFinder from opal_common.logger import logger from opal_common.schemas.policy import PolicyBundle from opal_server.config import opal_server_config diff --git a/packages/opal-server/opal_server/policy/watcher/callbacks.py b/packages/opal-server/opal_server/policy/watcher/callbacks.py index 62a30f16b..1b5f65590 100644 --- a/packages/opal-server/opal_server/policy/watcher/callbacks.py +++ b/packages/opal-server/opal_server/policy/watcher/callbacks.py @@ -3,13 +3,13 @@ from typing import List, Optional from git.objects import Commit -from opal_common.git.commit_viewer import ( +from opal_common.git_utils.commit_viewer import ( CommitViewer, FileFilter, find_ignore_match, has_extension, ) -from opal_common.git.diff_viewer import DiffViewer +from opal_common.git_utils.diff_viewer import DiffViewer from opal_common.logger import logger from opal_common.paths import PathUtils from opal_common.schemas.policy import ( diff --git a/packages/opal-server/opal_server/policy/watcher/factory.py b/packages/opal-server/opal_server/policy/watcher/factory.py index 10fb1b19c..6d94d6fc4 100644 --- a/packages/opal-server/opal_server/policy/watcher/factory.py +++ b/packages/opal-server/opal_server/policy/watcher/factory.py @@ -3,7 +3,7 @@ from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint from opal_common.confi.confi import load_conf_if_none -from opal_common.git.repo_cloner import RepoClonePathFinder +from opal_common.git_utils.repo_cloner import RepoClonePathFinder from opal_common.logger import logger from opal_common.sources.api_policy_source import ApiPolicySource from opal_common.sources.git_policy_source import GitPolicySource diff --git a/packages/opal-server/opal_server/policy/webhook/api.py b/packages/opal-server/opal_server/policy/webhook/api.py index c19595ad2..ef54c81b4 100644 --- a/packages/opal-server/opal_server/policy/webhook/api.py +++ b/packages/opal-server/opal_server/policy/webhook/api.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends, Request, status from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint -from opal_common.authentication.deps import JWTAuthenticator +from opal_common.authentication.authenticator import Authenticator from opal_common.logger import logger from opal_common.schemas.webhook import GitWebhookRequestParams from opal_server.config import PolicySourceTypes, opal_server_config @@ -15,7 +15,7 @@ def init_git_webhook_router( - pubsub_endpoint: PubSubEndpoint, authenticator: JWTAuthenticator + pubsub_endpoint: PubSubEndpoint, authenticator: Authenticator ): async def dummy_affected_repo_urls(request: Request) -> List[str]: return [] diff --git a/packages/opal-server/opal_server/pubsub.py b/packages/opal-server/opal_server/pubsub.py index 26d47c422..3b5c18f70 100644 --- a/packages/opal-server/opal_server/pubsub.py +++ b/packages/opal-server/opal_server/pubsub.py @@ -21,13 +21,12 @@ WebSocketRpcEventNotifier, ) from fastapi_websocket_rpc import RpcChannel -from opal_common.authentication.deps import WebsocketJWTAuthenticator -from opal_common.authentication.signer import JWTSigner from opal_common.authentication.types import JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import logger +from opal_server.authentication.authenticator import WebsocketServerAuthenticator from opal_server.config import opal_server_config from pydantic import BaseModel from starlette.datastructures import QueryParams @@ -121,7 +120,11 @@ class PubSub: """Wrapper for the Pub/Sub channel used for both policy and data updates.""" - def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): + def __init__( + self, + broadcaster_uri: str = None, + authenticator: Optional[WebsocketServerAuthenticator] = None, + ): """ Args: broadcaster_uri (str, optional): Which server/medium should the PubSub use for broadcasting. Defaults to BROADCAST_URI. @@ -159,7 +162,6 @@ def __init__(self, signer: JWTSigner, broadcaster_uri: str = None): not opal_server_config.BROADCAST_CONN_LOSS_BUGFIX_EXPERIMENT_ENABLED ), ) - authenticator = WebsocketJWTAuthenticator(signer) @self.api_router.get( "/pubsub_client_info", response_model=Dict[str, ClientInfo] diff --git a/packages/opal-server/opal_server/redis.py b/packages/opal-server/opal_server/redis_utils.py similarity index 100% rename from packages/opal-server/opal_server/redis.py rename to packages/opal-server/opal_server/redis_utils.py diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index 95181866a..60836994a 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -20,8 +20,9 @@ require_peer_type, restrict_optional_topics_to_publish, ) +from opal_common.authentication.authenticator import Authenticator from opal_common.authentication.casting import cast_private_key -from opal_common.authentication.deps import JWTAuthenticator, get_token_from_header +from opal_common.authentication.deps import get_token_from_header from opal_common.authentication.types import EncryptionKeyFormat, JWTClaims from opal_common.authentication.verifier import Unauthorized from opal_common.logger import logger @@ -78,7 +79,7 @@ def verify_private_key_or_throw(scope_in: Scope): def init_scope_router( scopes: ScopeRepository, - authenticator: JWTAuthenticator, + authenticator: Authenticator, pubsub_endpoint: PubSubEndpoint, ): router = APIRouter() diff --git a/packages/opal-server/opal_server/scopes/scope_repository.py b/packages/opal-server/opal_server/scopes/scope_repository.py index b3627741c..d9f5d9d20 100644 --- a/packages/opal-server/opal_server/scopes/scope_repository.py +++ b/packages/opal-server/opal_server/scopes/scope_repository.py @@ -1,7 +1,7 @@ from typing import List from opal_common.schemas.scopes import Scope -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB class ScopeNotFoundError(Exception): diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 533b397a8..f0104e7bf 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -7,7 +7,7 @@ import git from ddtrace import tracer from fastapi_websocket_pubsub import PubSubEndpoint -from opal_common.git.commit_viewer import VersionedFile +from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.logger import logger from opal_common.schemas.policy import PolicyUpdateMessageNotification from opal_common.schemas.policy_source import GitPolicyScopeSource diff --git a/packages/opal-server/opal_server/scopes/task.py b/packages/opal-server/opal_server/scopes/task.py index b3a577161..83b2b10f0 100644 --- a/packages/opal-server/opal_server/scopes/task.py +++ b/packages/opal-server/opal_server/scopes/task.py @@ -7,7 +7,7 @@ from opal_common.logger import logger from opal_server.config import opal_server_config from opal_server.policy.watcher.task import BasePolicyWatcherTask -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB from opal_server.scopes.scope_repository import ScopeRepository from opal_server.scopes.service import ScopesService diff --git a/packages/opal-server/opal_server/security/api.py b/packages/opal-server/opal_server/security/api.py index a17235163..2a562405a 100644 --- a/packages/opal-server/opal_server/security/api.py +++ b/packages/opal-server/opal_server/security/api.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status from opal_common.authentication.deps import StaticBearerAuthenticator @@ -7,7 +8,7 @@ from opal_common.schemas.security import AccessToken, AccessTokenRequest, TokenDetails -def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthenticator): +def init_security_router(signer: Optional[JWTSigner], authenticator: StaticBearerAuthenticator): router = APIRouter() @router.post( @@ -17,7 +18,7 @@ def init_security_router(signer: JWTSigner, authenticator: StaticBearerAuthentic dependencies=[Depends(authenticator)], ) async def generate_new_access_token(req: AccessTokenRequest): - if not signer.enabled: + if signer is None or not signer.enabled: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="opal server was not configured with security, cannot generate tokens!", diff --git a/packages/opal-server/opal_server/security/jwks.py b/packages/opal-server/opal_server/security/jwks.py index c55dfe5f3..3da016ecb 100644 --- a/packages/opal-server/opal_server/security/jwks.py +++ b/packages/opal-server/opal_server/security/jwks.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Optional from fastapi import FastAPI from fastapi.staticfiles import StaticFiles @@ -11,7 +12,7 @@ class JwksStaticEndpoint: def __init__( self, - signer: JWTSigner, + signer: Optional[JWTSigner], jwks_url: str, jwks_static_dir: str, ): @@ -25,7 +26,7 @@ def configure_app(self, app: FastAPI): # get the jwks contents from the signer jwks_contents = {} - if self._signer.enabled: + if self._signer is not None and self._signer.enabled: jwk = json.loads(self._signer.get_jwk()) jwks_contents = {"keys": [jwk]} diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 6a946a8c0..81a4d5722 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -8,8 +8,7 @@ from fastapi import Depends, FastAPI from fastapi_websocket_pubsub.event_broadcaster import EventBroadcasterContextManager -from opal_common.authentication.deps import JWTAuthenticator, StaticBearerAuthenticator -from opal_common.authentication.signer import JWTSigner +from opal_common.authentication.deps import StaticBearerAuthenticator from opal_common.confi.confi import load_conf_if_none from opal_common.config import opal_common_config from opal_common.logger import configure_logs, logger @@ -22,6 +21,7 @@ ServerSideTopicPublisher, TopicPublisher, ) +from opal_server.authentication.authenticator import ServerAuthenticator, WebsocketServerAuthenticator from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router from opal_server.data.data_update_publisher import DataUpdatePublisher @@ -32,7 +32,7 @@ from opal_server.policy.webhook.api import init_git_webhook_router from opal_server.publisher import setup_broadcaster_keepalive_task from opal_server.pubsub import PubSub -from opal_server.redis import RedisDB +from opal_server.redis_utils import RedisDB from opal_server.scopes.api import init_scope_router from opal_server.scopes.loader import load_scopes from opal_server.scopes.scope_repository import ScopeRepository @@ -49,7 +49,8 @@ def __init__( init_publisher: bool = None, data_sources_config: Optional[ServerDataSourceConfig] = None, broadcaster_uri: str = None, - signer: Optional[JWTSigner] = None, + authenticator: Optional[ServerAuthenticator] = None, + websocketAuthenticator: Optional[WebsocketServerAuthenticator] = None, enable_jwks_endpoint=True, jwks_url: str = None, jwks_static_dir: str = None, @@ -117,33 +118,22 @@ def __init__( self.broadcaster_uri = broadcaster_uri self.master_token = master_token - if signer is not None: - self.signer = signer + if authenticator is not None: + self.authenticator = authenticator else: - self.signer = JWTSigner( - private_key=opal_server_config.AUTH_PRIVATE_KEY, - public_key=opal_common_config.AUTH_PUBLIC_KEY, - algorithm=opal_common_config.AUTH_JWT_ALGORITHM, - audience=opal_common_config.AUTH_JWT_AUDIENCE, - issuer=opal_common_config.AUTH_JWT_ISSUER, - ) - if self.signer.enabled: - logger.info( - "OPAL is running in secure mode - will verify API requests with JWT tokens." - ) - else: - logger.info( - "OPAL was not provided with JWT encryption keys, cannot verify api requests!" - ) + self.authenticator = ServerAuthenticator() if enable_jwks_endpoint: self.jwks_endpoint = JwksStaticEndpoint( - signer=self.signer, jwks_url=jwks_url, jwks_static_dir=jwks_static_dir + signer=self.authenticator.signer(), jwks_url=jwks_url, jwks_static_dir=jwks_static_dir ) else: self.jwks_endpoint = None - self.pubsub = PubSub(signer=self.signer, broadcaster_uri=broadcaster_uri) + _websocketAuthenticator = websocketAuthenticator + if _websocketAuthenticator is None: + _websocketAuthenticator = WebsocketServerAuthenticator() + self.pubsub = PubSub(broadcaster_uri=broadcaster_uri, authenticator=_websocketAuthenticator) self.publisher: Optional[TopicPublisher] = None self.broadcast_keepalive: Optional[PeriodicPublisher] = None @@ -219,19 +209,17 @@ def _configure_monitoring(self): def _configure_api_routes(self, app: FastAPI): """mounts the api routes on the app object.""" - authenticator = JWTAuthenticator(self.signer) - data_update_publisher: Optional[DataUpdatePublisher] = None if self.publisher is not None: data_update_publisher = DataUpdatePublisher(self.publisher) # Init api routers with required dependencies data_updates_router = init_data_updates_router( - data_update_publisher, self.data_sources_config, authenticator + data_update_publisher, self.data_sources_config, self.authenticator ) - webhook_router = init_git_webhook_router(self.pubsub.endpoint, authenticator) + webhook_router = init_git_webhook_router(self.pubsub.endpoint, self.authenticator) security_router = init_security_router( - self.signer, StaticBearerAuthenticator(self.master_token) + self.authenticator.signer(), StaticBearerAuthenticator(self.master_token) ) statistics_router = init_statistics_router(self.opal_statistics) loadlimit_router = init_loadlimit_router(self.loadlimit_notation) @@ -240,7 +228,7 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( bundles_router, tags=["Bundle Server"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router(data_updates_router, tags=["Data Updates"]) app.include_router(webhook_router, tags=["Github Webhook"]) @@ -249,22 +237,22 @@ def _configure_api_routes(self, app: FastAPI): app.include_router( self.pubsub.api_router, tags=["Pub/Sub"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( statistics_router, tags=["Server Statistics"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) app.include_router( loadlimit_router, tags=["Client Load Limiting"], - dependencies=[Depends(authenticator)], + dependencies=[Depends(self.authenticator)], ) if opal_server_config.SCOPES: app.include_router( - init_scope_router(self._scopes, authenticator, self.pubsub.endpoint), + init_scope_router(self._scopes, self.authenticator, self.pubsub.endpoint), tags=["Scopes"], prefix="/scopes", ) diff --git a/packages/opal-server/opal_server/statistics.py b/packages/opal-server/opal_server/statistics.py index 6ae5ad619..14ea97f0a 100644 --- a/packages/opal-server/opal_server/statistics.py +++ b/packages/opal-server/opal_server/statistics.py @@ -124,9 +124,9 @@ async def _expire_old_servers(self): now = datetime.utcnow() still_alive = {} for server_id, last_seen in self._seen_servers.items(): - if ( - now - last_seen - ).total_seconds() < opal_server_config.STATISTICS_SERVER_KEEPALIVE_TIMEOUT: + if (now - last_seen).total_seconds() < float( + opal_server_config.STATISTICS_SERVER_KEEPALIVE_TIMEOUT + ): still_alive[server_id] = last_seen self._seen_servers = still_alive self._state.servers = {self._worker_id} | set(self._seen_servers.keys()) @@ -140,7 +140,7 @@ async def _periodic_server_keepalive(self): ServerKeepalive(worker_id=self._worker_id).dict(), ) await asyncio.sleep( - opal_server_config.STATISTICS_SERVER_KEEPALIVE_TIMEOUT / 2 + float(opal_server_config.STATISTICS_SERVER_KEEPALIVE_TIMEOUT) / 2 ) except asyncio.CancelledError: logger.debug("Statistics: periodic server keepalive cancelled") diff --git a/packages/opal-server/opal_server/tests/policy_repo_webhook_test.py b/packages/opal-server/opal_server/tests/policy_repo_webhook_test.py index a6633f984..8c8fd0bb7 100644 --- a/packages/opal-server/opal_server/tests/policy_repo_webhook_test.py +++ b/packages/opal-server/opal_server/tests/policy_repo_webhook_test.py @@ -29,7 +29,7 @@ from opal_common.utils import get_authorization_header from opal_server.config import PolicySourceTypes, opal_server_config -PORT = int(os.environ.get("PORT") or "9123") +PORT = int(os.environ.get("PORT") or "9125") # Basic server route config WEBHOOK_ROUTE = "/webhook" diff --git a/packages/opal-server/requires.txt b/packages/opal-server/requires.txt index 59c30201f..3a78295dc 100644 --- a/packages/opal-server/requires.txt +++ b/packages/opal-server/requires.txt @@ -5,6 +5,6 @@ pyjwt[crypto]>=2.1.0,<3 websockets>=10.3,<11 slowapi>=0.1.5,<1 # slowapi is stuck on and old `redis`, so fix that and switch from aioredis to redis -pygit2>=1.13.3,<2 +pygit2>=1.14.1,<1.15 asgiref>=3.5.2,<4 redis>=4.3.4,<5 diff --git a/packages/requires.txt b/packages/requires.txt index 5096c6000..c621d6ab0 100644 --- a/packages/requires.txt +++ b/packages/requires.txt @@ -8,4 +8,5 @@ pydantic[email]>=1.9.1,<2 typing-extensions;python_version<'3.8' uvicorn[standard]>=0.17.6,<1 fastapi-utils>=0.2.1,<1 -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability +cachetools>=5.3.3 diff --git a/requirements.txt b/requirements.txt index 3c0d2e61e..69533e9fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ pytest-rerunfailures wheel>=0.38.0 twine setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index b40546ae6..0c2c15cb8 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -1,5 +1,3 @@ -import os - from opal_common.logger import logger