CI workflow for python package publishing

.circleci/config.yml

@@ -2,12 +2,102 @@ version: 2.1
 orbs:
   elixir: membraneframework/elixir@1
 
-jobs: 
+executors:
+  linux_x86_executor:
+    docker:
+      - image: membraneframeworklabs/docker_membrane
+    resource_class: medium
+
+  linux_arm_executor:
+    docker:
+      - image: membraneframeworklabs/docker_membrane
+    resource_class: arm.medium
+
+  manylinux_x86_executor:
+    docker:
+      - image: quay.io/pypa/manylinux2014_x86_64
+    resource_class: medium
+
+  manylinux_arm_executor:
+    docker:
+      - image: quay.io/pypa/manylinux2014_aarch64
+    resource_class: arm.medium
+
+  macos_arm_executor:
+    macos:
+      xcode: 16.4.0
+    resource_class: macos.m1.medium.gen1
+
+commands:
+  mix_release_server:
+    parameters:
+      platform:
+        type: enum
+        enum: [macos-arm, linux-x86, linux-arm]
+      cache-version:
+        description: "Cache version number, can be used to force cache recreation"
+        type: integer
+        default: 1
+
+    steps:
+      - checkout
+      - when:
+          condition:
+            equal: [<< parameters.platform >>, macos-arm]
+          steps:
+            - run:
+                name: Install erlang and elixir
+                command: |
+                  brew install asdf openssl
+                  asdf plugin add elixir || true
+                  asdf plugin add erlang || true
+                  asdf install
+                  echo 'export PATH="${ASDF_DATA_DIR:-$HOME/.asdf}/shims:$PATH"' >> $BASH_ENV
+            - run:
+                name: Install hex and rebar
+                command: |
+                  mix local.hex --force
+                  mix local.rebar --force
+      - elixir/get_mix_deps:
+          cache-version: << parameters.cache-version >>
+      - elixir/use_build_cache:
+          cache-version: << parameters.cache-version >>
+      - run:
+          name: Mix release
+          command: mix release server
+      - run:
+          name: Tar the release
+          command: |
+            mkdir -p artifacts
+            tar -czvf artifacts/boombox-server-<< parameters.platform >>.tar.gz -C _build/prod/rel/server .
+      - persist_to_workspace:
+          root: ~/project/
+          paths:
+            - artifacts
+
+  publish_python_package_linux:
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Build wheel and repair with auditwheel
+          command: |
+            /opt/python/cp311-cp311/bin/pip install build twine
+            /opt/python/cp311-cp311/bin/python -m build --wheel
+            auditwheel repair dist/*.whl --wheel-dir ../wheelhouse
+      - run:
+          name: Publish to PyPI
+          working_directory: ~/project/wheelhouse
+          command: |
+            /opt/python/cp311-cp311/bin/pip install twine
+            /opt/python/cp311-cp311/bin/twine upload --username __token__ --password $PYPI_API_KEY *
+
+jobs:
   test_exclude_browser:
     docker:
       - image: membraneframeworklabs/docker_membrane
-        environment:
-          MIX_ENV: test
+    environment:
+      MIX_ENV: test
 
     working_directory: ~/app
     steps:
@@ -18,29 +108,174 @@ jobs:
       - run: mix compile
       - run: mix test --exclude browser
 
+
+  compare_versions:
+    docker:
+      - image: ubuntu
+
+    steps:
+      - checkout
+      - run:
+          name: Compare the versions from mix.exs and python/pyproject.toml
+          command: |
+            python_package_version=$(grep "^version =" python/pyproject.toml | awk '{print $3}' | tr -d ' "')
+            elixir_package_version=$(grep "^  @version" mix.exs | awk '{print $2}' | tr -d ' "')
+            if [ $python_package_version != $elixir_package_version ]; then
+                echo "package versions don't match - python: $python_package_version, elixir: $elixir_package_version"
+                exit 1
+            fi
+
+  mix_release_server_macos_arm:
+    executor: macos_arm_executor
+
+    environment:
+      MIX_ENV: prod
+
+    steps:
+      - mix_release_server:
+          platform: macos-arm
+
+  mix_release_server_linux_x86:
+    executor: linux_x86_executor
+
+    environment:
+      MIX_ENV: prod
+
+    steps:
+      - mix_release_server:
+          platform: linux-x86
+
+  mix_release_server_linux_arm:
+    executor: linux_arm_executor
+
+    environment:
+      MIX_ENV: prod
+
+    steps:
+      - mix_release_server:
+          platform: linux-arm
+
+  publish_github_release:
+    parameters:
+      version:
+        description: version of boombox
+        type: string
+
+    docker:
+      - image: ubuntu
+
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: Install ghr tool
+          command: |
+            apt update
+            apt install -y wget
+            wget https://github.com/tcnksm/ghr/releases/download/v0.16.0/ghr_v0.16.0_linux_amd64.tar.gz
+            tar -xf ghr_v0.16.0_linux_amd64.tar.gz
+      - run:
+          name: Publish release
+          command:
+            ./ghr_v0.16.0_linux_amd64/ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete << parameters.version >> ~/artifacts/
+
+  publish_python_package_macos_arm:
+    executor: macos_arm_executor
+    working_directory: ~/project/python
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Install build and publish tools
+          command: pip install build twine
+      - run:
+          name: Build wheel (no sdist)
+          command: python -m build --wheel
+      - run:
+          name: Publish to PyPI
+          command: twine upload --verbose --username __token__ --password $PYPI_API_KEY dist/*
+
+  publish_python_package_linux_x86:
+    executor: manylinux_x86_executor
+    working_directory: ~/project/python
+    steps:
+      - publish_python_package_linux
+
+  publish_python_package_linux_arm:
+    executor: manylinux_arm_executor
+    working_directory: ~/project/python
+    steps:
+      - publish_python_package_linux
+
+  build_python_docs:
+    docker:
+      - image: curlimages/curl:8.16.0
+
+    steps:
+      - run:
+          name: Trigger docs build
+          command:
+            curl -X POST -d token=$READTHEDOCS_WEBHOOK_TOKEN $READTHEDOCS_WEBHOOK_URL
+
 workflows:
-  version: 2
   build:
     jobs:
       - elixir/build_test:
-          filters: &filters
+          filters: &push_filters
             tags:
               only: /v.*/
       - elixir/lint:
-          filters:
-            <<: *filters
+          filters: *push_filters
       - test_exclude_browser:
-          filters:
-            <<: *filters
+          filters: *push_filters
+      - compare_versions:
+          filters: *push_filters
       - elixir/hex_publish:
           requires:
             - elixir/build_test
             - test_exclude_browser
             - elixir/lint
-          context:
-            - Deployment
-          filters:
+            - compare_versions
+          filters: &publish_filters
             branches:
               ignore: /.*/
             tags:
               only: /v.*/
+          context:
+            - Deployment
+      - mix_release_server_macos_arm: &mix_release_job_config
+          requires:
+            - elixir/build_test
+            - test_exclude_browser
+            - elixir/lint
+            - compare_versions
+          filters: *publish_filters
+          context:
+            - Deployment
+      - mix_release_server_linux_x86: *mix_release_job_config
+      - mix_release_server_linux_arm: *mix_release_job_config
+      - publish_github_release:
+          requires:
+            - mix_release_server_macos_arm
+            - mix_release_server_linux_x86
+            - mix_release_server_linux_arm
+          filters: *publish_filters
+          context:
+            - Deployment
+          version: << pipeline.git.tag >>
+      - publish_python_package_macos_arm: &publish_python_package_job_config
+          requires:
+            - publish_github_release
+          filters: *publish_filters
+          context:
+            - Deployment
+      - publish_python_package_linux_x86: *publish_python_package_job_config
+      - publish_python_package_linux_arm: *publish_python_package_job_config
+      - build_python_docs:
+          requires:
+            - publish_python_package_macos_arm
+            - publish_python_package_linux_x86
+            - publish_python_package_linux_arm
+          filters: *publish_filters
+          context:
+            - Deployment

mix.exs

@@ -52,14 +52,15 @@ defmodule Boombox.Mixfile do
       {:membrane_mp4_plugin, "~> 0.36.0"},
       {:membrane_realtimer_plugin, "~> 0.9.0"},
       {:membrane_http_adaptive_stream_plugin, "~> 0.20.1"},
-      {:membrane_rtmp_plugin, "~> 0.27.2"},
+      {:membrane_rtmp_plugin, "~> 0.29.1"},
       {:membrane_rtsp_plugin, "~> 0.6.1"},
       {:membrane_rtp_plugin, "~> 0.30.0"},
       {:membrane_rtp_format, "~> 0.10.0"},
       {:membrane_rtp_aac_plugin, "~> 0.9.0"},
       {:membrane_rtp_h264_plugin, "~> 0.20.0"},
       {:membrane_rtp_opus_plugin, "~> 0.10.0"},
       {:membrane_rtp_h265_plugin, "~> 0.5.2"},
+      {:membrane_vpx_plugin, "~> 0.4.2"},
       {:membrane_ffmpeg_swresample_plugin, "~> 0.20.0"},
       {:membrane_hackney_plugin, "~> 0.11.0"},
       {:membrane_ffmpeg_swscale_plugin, "~> 0.16.2"},
@@ -211,7 +212,10 @@ defmodule Boombox.Mixfile do
       |> Enum.map(&Path.relative_to(&1, base_dir))
       |> Map.new(&{Path.basename(&1), &1})
 
-    Path.wildcard("#{base_dir}/*/priv/bundlex/*/*")
+    # Restore symlinks
+    # <package>/priv/shared/precompiled/<precompiled_dir> -> ../../../../bundlex-<version>/priv/shared/<precompiled_dir>/lib
+    Path.join(base_dir, "*/priv/bundlex/*/*")
+    |> Path.wildcard()
     |> Enum.each(fn path ->
       name = Path.basename(path)
 
@@ -225,6 +229,9 @@ defmodule Boombox.Mixfile do
       end
     end)
 
+    # Restore symlinks
+    # bundlex-<version>/priv/shared/precompiled/<precompiled_dir>/lib/<some_lib>.dylib -> <some_lib>.<most_precise_version>.dylib
+    # e.g. libvpx.dylib -> libvpx.11.dylib
     Path.join(base_dir, "bundlex*/priv/shared/precompiled/*/lib")
     |> Path.wildcard()
     |> Enum.map(fn lib_dir ->