Add AArch64 unit test workflow

Author: Chizkiyahu

.github/workflows/pull.yml

@@ -34,7 +34,45 @@ jobs:
 
         # Run tests
         pytest
- 
+
+# runner: linux.arm64.2xlarge is not working for some reason
+# using matrix instead to dynamically select a runner
+  generate-matrix-linux-aarch64:
+    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
+    with:
+      package-type: wheel
+      os: linux-aarch64
+      test-infra-repository: meta-pytorch/tokenizers
+      test-infra-ref: main
+      with-cuda: disabled
+      with-rocm: disabled
+      build-python-only: enabled
+
+  unittest-linux-aarch64:
+    name: unittest-linux-aarch64
+    needs: generate-matrix-linux-aarch64
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    strategy:
+      matrix: ${{ fromJson(needs.generate-matrix-linux-aarch64.outputs.matrix) }}
+      fail-fast: false
+    with:
+      runner: ${{ matrix.validation_runner }}
+      submodules: 'true'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        set -ex
+        cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test
+        cmake --build build/test -j9 --config Debug
+        pushd build/test && ctest && popd
+
+        # Install tokenizers
+        pip install . -v
+        pip install pytest blobfile transformers>=4.53.1
+
+        # Run tests
+        pytest
+
   unittest-windows:
     uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
     with: