Merge pull request #1792 from microsoft/staging

Staging to main, prepare for release 1.1.1

.github/actions/azureml-test/action.yml

@@ -0,0 +1,131 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# ---------------------------------------------------------
+
+name: azureml-tests
+description: "Submit experiment to AzureML cluster"
+inputs:
+  # azureml experiment name
+  EXP_NAME:
+    required: true
+    type: string
+  # type of test - unit or nightly
+  TEST_KIND:
+    required: true
+    type: string
+  # test environment - cpu, gpu or spark
+  TEST_ENV:
+    required: false
+    type: string
+  # azureml compute credentials
+  AZUREML_TEST_CREDENTIALS:
+    required: true
+    type: string
+  # azureml compute subid
+  AZUREML_TEST_SUBID:
+    required: true
+    type: string
+  # python version
+  PYTHON_VERSION:
+    required: true
+    type: string
+  # test group name
+  TEST_GROUP:
+    required: true
+    type: string
+  # cpu cluster name
+  CPU_CLUSTER_NAME:
+    required: false
+    type: string
+    default: "cpu-cluster"
+  # gpu cluster name
+  GPU_CLUSTER_NAME:
+    required: false
+    type: string
+    default: "gpu-cluster"
+  # AzureML resource group name
+  RG:
+    required: false
+    type: string
+    default: "recommenders_project_resources"  
+  # AzureML workspace name
+  WS:
+    required: false
+    type: string
+    default: "azureml-test-workspace"  
+  # test logs path
+  TEST_LOGS_PATH:
+    required: false
+    type: string
+    default: '"test_logs.log"'
+  # pytest exit code
+  PYTEST_EXIT_CODE:
+    required: false
+    type: string
+    default: "pytest_exit_code.log"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with: 
+        python-version: "3.8"
+    - name: Install azureml-core and azure-cli on a GitHub hosted server
+      shell: bash
+      run: pip install azureml-core azure-cli
+    - name: Log in to Azure
+      uses: azure/login@v1
+      with:
+        creds: ${{inputs.AZUREML_TEST_CREDENTIALS}}
+    - name: Install wheel package
+      shell: bash
+      run: pip install wheel
+    - name: Create wheel from setup.py
+      shell: bash
+      run: python setup.py bdist_wheel
+    - name: Extract branch name
+      shell: bash
+      run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+      id: extract_branch
+    - name: Submit CPU tests to AzureML
+      shell: bash
+      if: contains(inputs.TEST_GROUP, 'cpu')
+      run: >-
+          python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}}
+          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+          --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
+          --testlogs ${{inputs.TEST_LOGS_PATH}} --testkind ${{inputs.TEST_KIND}}
+          --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
+    - name: Submit GPU tests to AzureML
+      shell: bash
+      if: contains(inputs.TEST_GROUP, 'gpu')
+      run: >-
+          python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.GPU_CLUSTER_NAME}}
+          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+          --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
+          --testlogs ${{inputs.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind ${{inputs.TEST_KIND}}
+          --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
+    - name: Submit PySpark tests to AzureML
+      shell: bash
+      if: contains(inputs.TEST_GROUP, 'spark')
+      run: >-
+          python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{inputs.CPU_CLUSTER_NAME}}
+          --subid ${{inputs.AZUREML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+          --rg ${{inputs.RG}} --wsname ${{inputs.WS}} --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}}
+          --testlogs ${{inputs.TEST_LOGS_PATH}} --add_spark_dependencies --testkind ${{inputs.TEST_KIND}}
+          --conda_pkg_python ${{inputs.PYTHON_VERSION}} --testgroup ${{inputs.TEST_GROUP}}
+    - name: Print test logs
+      shell: bash
+      run: cat ${{inputs.TEST_LOGS_PATH}}
+    - name: Get exit status
+      shell: bash
+      id: exit_status
+      run: echo ::set-output name=code::$(cat ${{inputs.PYTEST_EXIT_CODE}})
+    - name: Check Success/Failure
+      if: ${{ steps.execute_tests.outputs.code != 0 }}
+      uses: actions/github-script@v3
+      with:
+        script: |
+            core.setFailed('All tests did not pass!')

.github/actions/get-test-groups/action.yml

@@ -0,0 +1,29 @@
+name: get-test-groups
+description: "Get test group names from tests_groups.py"
+inputs:
+  # type of test - unit or nightly
+  TEST_KIND:
+    required: true
+    type: string
+  # test environment - cpu, gpu or spark
+  TEST_ENV:
+    required: false
+    type: string
+    default: 'cpu'
+outputs:
+  test_groups:
+    value: ${{steps.get_test_groups.outputs.test_groups}}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get test group names
+      id: get_test_groups
+      shell: bash
+      run: |
+        if [[ ${{ inputs.TEST_KIND }} == "nightly" ]]; then
+          test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import nightly_test_groups; print([t for t in nightly_test_groups.keys() if "${{inputs.TEST_ENV}}" in t])')
+        else
+          test_groups_str=$(python -c 'from tests.ci.azureml_tests.test_groups import unit_test_groups; print(list(unit_test_groups.keys()))')
+        fi
+        echo ::set-output name=test_groups::$test_groups_str

.github/workflows/azureml-cpu-nightly.yml

@@ -22,6 +22,26 @@ on:
     # to ensure we are running the build on the staging branch, we can add push policy for it
     branches: [staging]
 
+  # pull_request_target allows execution of workflows in the context
+  # of a base repository. When a PR is raised from a forked repo, it
+  # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as
+  # a repo level secret, which is needed to trigger execution of unit
+  # tests on AzureML compute.
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - synchronize
+    branches:
+      - 'main'
+    paths:
+      # Unit tests will be run only when there are changes in the
+      # unit tests related code including:
+      - examples/**
+      - recommenders/**
+      - tests/**
+      - setup.py
+
   # enable manual trigger
   workflow_dispatch:
     input:
@@ -33,11 +53,42 @@ on:
   workflow_call:
 
 jobs:
+  get-test-groups:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Get test group names
+        id: get_test_groups
+        uses: ./.github/actions/get-test-groups
+        with:
+          TEST_KIND: "nightly"
+          TEST_ENV: "cpu"
+      - name: Print test group names
+        run: echo ${{ steps.get_test_groups.outputs.test_groups }}
+        shell: bash
+    outputs:
+      test_groups: ${{ steps.get_test_groups.outputs.test_groups }}
 
-  cpu-nightly-tests:
-    uses: ./.github/workflows/azureml-template.yml
-    with:
-      EXP_NAME: 'nightly_tests'
-      TEST_KIND: 'nightly'
-      TEST_ENV: 'cpu'
-    secrets: inherit
+  execute-tests:
+    needs: get-test-groups
+    name: ${{ join(matrix.*, ', ') }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
+        test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Execute tests
+        uses: ./.github/actions/azureml-test
+        id: execute_tests
+        with:
+          EXP_NAME: 'nightly_tests'
+          TEST_KIND: 'nightly'
+          TEST_ENV: 'cpu'
+          AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }}
+          AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }}
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          TEST_GROUP: ${{ matrix.test-group }}

.github/workflows/azureml-gpu-nightly.yml

@@ -22,6 +22,26 @@ on:
     # to ensure we are running the build on the staging branch, we can add push policy for it
     branches: [staging]
 
+  # pull_request_target allows execution of workflows in the context
+  # of a base repository. When a PR is raised from a forked repo, it
+  # gives the workflow access to AZUREML_TEST_CREDENTIALS, stored as
+  # a repo level secret, which is needed to trigger execution of unit
+  # tests on AzureML compute.
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - synchronize
+    branches:
+      - 'main'
+    paths:
+      # Unit tests will be run only when there are changes in the
+      # unit tests related code including:
+      - examples/**
+      - recommenders/**
+      - tests/**
+      - setup.py
+
   # enable manual trigger
   workflow_dispatch:
     input:
@@ -33,11 +53,42 @@ on:
   workflow_call:
 
 jobs:
+  get-test-groups:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Get test group names
+        id: get_test_groups
+        uses: ./.github/actions/get-test-groups
+        with:
+          TEST_KIND: "nightly"
+          TEST_ENV: "gpu"
+      - name: Print test group names
+        run: echo ${{ steps.get_test_groups.outputs.test_groups }}
+        shell: bash
+    outputs:
+      test_groups: ${{ steps.get_test_groups.outputs.test_groups }}
 
-  gpu-nightly-tests:
-    uses: ./.github/workflows/azureml-template.yml
-    with:
-      EXP_NAME: 'nightly_tests'
-      TEST_KIND: 'nightly'
-      TEST_ENV: 'gpu'
-    secrets: inherit
+  execute-tests:
+    needs: get-test-groups
+    name: ${{ join(matrix.*, ', ') }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['"python=3.7"', '"python=3.8"', '"python=3.9"']
+        test-group:  ${{ fromJSON(needs.get-test-groups.outputs.test_groups) }}
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Execute tests
+        uses: ./.github/actions/azureml-test
+        id: execute_tests
+        with:
+          EXP_NAME: 'nightly_tests'
+          TEST_KIND: 'nightly'
+          TEST_ENV: 'gpu'
+          AZUREML_TEST_CREDENTIALS: ${{ secrets.AZUREML_TEST_CREDENTIALS }}
+          AZUREML_TEST_SUBID: ${{ secrets.AZUREML_TEST_SUBID }}
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          TEST_GROUP: ${{ matrix.test-group }}