Update SA-master-updated

.github/ISSUE_TEMPLATE/car_bug_report.yml

@@ -1,6 +1,6 @@
 name: Car bug report
 description: For issues with a particular car make or model
-labels: ["car bug"]
+labels: ["car", "bug"]
 body:
 
   - type: markdown

.github/workflows/badges.yaml

@@ -0,0 +1,53 @@
+name: badges
+on:
+  schedule:
+    - cron: '0 * * * *'
+  workflow_dispatch:
+
+env:
+  BASE_IMAGE: openpilot-base
+  DOCKER_REGISTRY: ghcr.io/commaai
+
+  BUILD: |
+      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
+      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
+      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+
+jobs:
+  badges:
+    name: create badges
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - name: Cache scons
+      id: scons-cache
+      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
+      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+      env:
+        CACHE_SKIP_SAVE: true
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+        restore-keys: |
+          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+          scons-
+
+    - name: Build Docker image
+      run: eval "$BUILD"
+
+    - name: Push badges
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py"
+
+        git checkout --orphan badges
+        git rm -rf --cached .
+        git config user.email "[email protected]"
+        git config user.name "Badge Researcher"
+
+        git add translation_badge_*.svg
+        git commit -m "Add/Update badges"
+        git push -f origin HEAD

.github/workflows/prebuilt.yaml

@@ -25,11 +25,12 @@ jobs:
       IMAGE_NAME: openpilot-prebuilt
     steps:
     - name: Wait for green check mark
-      uses: lewagon/wait-on-check-action@v0.2
+      uses: lewagon/wait-on-check-action@e2558238c09778af25867eb5de5a3ce4bbae3dcd
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
+        check-regexp: ^((?!.*(build master-ci).*).)*$
     - uses: actions/checkout@v3
       with:
         submodules: true

.github/workflows/release.yaml

@@ -12,11 +12,12 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     steps:
     - name: Wait for green check mark
-      uses: lewagon/wait-on-check-action@v0.2
+      uses: lewagon/wait-on-check-action@e2558238c09778af25867eb5de5a3ce4bbae3dcd
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build master-ci'
+        check-regexp: ^((?!.*(build prebuilt).*).)*$
     - uses: actions/checkout@v3
       with:
         submodules: true

.github/workflows/selfdrive_tests.yaml

@@ -17,13 +17,12 @@ env:
       docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
       docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
 
   BUILD_CL: |
-      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base_cl) || true
-      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
+      docker pull $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest || true
       docker build --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $CL_BASE_IMAGE /bin/sh -c
+  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
   UNIT_TEST: coverage run --append -m unittest discover
 
@@ -59,12 +58,22 @@ jobs:
       run: |
         TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
         cp Dockerfile.openpilot_base $STRIPPED_DIR
+        cp .pre-commit-config.yaml $STRIPPED_DIR
+        cp .pylintrc $STRIPPED_DIR
+        cp mypy.ini $STRIPPED_DIR
     - name: Build Docker image
-      run: eval "$BUILD"
+      run: |
+          eval "$BUILD"
+          rm $STRIPPED_DIR/Dockerfile.openpilot_base
     - name: Build openpilot and run checks
       run: |
           cd $STRIPPED_DIR
           ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
+                          pre-commit run --all && \
+                          rm .pre-commit-config.yaml && \
+                          rm .pylintrc && \
+                          rm mypy.ini && \
+                          release/check-dirty.sh && \
                           python -m unittest discover selfdrive/car"
 
   build_all:
@@ -90,12 +99,12 @@ jobs:
     - name: Build Docker image
       run: eval "$BUILD"
     - name: Build openpilot with all flags
-      run: ${{ env.RUN }} "scons -j$(nproc) --extras --test"
+      run: ${{ env.RUN }} "scons -j$(nproc) --extras --test && release/check-dirty.sh"
     - name: Cleanup scons cache
       run: |
           ${{ env.RUN }} "scons -j$(nproc) --extras --test && \
                           rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"
+                          scons -j$(nproc) --extras --test --cache-populate"
 
   #build_mac:
   #  name: build macos
@@ -201,12 +210,15 @@ jobs:
         submodules: true
     - name: Build Docker image
       run: eval "$BUILD"
+    - name: Push to container registry
+      run: |
+        $DOCKER_LOGIN
+        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
     - name: Build CL Docker image
       run: eval "$BUILD_CL"
     - name: Push to container registry
       run: |
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
         docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest
 
 #  static_analysis:
@@ -297,18 +309,22 @@ jobs:
                         $UNIT_TEST selfdrive/loggerd && \
                         $UNIT_TEST selfdrive/car && \
                         $UNIT_TEST selfdrive/locationd && \
+                        selfdrive/locationd/test/_test_locationd_lib.py && \
                         $UNIT_TEST selfdrive/athena && \
                         $UNIT_TEST selfdrive/thermald && \
-                        $UNIT_TEST selfdrive/hardware/tici && \
+                        $UNIT_TEST system/hardware/tici && \
                         $UNIT_TEST selfdrive/modeld && \
                         $UNIT_TEST tools/lib/tests && \
+                        ./selfdrive/ui/tests/create_test_translations.sh && \
+                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
+                        ./selfdrive/ui/tests/test_translations.py && \
                         ./common/tests/test_util && \
                         ./common/tests/test_swaglog && \
                         ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
                         ./selfdrive/loggerd/tests/test_logger &&\
                         ./system/proclogd/tests/test_proclog && \
-                        ./selfdrive/ui/replay/tests/test_replay && \
-                        ./selfdrive/camerad/test/ae_gray_test && \
+                        ./tools/replay/tests/test_replay && \
+                        ./system/camerad/test/ae_gray_test && \
                         coverage xml"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v2
@@ -501,3 +517,67 @@ jobs:
       run: |
         $DOCKER_LOGIN
         docker push $DOCKER_REGISTRY/openpilot-docs:latest
+
+  car_docs_diff:
+    name: comment on PR with car docs diff
+    runs-on: ubuntu-20.04
+    timeout-minutes: 50
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          ref: ${{ github.event.pull_request.base.ref }}
+      - name: Cache scons
+        id: scons-cache
+        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
+        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+        env:
+          CACHE_SKIP_SAVE: true
+        with:
+          path: /tmp/scons_cache
+          key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+          restore-keys: |
+            scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+            scons-
+      - name: Build Docker image
+        run: eval "$BUILD"
+      - name: Get base car info
+        run: |
+          ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
+          sudo chown -R $USER:$USER ${{ github.workspace }}
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Save car docs diff
+        id: save_diff
+        run: |
+          ${{ env.RUN }} "scons -j$(nproc)"
+          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info") || true
+          output="${output//$'\n'/'%0A'}"
+          echo "::set-output name=diff::$output"
+      - name: Find comment
+        if: ${{ env.AZURE_TOKEN != '' }}
+        uses: peter-evans/find-comment@1769778a0c5bd330272d749d12c036d65e70d39d
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: This PR makes changes to
+      - name: Update comment
+        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
+        uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: "${{ steps.save_diff.outputs.diff }}"
+          edit-mode: replace
+      - name: Delete comment
+        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.deleteComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: ${{ steps.fc.outputs.comment-id }}
+            })

.github/workflows/tools_tests.yaml

@@ -21,6 +21,46 @@ env:
        GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
 
 jobs:
+  build_latest_ubuntu:
+    name: build latest ubuntu
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Cache pyenv
+      id: ubuntu-latest-pyenv
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.pyenv
+          ~/.local/share/virtualenvs/
+        key: ubuntu-latest-python-${{ hashFiles('tools/ubuntu_setup.sh') }}-
+    - name: Cache scons
+      id: ubuntu-latest-scons
+      uses: actions/cache@v3
+      with:
+        path: /tmp/scons_cache
+        key: ubuntu-latest-scons-${{ hashFiles('.github/workflows/tools_test.yaml') }}-
+        restore-keys: |
+          ubuntu-latest-scons-${{ hashFiles('.github/workflows/tools_test.yaml') }}-
+          ubuntu-latest-scons-
+
+    - name: tools/ubuntu_setup.sh
+      run: |
+        source tools/openpilot_env.sh
+        tools/ubuntu_setup.sh
+    - name: Build openpilot
+      run: |
+        source tools/openpilot_env.sh
+        pipenv run scons -j$(nproc) --extras --test
+    - name: Cleanup scons cache
+      run: |
+        source tools/openpilot_env.sh
+        rm -rf /tmp/scons_cache/*
+        pipenv run scons -j$(nproc) --extras --test --cache-populate
+
   plotjuggler:
     name: plotjuggler
     runs-on: ubuntu-20.04

.gitignore

@@ -32,6 +32,7 @@ a.out
 *.class
 *.pyxbldc
 *.vcd
+*.qm
 config.json
 clcache
 compile_commands.json
@@ -45,12 +46,10 @@ system/proclogd/proclogd
 selfdrive/ui/_ui
 selfdrive/test/longitudinal_maneuvers/out
 selfdrive/visiond/visiond
-selfdrive/loggerd/loggerd
-selfdrive/loggerd/bootlog
 selfdrive/sensord/_gpsd
 selfdrive/sensord/_sensord
-selfdrive/camerad/camerad
-selfdrive/camerad/test/ae_gray_test
+system/camerad/camerad
+system/camerad/test/ae_gray_test
 selfdrive/modeld/_modeld
 selfdrive/modeld/_dmonitoringmodeld
 /src/

.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
     hooks:
     -   id: mypy
         exclude: '^(pyextra/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)/'
-        additional_dependencies: ['lxml', 'numpy', 'types-atomicwrites', 'types-pycurl', 'types-requests', 'types-certifi']
+        additional_dependencies: ['types-PyYAML', 'lxml', 'numpy', 'types-atomicwrites', 'types-pycurl', 'types-requests', 'types-certifi']
         args:
         - --warn-redundant-casts
         - --warn-return-any
@@ -54,7 +54,7 @@ repos:
         entry: cppcheck
         language: system
         types: [c++]
-        exclude: '^(third_party/)|(pyextra/)|(cereal/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)/|(installer/)'
+        exclude: '^(third_party/)|(pyextra/)|(cereal/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)/|(installer/)'
         args:
         - --error-exitcode=1
         - --language=c++

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=scipy cereal.messaging.messaging_pyx
+extension-pkg-whitelist=scipy,cereal.messaging.messaging_pyx,PyQt5,av
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
@@ -250,13 +250,6 @@ max-line-length=100
 # Maximum number of lines in a module
 max-module-lines=1000
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
-               dict-separator
-
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.
 single-line-class-stmt=no

Dockerfile.openpilot

@@ -8,11 +8,6 @@ ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
 RUN mkdir -p ${OPENPILOT_PATH}
 WORKDIR ${OPENPILOT_PATH}
 
-COPY Pipfile Pipfile.lock $OPENPILOT_PATH
-RUN pip install --no-cache-dir pipenv==2021.5.29 pip==21.3.1 && \
-    pipenv install --system --deploy --dev --clear && \
-    pip uninstall -y pipenv
-
 COPY SConstruct ${OPENPILOT_PATH}
 
 COPY ./pyextra ${OPENPILOT_PATH}/pyextra
@@ -30,4 +25,4 @@ COPY ./panda ${OPENPILOT_PATH}/panda
 COPY ./selfdrive ${OPENPILOT_PATH}/selfdrive
 COPY ./system ${OPENPILOT_PATH}/system
 
-RUN scons -j$(nproc)
+RUN scons --cache-readonly -j$(nproc)