feat+ci+lint: add incremental linting

* add `inv lint` command to tasks.py
* add `lint` workflow to .github/workflows/lint.yaml
* update README.md with linting instructions

Author: dshemetov

.github/workflows/ci.yaml

@@ -65,7 +65,7 @@ jobs:
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
           docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis
-          
+
 
       - run: |
           wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for

.github/workflows/lint.yaml

@@ -0,0 +1,29 @@
+name: Lint
+
+on:
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+          cache: "pip"
+          cache-dependency-path: "requirements.lint.txt"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.lint.txt
+      - name: Commit Range
+        id: commit-range
+        uses: akaihola/darker/.github/actions/[email protected]
+      - name: Run lint
+        run: |
+          inv lint --revision=${{ steps.commit-range.outputs.commit-range }}

.gitignore

@@ -43,3 +43,5 @@ __pycache__/
 rsconnect/
 
 *.Rproj
+venv
+env

README.md

@@ -64,6 +64,45 @@ $ cd repos
 $ pip install -e . --config-settings editable_mode=strict
 ```
 
+## Linting and Formatting
+
+The command `inv lint` will lint your changes (using
+[lint-diffs](https://github.com/AtakamaLLC/lint-diffs/)) and `inv format` will
+format your changes (using [darker](https://github.com/akaihola/darker)). This
+will allow us to incrementally bring this repo up to PEP8 compliance. There is
+[a CI action](.github/workflows/lint.yaml) to ensure that all new code is
+compliant.
+
+To setup the linter commands locally, run the following commands in the root of
+the repository:
+
+```sh
+# Create a virtual environment
+python -m venv venv
+source venv/bin/activate
+# Install lint dependencies
+pip install -r requirements.lint.txt
+# Run lint
+inv lint
+# Run formatter (will tell you which files it modified)
+inv format
+```
+
+If you get the error `ERROR:darker.git:fatal: Not a valid commit name <hash>`,
+then it's likely because your local main branch is not up to date; either you
+need to rebase or merge. Note that `darker` reads from `pyproject.toml` for
+default settings.
+
+If the lines you change are in a file that uses 2 space indentation, `darker`
+will indent the lines around your changes and not the rest, which will likely
+break the code; in that case, you should probably just pass the whole file
+through black. You can do that with the following command (using the same
+virtual environment as above):
+
+```sh
+env/bin/black <file>
+```
+
 # COVIDcast
 
 At the present, our primary focus is developing and expanding the

pyproject.toml

@@ -1,26 +1,37 @@
 [tool.black]
-line-length = 100
+line-length = 120
 target-version = ['py38']
-include = 'server,tests/server'
+
+[tool.darker]
+revision = 'origin/dev...'
+color = true
+isort = true
+
+[tool.isort]
+profile = "black"
+known_third_party = ["pytest"]
 
 [tool.pylint]
-    [tool.pylint.'MESSAGES CONTROL']
-    max-line-length = 100
-    disable = [
-        'logging-format-interpolation',
-        # Allow pytest functions to be part of a class
-        'no-self-use',
-        'too-many-locals',
-        'too-many-arguments',
-        # Allow pytest classes to have one test
-        'too-few-public-methods',
-    ]
+[tool.pylint.main]
+max-line-length = 120
+disable = [
+    'logging-format-interpolation',
+    # Allow pytest functions to be part of a class
+    'no-self-use',
+    'too-many-locals',
+    'too-many-arguments',
+    'too-many-branches',
+    'too-many-statements',
+    # Allow pytest classes to have one test
+    'too-few-public-methods',
+]
+enable = 'useless-suppression'
 
-    [tool.pylint.'BASIC']
-    # Allow arbitrarily short-named variables.
-    variable-rgx = ['[a-z_][a-z0-9_]*']
-    argument-rgx = [ '[a-z_][a-z0-9_]*' ]
-    attr-rgx = ['[a-z_][a-z0-9_]*']
+[tool.pylint.basic]
+# Allow arbitrarily short-named variables.
+variable-rgx = '[A-Za-z_][a-z0-9_]*'
+argument-rgx = '[A-Za-z_][a-z0-9_]*'
+attr-rgx = '[A-Za-z_][a-z0-9_]*'
 
-    [tool.pylint.'DESIGN']
-    ignored-argument-names = ['(_.*|run_as_module)']
+[tool.pylint.design]
+ignored-argument-names = ['(_.*|run_as_module)']

requirements.lint.txt

@@ -0,0 +1,6 @@
+# Keep versions in sync with cmu-delphi/covidcast-indicators
+darker[isort]~=2.1.1
+invoke>=1.4.1
+lint_diffs==0.1.22
+pylint==2.8.3
+requests

tasks.py

@@ -1,3 +1,8 @@
+"""Repo tasks."""
+
+import pathlib
+
+import requests
 from invoke import task
 
 
@@ -12,9 +17,6 @@ def update_gdoc(
     sources_url="https://docs.google.com/spreadsheets/d/e/2PACX-1vRfXo-qePhrYGAoZqewVnS1kt9tfnUTLgtkV7a-1q7yg4FoZk0NNGuB1H6k10ah1Xz5B8l1S1RB17N6/pub?gid=0&single=true&output=csv",
     signal_url="https://docs.google.com/spreadsheets/d/e/2PACX-1vRfXo-qePhrYGAoZqewVnS1kt9tfnUTLgtkV7a-1q7yg4FoZk0NNGuB1H6k10ah1Xz5B8l1S1RB17N6/pub?gid=329338228&single=true&output=csv",
 ):
-    import requests
-    import pathlib
-
     base_dir = pathlib.Path("./src/server/endpoints/covidcast_utils/")
 
     def _migrate_file(url: str, filename: str):
@@ -26,3 +28,25 @@ def _migrate_file(url: str, filename: str):
 
     _migrate_file(sources_url, "db_sources.csv")
     _migrate_file(signal_url, "db_signals.csv")
+
+
+@task
+def lint(c, revision="origin/dev"):
+    """Lint.
+
+    Linters automatically read from `pyproject.toml`.
+    """
+    c.run(f"darker --diff --check --revision {revision} .", warn=True)
+    c.run("echo '\n'")
+    out = c.run(f"git diff -U0 {revision} | lint-diffs")
+    if out.stdout.strip() != "=== pylint: mine=0, always=0":
+        print(out.stdout)
+
+
+@task
+def format(c, revision="origin/dev"):  # pylint: disable=redefined-builtin
+    """Format.
+
+    Darker will format files for you and print which ones changed.
+    """
+    c.run(f"darker --verbose --revision {revision} .", warn=True)