Merge branch 'visit-try-orelse-finalbody-and-if-orelse' of github.com…

…:twisted/pydoctor into visit-try-orelse-finalbody-and-if-orelse

.github/workflows/pydoctor_primer.yaml

@@ -0,0 +1,117 @@
+name: Run pydoctor_primer
+
+on:
+  # Only run on PR, since we diff against master
+  pull_request:
+    paths-ignore:
+    - 'pydoctor/test/**'
+    - 'docs/**'
+    - '*.rst'
+    - '*.txt'
+    - '*.in'
+    - '*.md'
+    - '.*'
+    - 'setup.py'
+
+
+jobs:
+  pydoctor_primer:
+    name: Run pydoctor_primer
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: pydoctor_to_test
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install pydoctor_primer
+        run: |
+          python -m pip install -U pip
+          pip install git+https://github.com/twisted/pydoctor_primer.git
+      - name: Run pydoctor_primer
+        shell: bash
+        run: |
+          cd pydoctor_to_test
+          echo "new commit"
+          git rev-list --format=%s --max-count=1 $GITHUB_SHA
+
+          MERGE_BASE=$(git merge-base $GITHUB_SHA origin/$GITHUB_BASE_REF)
+          git checkout -b base_commit $MERGE_BASE
+          echo "base commit"
+          git rev-list --format=%s --max-count=1 base_commit
+
+          echo ''
+          cd ..
+          # fail action if exit code isn't zero or one
+          (
+            pydoctor_primer \
+            --repo pydoctor_to_test \
+            --new $GITHUB_SHA --old base_commit \
+            --debug \
+            --output concise \
+            -j 8 \
+            | tee diff.txt
+          ) || [ $? -eq 1 ]
+
+      - name: Post comment
+        id: post-comment
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const MAX_CHARACTERS = 30000
+            const MAX_CHARACTERS_PER_PROJECT = MAX_CHARACTERS / 3
+
+            const fs = require('fs')
+            let data = fs.readFileSync('diff.txt', { encoding: 'utf8' })
+
+            function truncateIfNeeded(original, maxLength) {
+              if (original.length <= maxLength) {
+                return original
+              }
+              let truncated = original.substring(0, maxLength)
+              // further, remove last line that might be truncated
+              truncated = truncated.substring(0, truncated.lastIndexOf('\n'))
+              let lines_truncated = original.split('\n').length - truncated.split('\n').length
+              return `${truncated}\n\n... (truncated ${lines_truncated} lines) ...`
+            }
+
+            const projects = data.split('\n\n')
+            // don't let one project dominate
+            data = projects.map(project => truncateIfNeeded(project, MAX_CHARACTERS_PER_PROJECT)).join('\n\n')
+            // posting comment fails if too long, so truncate
+            data = truncateIfNeeded(data, MAX_CHARACTERS)
+
+            console.log("Diff from pydoctor_primer:")
+            console.log(data)
+
+            let body
+            if (data.trim()) {
+              body = 'Diff from [pydoctor_primer](https://github.com/tristanlatr/pydoctor_primer), showing the effect of this PR on open source code:\n```diff\n' + data + '```'
+            } else {
+              body = "According to [pydoctor_primer](https://github.com/tristanlatr/pydoctor_primer), this change doesn't affect pydoctor warnings on a corpus of open source code. ✅"
+            }
+            const ev = JSON.parse(
+              fs.readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')
+            )
+            const prNumber = ev.pull_request.number
+
+            await github.rest.issues.createComment({
+              issue_number: prNumber,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body
+            })
+            return prNumber
+
+      - name: Hide old comments
+        uses: kanga333/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          leave_visible: 1
+          issue_number: ${{ steps.post-comment.outputs.result }}

.github/workflows/system.yaml

@@ -18,16 +18,9 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Set up CPython
-      if: ${{ matrix.tox_target == 'python-igraph-apidocs' }}
       uses: actions/setup-python@v2
       with:
-        python-version: '3.8'
-
-    - name: Set up PyPy
-      if: ${{ matrix.tox_target != 'python-igraph-apidocs' }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: 'pypy-3.6'
+        python-version: '3.11'
 
     - name: Install tox
       run: |

.github/workflows/unit.yaml

@@ -14,13 +14,13 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, 3.10.8, 3.11.0, pypy-3.6]
+        python-version: [pypy-3.7, 3.7, 3.8, 3.9, '3.10', 3.11, '3.12.0-rc.3']
         os: [ubuntu-20.04]
         include:
           - os: windows-latest
-            python-version: 3.6
+            python-version: 3.7
           - os: macos-latest
-            python-version: 3.6
+            python-version: 3.7
 
     steps:
     - uses: actions/checkout@v2
@@ -45,8 +45,8 @@ jobs:
       run: |
         tox -e test
 
-    - name: Run unit tests with latest Twisted version
-      if: matrix.python-version != '3.6' && matrix.python-version != 'pypy-3.6'
+    - name: Run unit tests with latest Twisted version (only for python 3.8 and later)
+      if: matrix.python-version != '3.7' && matrix.python-version != 'pypy-3.7'
       run: |
         tox -e test-latest-twisted
 

README.rst

@@ -76,10 +76,45 @@ What's New?
 in development
 ^^^^^^^^^^^^^^
 
+This is the last major release to support Python 3.7.
+
+* Drop support for Python 3.6
+* Add support for Python 3.12
+* `ExtRegistrar.register_post_processor()` now supports a `priority` argument that is an int.
+  Highest priority callables will be called first during post-processing.
+* Fix too noisy ``--verbose`` mode (suppres some ambiguous annotations warnings).
+
+pydoctor 23.9.1
+^^^^^^^^^^^^^^^
+
+* Fix regression in link not found warnings' line numbers.
+
+pydoctor 23.9.0
+^^^^^^^^^^^^^^^
+
+This is the last major release to support Python 3.6.
+
+* Do not show `**kwargs` when keywords are specifically documented with the `keyword` field
+  and no specific documentation is given for the `**kwargs` entry.
 * Fix annotation resolution edge cases: names are resolved in the context of the module 
   scope when possible, when impossible, the theoretical runtime scopes are used. A warning can
   be reported when an annotation name is ambiguous (can be resolved to different names 
   depending on the scope context) with option ``-v``.
+* Ensure that explicit annotation are honored when there are multiple declarations of the same name.
+* Use stricter verification before marking an attribute as constant: 
+   - instance variables are never marked as constant
+   - a variable that has several definitions will not be marked as constant
+   - a variable declaration under any kind of control flow block will not be marked as constant
+* Do not trigger warnings when pydoctor cannot make sense of a potential constant attribute 
+  (pydoctor is not a static checker).
+* Fix presentation of type aliases in string form.
+* Improve the AST colorizer to output less parenthesis when it's not required.
+* Fix colorization of dictionary unpacking.
+* Improve the class hierarchy such that it links top level names with intersphinx when possible.
+* Add highlighting when clicking on "View In Hierarchy" link from class page.
+* Recognize variadic generics type variables (PEP 646).
+* Fix support for introspection of cython3 generated modules.
+* Instance variables are marked as such across subclasses.
 
 pydoctor 23.4.1
 ^^^^^^^^^^^^^^^

docs/source/codedoc.rst

@@ -338,7 +338,8 @@ Branch priorities
 -----------------
 
 When pydoctor deals with try/except/else or if/else block, it makes sure that the names defined in 
-the "principal" branch do not get overriden by names defined in the except hanlders or ifs' else block. 
+the "principal" flow do not get overridden by names defined in the except handlers or ifs' else block. 
+New names defined in these block will be processed normally.
 
 Meaning that in the context of the code below, ``ssl`` would resolve to ``twisted.internet.ssl``:
 

docs/source/contrib.rst

@@ -17,23 +17,65 @@ If you like the project and think you could help with making it better, there ar
 Any contribution would be of great help and I will highly appreciate it! If you have any questions, please create a new issue.
 
 
-Pre-commit checks
------------------
+Development process
+-------------------
 
-Make sure all the tests pass and the code pass the coding standard checks::
+Create a fork of the git repository and checkout a new branch from ``master`` branch. 
+The branch name may start with an associated issue number so that we can easily 
+cross-reference them. For example, use ``1234-some-brach-name`` as the name of the branch working to fix issue ``1234``.
+Once you're ready to run a full batterie of tests to your changes, open a pull request.
 
-    tox -p all
+Don't forget to sync your fork once in while to work from the latest revision.
 
-That should be the minimum check to run on your local system.
+Pre-commit checks
+-----------------
+
+Make sure all the unit tests pass and the code pass the coding standard checks.
+
+We use `tox <https://tox.wiki/en/stable/>`_ for running our checks, but you can roughly do the same thing from your python environment. 
+
+.. list-table:: Pre-commit checks
+   :widths: 10 45 45
+   :header-rows: 1
+
+   * - \
+     - Using `tox`
+     - Using your environment
+   * - Run unit tests
+     - ``tox -e test``
+     - ``pip install '.[test]' && pytest pydoctor``
+   * - Run pyflakes
+     - ``tox -e pyflakes``
+     - ``pip install pyflakes && find pydoctor/ -name \*.py ! -path '*/testpackages/*' ! -path '*/sre_parse36.py' ! -path '*/sre_constants36.py' | xargs pyflakes``
+   * - Run mypy
+     - ``tox -e mypy``
+     - ``pip install '.[mypy]' && mypy pydoctor``
+   * - Run pydoctor on it's own source
+     - ``tox -e apidocs``
+     - ``pip install . && pydoctor --privacy "HIDDEN:pydoctor.test" -q -W pydoctor``
+
+These should be the minimum check to run on your local system.
 A pull request will trigger more tests and most probably there is a tox
 environment dedicated to that extra test.
 
+Other things hapenning when a PR is open
+----------------------------------------
+
+- System tests: these tests checks if pydoctor can generate the documentation for a few
+  specific packages that have been considered as problematic in the past.
+- Pydoctor primer: this is to pydoctor what ``mypy_primer`` is to ``mypy``. 
+  It runs pydoctor on a corpus of open source code and compares the output of the application before and after a modification in the code.
+  Then it reports in comments the result for a PR. The source code of this tool is here: https://github.com/twisted/pydoctor_primer.
+- Readthedocs build: For every PR, the sphinx documentation is built and available at ``https://pydoctor--{pr-number}.org.readthedocs.build/en/``.
 
 Review process and requirements
 -------------------------------
 
 - Code changes and code added should have tests: untested code is buggy code. Except special cases, overall test coverage should be increased.
 - If your pull request is a work in progress, please mark it as draft such that reviewers do not loose time on a PR that is not ready yet.
+- There is no strict coding style standard. Since pydoctor is more than 20 years old and we have vendored some code from 
+  other packages as well (namely epydoc and sre_parse), so we can’t really enforce the same style everywhere. It's up to the reviewers
+  to request refactors when the code is too ugly.
 - All code changes must be reviewed by at least one person who is not an author of the code being added. 
   This helps prevent bugs from slipping through the net and gives another source for improvements.
   If the author of the PR is one of the core developers of pydoctor* and no one has reviewed their PR after 9 calendar days, they can review the code changes themselves and proceed with next steps. 
@@ -123,6 +165,32 @@ Such new packages shouldn't get vendored. They need to be packaged in
 Debian. Best is to get in contact with the DPT to talk about about new
 requirements and the best way to get things done.
 
+Profiling pydoctor with austin and speedscope
+---------------------------------------------
+
+1. Install austin (https://github.com/P403n1x87/austin)
+2. Install austin-python (https://pypi.org/project/austin-python/)
+3. Run program under austin 
+
+   .. code::
+
+      $ sudo austin -i 1ms -C -o pydoctor.austin pydoctor <pydoctor args>
+
+4. Convert .austin to .speedscope (austin2speedscope comes from austin-python)
+
+   .. code::
+  
+    $ austin2speedscope pydoctor.austin pydoctor.speedscope
+
+
+5. Open https://speedscope.app and load pydoctor.speedscope into it.
+
+Note on sampling interval
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On our large repo I turn down the sampling interval from 100us to 1ms to make 
+the resulting ``.speedscope`` file a manageable size (15MB instead of 158MB which is too large to put into a gist.)
+
 Author Design Notes
 -------------------
 

docs/source/quickstart.rst

@@ -54,3 +54,16 @@ Output files are static HTML pages which require no extra server-side support.
 
 Here is a `GitHub Action example <publish-github-action.html>`_ to automatically
 publish your API documentation to your default GitHub Pages website.
+
+Return codes
+------------
+
+Pydoctor is a pretty verbose tool by default. It’s quite unlikely that you get a zero exit code on the first run. 
+But don’t worry, pydoctor should have produced useful HTML pages no matter your project design or docstrings. 
+
+Exit codes includes:
+
+- ``0``: All docstrings are well formatted (warnings may be printed).
+- ``1``: Pydoctor crashed with traceback (default Python behaviour).
+- ``2``: Some docstrings are mal formatted.
+- ``3``: Pydoctor detects some warnings and ``--warnings-as-errors`` is enabled.

docs/tests/test_standard_library_docs.py

@@ -23,4 +23,11 @@ def test_std_lib_docs() -> None:
         elif entry.is_dir() and entry.joinpath('__init__.py').exists(): # Package
             assert BASE_DIR.joinpath('Lib.'+entry.name+'.html').exists()
 
-
+def test_std_lib_logs() -> None:
+    """
+    'Cannot parse file' do not appear too much.
+    This test expect a run.log file in cpython-output directory
+    """
+    log = (BASE_DIR / 'run.log').read_text()
+    assert log.count('cannot parse file') == 4
+

pydoctor/_configparser.py

@@ -20,6 +20,8 @@
 >>> parser = ArgumentParser(..., default_config_files=['./pyproject.toml', 'setup.cfg', 'my_super_tool.ini'], config_file_parser_class=MixedParser)
 
 """
+from __future__ import annotations
+
 import argparse
 from collections import OrderedDict
 import re