Add option --privacy

README.rst

@@ -72,9 +72,11 @@ What's New?
 
 in development
 ^^^^^^^^^^^^^^
+* Add option ``--privacy`` to set the privacy of specific objects when default rules doesn't fit the use case.
 * Option ``--docformat=plaintext`` overrides any assignments to ``__docformat__`` 
   module variable in order to focus on potential python code parsing errors.
 
+
 pydoctor 22.3.0
 ^^^^^^^^^^^^^^^
 * Add client side search system based on lunr.js.

docs/source/codedoc.rst

@@ -293,6 +293,9 @@ Modules, classes and functions of which the name starts with an underscore are c
             This method is public.
             """
 
+.. note::
+    Pydoctor actually supports 3 types of privacy: public, private and hidden. 
+    See :ref:`Override objects privacy <customize-privacy>` for more informations.
 
 Re-exporting
 ------------

docs/source/conf.py

@@ -104,7 +104,8 @@
         '--html-output={outdir}/api/',  # Make sure to have a trailing delimiter for better usage coverage.
         '--project-name=pydoctor',
         f'--project-version={version}',
-        '--docformat=epytext',
+        '--docformat=epytext', 
+        '--privacy=HIDDEN:pydoctor.test',
         '--project-url=../index.html',
         f'{_pydoctor_root}/pydoctor',
         ] + _common_args,
@@ -113,6 +114,7 @@
         '--project-name=pydoctor with a twisted theme',
         f'--project-version={version}',
         '--docformat=epytext',
+        '--privacy=HIDDEN:pydoctor.test',
         '--project-url=../customize.html',
         '--theme=base',
         f'--template-dir={_pydoctor_root}/docs/sample_template',

docs/source/customize.rst

@@ -32,6 +32,53 @@ HTML templates have their own versioning system and warnings will be triggered w
     This example is using new ``pydoctor`` option, ``--theme=base``. 
     This means that bootstrap CSS will not be copied to build directory.
 
+.. _customize-privacy:
+
+Override objects privacy (show/hide)
+------------------------------------
+
+Pydoctor supports 3 types of privacy.
+Below is the description of each type and the default association:
+
+- ``PRIVATE``: By default for objects whose name starts with an underscore and are not a dunder method. 
+  Rendered in HTML, but hidden via CSS by default.
+
+- ``PUBLIC``: By default everything else that is not private.
+  Always rendered and visible in HTML.
+
+- ``HIDDEN``: Nothing is hidden by default.
+  Not rendered at all and no links can be created to hidden objects. 
+  Not present in the search index nor the intersphinx inventory.
+  Basically excluded from API documentation. If a module/package/class is hidden, then all it's members are hidden as well.
+
+When the default rules regarding privacy doesn't fit your use case,
+use the ``--privacy`` command line option.
+It can be used multiple times to define multiple privacy rules::
+
+  --privacy=<PRIVACY>:<PATTERN>
+
+where ``<PRIVACY>`` can be one of ``PUBLIC``, ``PRIVATE`` or ``HIDDEN`` (case insensitive), and ``<PATTERN>`` is fnmatch-like 
+pattern matching objects fullName.
+
+Privacy tweak examples
+^^^^^^^^^^^^^^^^^^^^^^
+- ``--privacy="PUBLIC:**"``
+  Makes everything public.
+
+- ``--privacy="HIDDEN:twisted.test.*" --privacy="PUBLIC:twisted.test.proto_helpers"``
+  Makes everything under ``twisted.test`` hidden except ``twisted.test.proto_helpers``, which will be public.
+
+- ``--privacy="PRIVATE:**.__*__" --privacy="PUBLIC:**.__init__"``
+  Makes all dunder methods private except ``__init__``.
+
+.. important:: The order of arguments matters. Pattern added last have priority over a pattern added before,
+  but an exact match wins over a fnmatch.
+
+.. note:: See :py:mod:`pydoctor.qnmatch` for more informations regarding the pattern syntax.
+
+.. note:: Quotation marks should be added around each rule to avoid shell expansions.
+    Unless the arguments are passed directly to pydoctor, like in Sphinx's ``conf.py``, in this case you must not quote the privacy rules.
+
 Use a custom system class
 -------------------------
 

docs/tests/test.py

@@ -5,8 +5,10 @@
 #
 import os
 import pathlib
-import json
 from typing import List
+import xml.etree.ElementTree as ET
+import json
+
 from lunr.index import Index
 
 from sphinx.ext.intersphinx import inspect_main
@@ -207,3 +209,27 @@ def test_search(query:str, expected:List[str]) -> None:
                 ['pydoctor.epydoc.markup.restructuredtext.ParsedRstDocstring'])
         test_search('pydoctor.epydoc.markup.restructuredtext.ParsedRstDocstring', 
                 ['pydoctor.epydoc.markup.restructuredtext.ParsedRstDocstring'])
+
+def test_pydoctor_test_is_hidden():
+    """
+    Test that option --privacy=HIDDEN:pydoctor.test makes everything under pydoctor.test HIDDEN.
+    """
+
+    def getText(node: ET.Element) -> str:
+        return ''.join(node.itertext()).strip().replace('\u200b', '')
+
+    with open(BASE_DIR / 'api' / 'all-documents.html', 'r', encoding='utf-8') as stream:
+        document = ET.fromstring(stream.read())
+        for liobj in document.findall('body/div/ul/li[@id]'):
+
+            if not str(liobj.get("id")).startswith("pydoctor"):
+                continue # not a all-documents list item, maybe in the menu or whatever.
+
+            # figure obj name
+            fullName = getText(liobj.findall('./div[@class=\'fullName\']')[0])
+
+            if fullName.startswith("pydoctor.test"):
+                # figure obj privacy
+                privacy = getText(liobj.findall('./div[@class=\'privacy\']')[0])
+                # check that it's indeed private
+                assert privacy == 'HIDDEN'

pydoctor/driver.py

@@ -84,9 +84,23 @@ def parse_path(option: Option, opt: str, value: str) -> Path:
     except Exception as ex:
         raise OptionValueError(f"{opt}: invalid path: {ex}")
 
+def parse_privacy_tuple(option: Option, opt: str, val:str) -> Tuple[model.PrivacyClass, str]:
+    """
+    Parse string like 'public:match*' to a tuple (PrivacyClass.PUBLIC, 'match*').
+    """
+    parts = val.split(':')
+    if len(parts)!=2:
+        raise OptionValueError(f"{opt}: malformatted value {val!r} should be like '<privacy>:<PATTERN>'.")
+    try:
+        priv = model.PrivacyClass[parts[0].strip().upper()]
+    except:
+        raise OptionValueError(f"{opt}: unknown privacy value {parts[0]!r} should be one of {', '.join(repr(m.name) for m in model.PrivacyClass)}")
+    else:
+        return (priv, parts[1].strip())
+
 class CustomOption(Option):
-    TYPES = Option.TYPES + ("path",)
-    TYPE_CHECKER = dict(Option.TYPE_CHECKER, path=parse_path)
+    TYPES = Option.TYPES + ("path","privacy_tuple")
+    TYPE_CHECKER = dict(Option.TYPE_CHECKER, path=parse_path, privacy_tuple=parse_privacy_tuple)
 
 def getparser() -> OptionParser:
     parser = OptionParser(
@@ -158,6 +172,13 @@ def getparser() -> OptionParser:
         choices=list(get_themes()) ,
         help=("The theme to use when building your API documentation. "),
     )
+    parser.add_option(
+        '--privacy', action='append', dest='privacy',
+        metavar='<PRIVACY>:<PATTERN>', default=[], type="privacy_tuple",
+        help=("Set the privacy of specific objects when default rules doesn't fit the use case. "
+              "Format: '<PRIVACY>:<PATTERN>', where <PRIVACY> can be one of 'PUBLIC', 'PRIVATE' or "
+              "'HIDDEN' (case insensitive), and <PATTERN> is fnmatch-like pattern matching objects fullName. "
+              "Pattern added last have priority over a pattern added before, but an exact match wins over a fnmatch. Can be repeated."))
     parser.add_option(
         '--html-subject', dest='htmlsubjects', action='append',
         help=("The fullName of objects to generate API docs for"

pydoctor/model.py

@@ -24,6 +24,7 @@
 from collections import OrderedDict
 from urllib.parse import quote
 
+from pydoctor import qnmatch
 from pydoctor.epydoc.markup import ParsedDocstring
 from pydoctor.sphinx import CacheT, SphinxInventory
 
@@ -76,13 +77,15 @@ class PrivacyClass(Enum):
 
     @cvar HIDDEN: Don't show the object at all.
     @cvar PRIVATE: Show, but de-emphasize the object.
-    @cvar VISIBLE: Show the object as normal.
+    @cvar PUBLIC: Show the object as normal.
     """
 
     HIDDEN = 0
     PRIVATE = 1
-    VISIBLE = 2
-
+    PUBLIC = 2
+    # For compatibility
+    VISIBLE = PUBLIC
+
 class DocumentableKind(Enum):
     """
     L{Enum} containing values indicating the possible object types.
@@ -334,15 +337,19 @@ def isVisible(self) -> bool:
 
         This is just a simple helper which defers to self.privacyClass.
         """
-        return self.privacyClass is not PrivacyClass.HIDDEN
+        isVisible = self.privacyClass is not PrivacyClass.HIDDEN
+        # If a module/package/class is hidden, all it's members are hidden as well.
+        if isVisible and self.parent:
+            isVisible = self.parent.isVisible
+        return isVisible
 
     @property
     def isPrivate(self) -> bool:
         """Is this object considered private API?
 
         This is just a simple helper which defers to self.privacyClass.
         """
-        return self.privacyClass is not PrivacyClass.VISIBLE
+        return self.privacyClass is not PrivacyClass.PUBLIC
 
     @property
     def module(self) -> 'Module':
@@ -651,6 +658,11 @@ def __init__(self, options: Optional[Values] = None):
         self.buildtime = datetime.datetime.now()
         self.intersphinx = SphinxInventory(logger=self.msg)
 
+        # since privacy handling now uses fnmatch, we cache results so we don't re-run matches all the time.
+        # it's ok to cache privacy class results since the potential renames (with reparenting) happends before we begin to
+        # generate HTML, which is when we call Documentable.PrivacyClass.
+        self._privacyClassCache: Dict[int, PrivacyClass] = {}
+
     @property
     def root_names(self) -> Collection[str]:
         """The top-level package/module names in this system."""
@@ -762,12 +774,39 @@ def objectsOfType(self, cls: Type[T]) -> Iterator[T]:
                 yield o
 
     def privacyClass(self, ob: Documentable) -> PrivacyClass:
+        ob_id = id(ob)
+        cached_privacy = self._privacyClassCache.get(ob_id)
+        if cached_privacy is not None:
+            return cached_privacy
+
+        # kind should not be None, this is probably a relica of a past age of pydoctor.
+        # but keep it just in case.
         if ob.kind is None:
             return PrivacyClass.HIDDEN
+
+        privacy = PrivacyClass.PUBLIC
         if ob.name.startswith('_') and \
                not (ob.name.startswith('__') and ob.name.endswith('__')):
-            return PrivacyClass.PRIVATE
-        return PrivacyClass.VISIBLE
+            privacy = PrivacyClass.PRIVATE
+
+        # Precedence order: CLI arguments order
+        # Check exact matches first, then qnmatch
+        fullName = ob.fullName()
+        _found_exact_match = False
+        for priv, match in reversed(self.options.privacy):
+            if fullName == match:
+                privacy = priv
+                _found_exact_match = True
+                break
+        if not _found_exact_match:
+            for priv, match in reversed(self.options.privacy):
+                if qnmatch.qnmatch(fullName, match):
+                    privacy = priv
+                    break
+
+        # Store in cache
+        self._privacyClassCache[ob_id] = privacy
+        return privacy
 
     def addObject(self, obj: Documentable) -> None:
         """Add C{object} to the system."""

pydoctor/qnmatch.py

@@ -0,0 +1,71 @@
+"""
+Provides a modified L{fnmatch} function specialized for python objects fully qualified name pattern matching.
+
+Special patterns are::
+
+    **      matches everything (recursive)
+    *       matches everything except "." (one level ony)
+    ?       matches any single character
+    [seq]   matches any character in seq
+    [!seq]  matches any char not in seq
+"""
+import functools
+import re
+from typing import Any, Callable
+
+@functools.lru_cache(maxsize=256, typed=True)
+def _compile_pattern(pat: str) -> Callable[[str], Any]:
+    res = translate(pat)
+    return re.compile(res).match
+
+def qnmatch(name:str, pattern:str) -> bool:
+    """Test whether C{name} matches C{pattern}.
+    """
+    match = _compile_pattern(pattern)
+    return match(name) is not None
+
+# Barely changed from https://github.com/python/cpython/blob/3.8/Lib/fnmatch.py
+# Not using python3.9+ version because implementation is significantly more complex.
+def translate(pat:str) -> str:
+    """Translate a shell PATTERN to a regular expression.
+    There is no way to quote meta-characters.
+    """
+    i, n = 0, len(pat)
+    res = ''
+    while i < n:
+        c = pat[i]
+        i = i+1
+        if c == '*':
+            # Changes begins: understands '**'.
+            if i < n and pat[i] == '*':
+                res = res + '.*?'
+                i = i + 1
+            else:
+                res = res + r'[^\.]*?'
+            # Changes ends.
+        elif c == '?':
+            res = res + '.'
+        elif c == '[':
+            j = i
+            if j < n and pat[j] == '!':
+                j = j+1
+            if j < n and pat[j] == ']':
+                j = j+1
+            while j < n and pat[j] != ']':
+                j = j+1
+            if j >= n:
+                res = res + '\\['
+            else:
+                stuff = pat[i:j]
+                # Changes begins: simplifications handling backslashes and hyphens not required for fully qualified names.
+                stuff = stuff.replace('\\', r'\\')
+                i = j+1
+                if stuff[0] == '!':
+                    stuff = '^' + stuff[1:]
+                elif stuff[0] in ('^', '['):
+                    stuff = '\\' + stuff
+                res = '%s[%s]' % (res, stuff)
+                # Changes ends.
+        else:
+            res = res + re.escape(c)
+    return r'(?s:%s)\Z' % res