Add django model annotation reporting

Author: bmedx

Diff for: .annotations_sample

@@ -1,5 +1,6 @@
-source_path: ../devstack-docker/edx-platform
+source_path: ../
 report_path: reports
+safelist_path: .pii_safe_list.yml
 annotations:
     pii:
         - ".. pii::"

Diff for: .coveragerc

@@ -4,8 +4,4 @@ data_file = .coverage
 source=code_annotations
 omit =
     test_settings
-    *migrations*
-    *admin.py
-    *static*
-    *templates*
     code_annotations/extensions/tests/python_test_files/*

Diff for: code_annotations/find_annotations.py renamed to code_annotations/base.py

@@ -6,47 +6,45 @@
 import os
 import pprint
 import re
+from abc import ABCMeta, abstractmethod
 
 import six
 import yaml
-from stevedore import named
 
 from code_annotations.exceptions import ConfigurationException
 from code_annotations.helpers import VerboseEcho, read_configuration
 
 
-class StaticSearch(object):
+@six.add_metaclass(ABCMeta)
+class BaseSearch(object):
     """
-    Handles static code searching for annotations.
+    Base class for searchers.
     """
 
-    def __init__(self, config, source_path, report_path, verbosity):
+    def __init__(self, config, report_path, verbosity):
         """
         Initialize for StaticSearch.
 
         Args:
             config: Configuration file path
-            source_path: Directory to be searched for annotations
             report_path: Directory to write the report file to
             verbosity: Integer representing verbosity level (0-3)
         """
         self.config = {}
         self.errors = []
         self.groups = {}
         self.choices = {}
-        self.mgr = None
 
         # Global logger for this script, shared with extensions
         self.echo = VerboseEcho()
-        self.configure(config, source_path, report_path, verbosity)
+        self.configure(config, report_path, verbosity)
 
-    def configure(self, config_file_path, source_path, report_path, verbosity):
+    def configure(self, config_file_path, report_path, verbosity):
         """
         Read the configuration file and handle command line overrides.
 
         Args:
             config_file_path: Location of the configuration file
-            source_path: Path to the code to be searched
             report_path: Directory where the report will be generated
             verbosity: Integer indicating the runtime verbosity level
 
@@ -58,65 +56,21 @@ def configure(self, config_file_path, source_path, report_path, verbosity):
 
         self.config = read_configuration(config_file_path)
 
-        if not source_path and 'source_path' not in self.config:
-            raise ConfigurationException('source_path not given and not in configuration file')
-
-        if not report_path and 'report_path' not in self.config:
+        if 'report_path' not in self.config and not report_path:
             raise ConfigurationException('report_path not given and not in configuration file')
 
-        if source_path:
-            self.config['source_path'] = source_path
-
         if report_path:
             self.config['report_path'] = report_path
 
         self.config['verbosity'] = verbosity
         self.echo.set_verbosity(verbosity)
 
-        self.configure_extensions()
         self.configure_groups_and_choices()
 
         self.echo.echo_v("Verbosity level set to {}".format(verbosity))
         self.echo.echo_v("Configuration:")
         self.echo.echo_v(self.config)
-        self.echo(
-            "Configured for source path: {}, report path: {}".format(
-                self.config['source_path'],
-                self.config['report_path'])
-        )
-
-    def configure_extensions(self):
-        """
-        Configure the Stevedore NamedExtensionManager.
-
-        Raises:
-            ConfigurationException
-        """
-        # These are the names of all of our configured extensions
-        configured_extension_names = self.config['extensions'].keys()
-
-        # Load Stevedore extensions that we are configured for (and only those)
-        self.mgr = named.NamedExtensionManager(
-            names=configured_extension_names,
-            namespace='annotation_finder.searchers',
-            invoke_on_load=True,
-            on_load_failure_callback=self.load_failed_handler,
-            invoke_args=(self.config, self.echo),
-        )
-
-        # Output extension names listed in configuration
-        self.echo.echo_vv("Configured extension names: {}".format(" ".join(configured_extension_names)))
-
-        # Output found extension entry points from setup.py|cfg (whether or not they were loaded)
-        self.echo.echo_vv("Stevedore entry points found: {}".format(str(self.mgr.list_entry_points())))
-
-        # Output extensions that were actually able to load
-        self.echo.echo_v("Loaded extensions: {}".format(" ".join([x.name for x in self.mgr.extensions])))
-
-        if len(self.mgr.extensions) != len(configured_extension_names):
-            raise ConfigurationException('Not all configured extensions could be loaded! Asked for {} got {}.'.format(
-                configured_extension_names, self.mgr.extensions
-            ))
+        self.echo("Configured for report path: {}".format(self.config['report_path']))
 
     def configure_groups_and_choices(self):
         """
@@ -166,50 +120,6 @@ def configure_groups_and_choices(self):
         self.echo.echo_v("Groups configured: {}".format(self.groups))
         self.echo.echo_v("Choices configured: {}".format(self.choices))
 
-    def load_failed_handler(self, *args, **kwargs):
-        """
-        Handle failures to load an extension.
-
-        Dumps the error and raises an exception. By default these
-        errors just fail silently.
-
-        Args:
-            *args:
-            **kwargs:
-
-        Raises:
-            ConfigurationException
-        """
-        self.echo(args)
-        self.echo(kwargs)
-        raise ConfigurationException('Failed to load a plugin, aborting.')
-
-    def search_extension(self, ext, file_handle, file_extensions_map, filename_extension):
-        """
-        Execute a search on the given file using the given extension.
-
-        Args:
-            ext: Extension to execute the search on
-            file_handle: An open file handle search
-            file_extensions_map: Dict mapping of extension names to configured filename extensions
-            filename_extension: The filename extension of the file being searched
-
-        Returns:
-            Tuple of (extension name, list of found annotation dicts)
-        """
-        # Only search this file if we are configured for its extension
-        if filename_extension in file_extensions_map[ext.name]:
-            # Reset the read handle to the beginning of the file in case another
-            # extension moved it
-            file_handle.seek(0)
-
-            ext_results = ext.obj.search(file_handle)
-
-            if ext_results:
-                return ext.name, ext_results
-
-        return ext.name, None
-
     def format_file_results(self, all_results, results):
         """
         Add all extensions' search results for a file to the overall results.
@@ -221,9 +131,7 @@ def format_file_results(self, all_results, results):
         Returns:
             None, modifies all_results
         """
-        # "_" here is the extension name, as required by Stevedore map(). Each
-        # annotation already has the extension name so we can ignore it
-        for _, annotations in results:
+        for annotations in results:
             if not annotations:
                 continue
 
@@ -259,13 +167,15 @@ def _check_results_choices(self, annotation):
         Args:
             annotation: A single search result dict.
         """
+        # Not a choice type of annotation, nothing to do
         if annotation['annotation_token'] not in self.choices:
             return
 
         token = annotation['annotation_token']
         found_valid_choices = []
 
-        # If there are no choices on the line, split will return this
+        # If the line begins with an annotation token that should have choices, but has no text after the token,
+        # the first split will be empty.
         if annotation['annotation_data'][0] != "":
             for choice in annotation['annotation_data']:
                 if choice not in self.choices[token]:
@@ -383,62 +293,30 @@ def _add_annotation_error(self, annotation, message):
             annotation: A single annotation dict found in search()
             message: Custom error message to be added
         """
-        error = "{}::{}: {}".format(annotation['filename'], annotation['line_number'], message)
+        if 'extra' in annotation and 'object_id' in annotation['extra']:
+            error = "{}::{}: {}".format(annotation['filename'], annotation['extra']['object_id'], message)
+        else:
+            error = "{}::{}: {}".format(annotation['filename'], annotation['line_number'], message)
         self.errors.append(error)
 
-    def _search_one_file(self, full_name, known_extensions, file_extensions_map, all_results):
+    def _add_error(self, message):
         """
-        Perform an annotation search on a single file, using all extensions it is configured for.
+        Add an error message to self.errors.
 
         Args:
-            full_name: Complete filename
-            known_extensions: List of all file name extensions we are configured to work on
-            file_extensions_map: Mapping of file name extensions to Stevedore extensions
-            all_results: A dict of annotations returned from search()
+            message: Custom error message to be added
         """
-        filename_extension = os.path.splitext(full_name)[1][1:]
-
-        if filename_extension not in known_extensions:
-            self.echo.echo_vvv(
-                "{} is not a known extension, skipping ({}).".format(filename_extension, full_name)
-            )
-            return
-
-        self.echo.echo_vvv(full_name)
-
-        # TODO: This should probably be a generator so we don't have to store all results in memory
-        with open(full_name, 'r') as file_handle:
-            # Call search_extension on all loaded extensions
-            results = self.mgr.map(self.search_extension, file_handle, file_extensions_map, filename_extension)
-
-            # Format and add the results to our running full set
-            self.format_file_results(all_results, results)
+        self.errors.append(message)
 
+    @abstractmethod
     def search(self):
         """
         Walk the source tree, send known file types to extensions.
 
         Returns:
-            Filename of the generated report
+            Dict of {filename: annotations} for all files with found annotations.
         """
-        # Index the results by extension name
-        file_extensions_map = {}
-        known_extensions = set()
-        for extension_name in self.config['extensions']:
-            file_extensions_map[extension_name] = self.config['extensions'][extension_name]
-            known_extensions.update(self.config['extensions'][extension_name])
-
-        all_results = {}
-
-        if os.path.isfile(self.config['source_path']):
-            self._search_one_file(self.config['source_path'], known_extensions, file_extensions_map, all_results)
-        else:
-            for root, _, files in os.walk(self.config['source_path']):
-                for filename in files:
-                    full_name = os.path.join(root, filename)
-                    self._search_one_file(full_name, known_extensions, file_extensions_map, all_results)
-
-        return all_results
+        pass  # pragma: no cover
 
     def report(self, all_results):
         """