Literate programming (#515)

* add script to parse code blocks

* add extra checks, remove current checks

* add code-blocks to style-guide

* ignore generated files

* print only severe messages

* fix newlines

* fix comma regex, addresses getodk/docs#471

* custom check for but, addresses getodk/docs#467

* exclude inbuilt but and spelling checks, addresses getodk/docs#467

* add encoding

* improve uk us check, addresses getodk/docs#471 (comment)

* fix messages

* separate checks for jargon

* add comment for uk-us check

* add extra latin abbr

* add new adverbs

* remove repeated phrase

* fix message writing style

* fix variable naming

* add function to remove generated scripts

* add show hide feature for style test code blocks

* add python directive to ignore list

* add math directive to ignore list

Author: ankita240796

.gitignore

@@ -1,3 +1,5 @@
+style-checks.py
+extra.py
 build/
 __pycache__/
 .DS_Store

src/_static/js/custom.js

@@ -7,6 +7,13 @@ $(function(){
 	    "<div class='details'></div>"
 	)
     });
+
+    // Wrap style-test code blocks with a details div.
+    $(".style-checks, .extra-checks, .proselint-extra-checks").each(function(){
+	$(this).wrap(
+	    "<div class='details'></div>"
+	)
+    });
     // Add a toggler to each details element.
     var id = 1000000
     $(".details").each(function(index){
@@ -16,13 +23,13 @@ $(function(){
 	$(this).before(togl);
 	$(this).addClass('hidden');
 	$(this).has("img:only-child").addClass('screenshot-only');
+	$(this).has(".style-checks, .extra-checks, .proselint-extra-checks").addClass('code-only');
 	if ( $(this).is("img") ) {
 	    $(this).addClass("screenshot-only");
 	}
     });
 
     // Add class to list items that include togglers
-
     $("li").has(".details").each(function(index){
 	$(this).addClass("has-details");
     });
@@ -32,12 +39,16 @@ $(function(){
     showing_label = "[<span class='toggle-sign'>&minus;</span>] hide details";
     ss_hiding_label = "[<span class='toggle-sign'>&plus;</span>] show screenshot";
     ss_showing_label = "[<span class='toggle-sign'>&minus;</span>] hide screenshot";
+    code_hiding_label = "[<span class='toggle-sign'>&plus;</span>] show code<br>";
+    code_showing_label = "[<span class='toggle-sign'>&minus;</span>] hide code<br>";
 
 
     // Add labels and onclick function to togglers
     $(".toggler").each(function(index){
 	if ($(this).siblings().hasClass('screenshot-only')) {
 	    $(this).html(ss_hiding_label);
+	} else if ($(this).siblings().hasClass('code-only')) {
+	    $(this).html(code_hiding_label);
 	} else {
 	    $(this).html(hiding_label);
 	}
@@ -48,6 +59,8 @@ $(function(){
 		toggle_image.removeClass("hidden");
 		if ($(this).siblings().hasClass('screenshot-only')) {
 		    $(this).html(ss_showing_label);
+		} else if ($(this).siblings().hasClass('code-only')) {
+		    $(this).html(code_showing_label);
 		} else {
 		    $(this).html(showing_label);
 		}
@@ -56,6 +69,8 @@ $(function(){
 		toggle_image.addClass("hidden");
 		if ($(this).siblings().hasClass('screenshot-only')) {
 		    $(this).html(ss_hiding_label);
+		} else if ($(this).siblings().hasClass('code-only')) {
+		    $(this).html(code_hiding_label);
 		} else {
 		    $(this).html(hiding_label);
 		}

src/docs-style-guide.rst

@@ -0,0 +1,12 @@
+"""Utility module to run the extra checks on docs."""
+import importlib
+e = importlib.import_module('style-guide.extra', None)
+
+def check(text):
+    """Run extra checks."""
+    text = text.read()
+    error = []
+    errors = e.check_quotes(text) + e.check_curlyquotes(text)
+    errors += e.check_label(text)
+    errors = sorted(errors, key=lambda e: (e[2], e[3]))
+    return errors

style-guide/check-howto.py

@@ -1,13 +1,12 @@
 # -*- coding: utf-8 -*-
 
-"""uk vs. us."""
-
-from proselint.tools import memoize, preferred_forms_check
+"""Extra checks for proselint not generated by code parsing."""
 
+from proselint.tools import memoize, existence_check, preferred_forms_check
 
 @memoize
 def check_ukus(text):
-    """uk vs. us."""
+    """UK vs. US spelling usage."""
     err = "style-guide.uk-us"
     msg = "uk-vs-us-spell-check. '{}' is the preferred spelling."
 
@@ -22,6 +21,7 @@ def check_ukus(text):
         ["acclimatizes",        ["acclimatises"]],
         ["acclimatizing",       ["acclimatising"]],
         ["accouterments",       ["accoutrements"]],
+        ["acknowledgment",      ["acknowledgement"]],
         ["eon",                 ["aeon"]],
         ["eons",                ["aeons"]],
         ["aerogram",            ["aerogramme"]],
@@ -1733,3 +1733,23 @@ def check_ukus(text):
     ]
 
     return preferred_forms_check(text, preferences, err, msg)
+
+@memoize
+def check_space(text):
+    """Use single space after sentence end."""
+    err = "style-guide.check-space"
+    msg = "Missing space after period."
+    regex = "[\.\?!][A-Z][a-z]"
+
+    return existence_check(text, [regex], err, msg, ignore_case=False, 
+                         require_padding=False)
+
+@memoize
+def check_but(text):
+    """Don't start a paragraph with but."""
+    err = "style-guide.check-but"
+    msg = u"No paragraph should start with a 'But'."
+    regex = "\n[ ]*\n[ ]*But"
+
+    return existence_check(text, [regex], err, msg, ignore_case=False,
+          require_padding=False)

style-guide/check-same.py

@@ -10,9 +10,13 @@
 import importlib
 import proselint
 from blessings import Terminal
+from docutils.core import publish_doctree
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
+# path of docs directory
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
 # variables to hold error and warning count
 err_cnt = 0
 warn_cnt = 0
@@ -25,10 +29,73 @@
 os.environ["TERMINFO"]= "/etc/terminfo"
 t = Terminal()
 
+def parse_code():
+    """Parse python code-blocks."""
+    global dir_path
+    test_file = dir_path + '/style-guide/style-checks.py'
+    extra_file = dir_path + '/style-guide/extra.py'
+    
+    def is_style_code_block(node):
+        """Check for style-checks python code-blocks."""
+        return (node.tagname == 'literal_block'
+            and 'code' in node.attributes['classes']
+            and 'python' in node.attributes['classes']
+            and 'style-checks' in node.attributes['classes'])
+    
+    def is_extra_code_block(node):
+        """Check for extra-checks python code-blocks."""
+        return (node.tagname == 'literal_block'
+            and 'code' in node.attributes['classes']
+            and 'python' in node.attributes['classes']
+            and 'extra-checks' in node.attributes['classes'])
+
+    style_guide = open(dir_path + "/src/docs-style-guide.rst", "r")
+    
+    # publish doctree, report only severe errors 
+    doctree = publish_doctree(style_guide.read(),
+        settings_overrides = {'report_level': 4})
+    
+    # write source code into style-check file
+    code_blocks = doctree.traverse(condition=is_style_code_block)
+    source_code = [block.astext() for block in code_blocks]
+    
+    f = open(test_file,"w+")
+    f.write("# -*- coding: utf-8 -*-\n\n")
+    f.write('"""Style Guide testing."""\n\n')
+
+    # modules to import from proselint 
+    modules = "memoize, existence_check, preferred_forms_check"
+    f.write("from proselint.tools import %s\n" %modules)
+
+    for line in source_code:
+        if not line.endswith('\n'):
+            line = line + "\n"
+        if "@memoize" in line:
+            line = "\n" + line    
+        f.write(line)
+    
+    # write source code into extra file
+    code_blocks = doctree.traverse(condition=is_extra_code_block)
+    source_code = [block.astext() for block in code_blocks]
+    
+    f = open(extra_file,"w+")
+    f.write("# -*- coding: utf-8 -*-\n\n")
+    f.write('"""Style Guide testing."""\n\n')
+    f.write("import re\n")
+    f.write("from proselint.tools import line_and_column\n")
+
+    for line in source_code:
+        if not line.endswith('\n'):
+            line = line + "\n"
+        if "def" in line:
+            line = "\n" + line    
+        f.write(line)
+
+
 def add_checks():      
     """Add checks to proselint."""
-    file_path = os.path.realpath(__file__)
-    src = file_path[0:file_path.rfind('/')] + '/style-guide'
+    global dir_path
+    src = dir_path + '/style-guide'
     dest = os.path.dirname(proselint.__file__)
     dest_prc = dest + '/.proselintrc'
     dest = dest + '/checks/style-guide'
@@ -70,7 +137,8 @@ def add_checks():
 def remove_lines(text):
     """Remove ignored lines  and directive blocks from text."""
     directive_list = [".. image::", ".. figure::", ".. video::", ".. code::",
-                      ".. code-block::", ".. csv-table::", ".. toctree::",]
+                      ".. code-block::", ".. csv-table::", ".. toctree::",
+                      ".. py:", ".. math::",]
 
     index = 0
     length = len(text)
@@ -187,6 +255,7 @@ def exclude_checks():
     """Removes the checks which are to be excluded."""
     list_exclude = [
                      "typography.symbols", "weasel_words.very",
+                     "misc.but", "consistency.spelling",
                   ]
     dest = os.path.dirname(proselint.__file__)
     dest_prc = dest + '/.proselintrc'
@@ -206,10 +275,10 @@ def exclude_checks():
 
 def get_paths(paths):
     """Return a list of files to run the checks on."""
-    file_path = os.path.realpath(__file__)
+    global dir_path
 
     # find path for all .rst files
-    search_path = file_path[0:file_path.rfind('/')] + "/src"
+    search_path = dir_path + "/src"
 
     # Make a list of paths to check for
     path_list = []
@@ -235,13 +304,13 @@ def get_paths(paths):
 
 def get_changed_files():
     """Return currently modified rst files."""
-    file_path = os.path.realpath(__file__)
-    repo_path = file_path[0:file_path.rfind('/')]
+    global dir_path
+    repo_path = dir_path
     repo = git.Repo(repo_path)
-    changedFiles = [item.a_path for item in repo.index.diff(None)]
-    changedFiles += repo.untracked_files
-    changedFiles = tuple(changedFiles)
-    return changedFiles
+    changed_files = [item.a_path for item in repo.index.diff(None)]
+    changed_files += repo.untracked_files
+    changed_files = tuple(changed_files)
+    return changed_files
 
 
 def run_checks(paths, disp, fix):
@@ -266,7 +335,7 @@ def run_checks(paths, disp, fix):
         text = remove_lines(text)
 
         # Import extra check module from style-guide 
-        extra = importlib.import_module('style-guide.extra', None)
+        extra = importlib.import_module('style-guide.extrarun', None)
 
         # run checks for quotes, curly quotes, section labels
         errors = extra.check(temp_file(text))
@@ -335,7 +404,7 @@ def disp_checks(errors, filename, shortname):
         # e[0]=check
         # Set warning or error severity
         # Don't set errors for style-guide as they might be examples
-        if e[0] in list_errors and "style-guide.rst" not in shortname:
+        if e[0] in list_errors and shortname != "docs-style-guide.rst":
             severity = "error"
         else:
             severity = "warning" 
@@ -365,8 +434,20 @@ def disp_cnt():
     global warn_cnt
     global t
 
-    print(t.yellow("Found %d warnings" %warn_cnt))
-    print(t.red("Found %d errors") %err_cnt)
+    warn = "%d warnings." %warn_cnt
+    err = "%d errors." %err_cnt
+
+    if warn_cnt == 0:
+        warn = "no warnings!"
+    elif warn_cnt == 1:
+        warn = "1 warning."
+    if err_cnt == 0:
+        err = "no errors!"
+    elif err_cnt == 1:
+        err = "1 error."
+
+    print(t.yellow("Testing found %s" %warn))
+    print(t.red("Testing found %s" %err))
     if err_cnt:
         raise Exception("Style-guide testing failed! Fix the errors")
 
@@ -426,17 +507,30 @@ def gen_list(paths = None):
 
     return err_list
 
+def remove_file():
+    """Remove generated check files."""
+    global dir_path
+    test_file = dir_path + '/style-guide/style-checks.py'
+    extra_file = dir_path + '/style-guide/extra.py'
+    os.remove(test_file)
+    os.remove(extra_file)
+
 
 @click.command(context_settings = CONTEXT_SETTINGS)
 @click.option('--diff', '-d', is_flag = True, 
                help = "Run check on the modified files")
 @click.option('--fix', '-f', is_flag = True, 
-               help = "Removes the fixable errors")
+               help = "Remove the fixable errors")
 @click.option('--out_path','-o', type = click.Path())
+@click.option('--store','-s',is_flag = True,
+               help = "Store the generated test scripts")
 @click.argument('in_path', nargs = -1, type = click.Path())
 def style_test(in_path = None, out_path = None, diff = None, 
-                fix = None, output = None):
-    """A CLI for style guide testing"""
+                fix = None, output = None, store = None):
+    """A CLI for style guide testing."""
+    # generate source code for checks
+    parse_code()
+
     # add custom style-guide checks to proselint
     add_checks()
 
@@ -456,7 +550,10 @@ def style_test(in_path = None, out_path = None, diff = None,
     # generate output
     if out_path:
         gen_out(out_path)
-    
+
+    # remove generated test scripts
+    if not store:
+        remove_file()
 
 if __name__ == '__main__':
     style_test()

style-guide/check-semicolon.py

@@ -1,11 +1,2 @@
-        "style-guide.spelling-odk"      : true,
-        "style-guide.uk-us"             : true,
-        "style-guide.latin-abbr"        : true,
-        "style-guide.unneed-adverb"     : true,
-        "style-guide.filler-phrase"     : true,
-        "style-guide.personal-pronoun"  : true,
-        "style-guide.serial-comma"      : true,
-        "style-guide.check-same"        : true,
-        "style-guide.check-semicolon"   : true,
-        "style-guide.check-howto"       : true,
-        "style-guide.check-space"       : true
+        "style-guide.style-checks"      : true,
+        "style-guide.proselint-extra"   : true