Generate index page from python code containing docs chooser. (#18)

.github/workflows/python-app.yml

@@ -46,3 +46,9 @@ jobs:
         run: poetry install
       - name: Run app
         run: poetry run python src/main.py ${{ secrets.GITHUB_TOKEN }}
+      - name: Deploy to GH pages
+        if: github.ref == 'refs/heads/main'
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: gen

.pylintrc

@@ -18,7 +18,7 @@ ignore-patterns=
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-#init-hook=
+#init-hook='import sys; sys.path.append("/src/")'
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.

pyproject.toml

@@ -11,6 +11,7 @@ black = "^20.8b1"
 mypy = "^0.800"
 PyExecJS = "^1.5.1"
 pylint = "^2.6.2"
+airium = {extras = ["parse"], version = "^0.2.2"}
 
 [tool.poetry.dev-dependencies]
 

src/constants.py

@@ -15,9 +15,13 @@
 DIRECTORY: str = "distribution/std-lib"
 
 #: The commit from which scala parser will be obtained.
-PARSER_COMMIT: str = "5c735b0ae7fc14bf72f5c0bf43b17c3fdf5d86b4"
+PARSER_COMMIT: str = "43c6fde4ef5873c645aa1ff196d7b36864831468"
 #: The URL used to download scala parser package.
 PARSER_URL: str = "https://packages.luna-lang.org/parser-js/nightly/"
+#: The method in scala parser used to generate documentation from AST.
+PARSE_AST_METHOD: str = "$e_doc_parser_generate_html_source"
+#: The method in scala parser used to generate documentation from doc code.
+PARSE_PURE_METHOD: str = "$e_doc_parser_generate_html_from_doc"
 #: The URL leading to IDE repository.
 IDE_REPO_URL: str = "https://raw.githubusercontent.com/enso-org/ide/"
 #: The branch in the above repository from which stylesheet will be downloaded.
@@ -40,3 +44,5 @@
 PARSER_FILE: str = "scala-parser.js"
 #: The stylesheet file name.
 STYLE_FILE: str = "style.css"
+#: The generated HTML index page.
+INDEX_FILE: str = "index.html"

src/create_static.py

@@ -0,0 +1,148 @@
+"""
+Methods for generating static HTML content.
+"""
+from typing import List
+import logging
+from airium import Airium
+
+
+def create_index_page(out_dir: str, out_name: str, gen_files: List[str]) -> None:
+    """
+    Generates index page.
+    """
+    template = Airium()
+    logging.info("Generating: %s", out_name)
+    with template.html():
+        with template.head():
+            template.title(_t="Enso Reference")
+            template.link(href="style.css", rel="stylesheet")
+            template.link(href="favicon.ico", rel="icon")
+            template.style(_t="ul { padding-inline-start: 15px; }")
+            template.style(
+                _t="""ul, .section ul {
+                              list-style: none;
+                              padding: 0;
+                              margin: 0;
+                              word-wrap: initial;
+                            }
+                            
+                            ul li {
+                              padding: 5px 10px;
+                            }
+                            
+                            .section ul { display: none; }
+                            .section input:checked ~ ul { display: block; }
+                            .section input[type=checkbox] { display: none; }
+                            .section { 
+                              position: relative; 
+                              padding-left: 20px !important;
+                            }
+                            
+                            .section label:after {
+                              content: "+";
+                              position: absolute;
+                              top: 0; left: 0;
+                              padding: 0;
+                              text-align: center;
+                              font-size: 17px;
+                              color: cornflowerblue;
+                              transition: all 0.3s;
+                            }
+                            
+                            .section input:checked ~ label:after { 
+                              color: cadetblue;
+                              transform: rotate(45deg);
+                            }
+                            
+                            @media only screen and (max-width: 1100px) {
+                                #tree {
+                                    width: 30% !important;
+                                }
+                            }
+                            """
+            )
+            template.script(
+                _t="""function set_frame_content(file) {
+                          document.getElementById("frame").src = file
+                      }"""
+            )
+        with template.body(style="background-color: #333"):
+            template.h2(
+                style="""text-align: center;
+                         padding: 15px; 
+                         margin: 0; 
+                         color: #fafafa""",
+                _t="Enso Reference",
+            )
+            with template.div(
+                style="background-color: #fafafa; display: flex; height: 100%"
+            ):
+                with template.div(
+                    id="tree",
+                    style="""background-color: #efefef;
+                             border-radius: 14px; 
+                             width: 20%; 
+                             margin: 15px; 
+                             padding-left: 20px;
+                             overflow: scroll;
+                             height: 90%;""",
+                ):
+                    grouped_file_names = group_by_prefix(gen_files)
+                    create_html_tree(template, "", grouped_file_names, gen_files)
+                template.iframe(
+                    frameborder="0",
+                    height="100%",
+                    id="frame",
+                    src="Base-Main.html",
+                    width="100%",
+                    target="_blank",
+                )
+
+    html_file = open(out_dir + "/" + out_name, "w")
+    html_file.write(str(template))
+    html_file.close()
+
+
+def create_html_tree(
+    template: Airium, curr_beg: str, ele, all_existing_files: List[str]
+) -> None:
+    """
+    Method used to create all of HTML tree chooser's branches and leaves.
+    """
+    if len(ele) > 0:
+        with template.ul():
+            for key, value in ele.items():
+                file_name = curr_beg + "-" + key
+                action = ""
+                if file_name in all_existing_files:
+                    action = "set_frame_content('" + file_name + ".html')"
+                if len(value) > 0:
+                    klass = "section"
+                    with template.li(klass=klass):
+                        template.input(type="checkbox", id=file_name)
+                        template.label(for_=file_name, _t=key, onclick=action)
+                        beg = curr_beg
+                        if len(curr_beg) == 0:
+                            beg = key
+                        else:
+                            beg = beg + "-" + key
+                        create_html_tree(template, beg, value, all_existing_files)
+                else:
+                    template.li(onclick=action, _t=key)
+
+
+def group_by_prefix(strings: List[str]) -> dict:
+    """
+    Groups strings by common prefixes
+    """
+    strings_by_prefix: dict = {}
+    for string in strings:
+        if len(string.split("-")) <= 1:
+            strings_by_prefix.setdefault(string, [])
+            continue
+        prefix, suffix = map(str.strip, string.split("-", 1))
+        group = strings_by_prefix.setdefault(prefix, [])
+        group.append(suffix)
+    for key, string_group in strings_by_prefix.items():
+        strings_by_prefix[key] = group_by_prefix(string_group)
+    return strings_by_prefix

src/download_helpers.py

@@ -2,6 +2,7 @@
 Helper methods for downloading files and folders.
 """
 import os
+import logging
 import base64
 import requests
 from github import Github
@@ -35,7 +36,7 @@ def __download_directory(repository: Repository, sha: str, server_path: str) ->
     contents = repository.get_dir_contents(server_path, ref=sha)
 
     for content in contents:
-        print("Downloading: %s" % content.path)
+        logging.info("Downloading: %s", content.path)
         if content.type == "dir" and not content.path.endswith("THIRD-PARTY"):
             os.makedirs(content.path)
             __download_directory(repository, sha, content.path)
@@ -49,7 +50,7 @@ def __download_directory(repository: Repository, sha: str, server_path: str) ->
                     file_out.write(file_data)
                     file_out.close()
             except (GithubException, IOError) as exc:
-                print("Error processing %s: %s", content.path, exc)
+                logging.info("Error processing %s: %s", content.path, exc)
 
 
 def download_from_git(
@@ -70,5 +71,5 @@ def download_from_url(url: str, to_file: str) -> None:
     Downloads file from given URL.
     """
     request = requests.get(url, allow_redirects=True)
-    print("Downloading: %s" % url)
+    logging.info("Downloading: %s", url)
     open(to_file, "wb").write(request.content)

src/main.py

@@ -2,24 +2,28 @@
 Enso standard library documentation generator.
 """
 import argparse
+import logging
 import constants
 from downloaders import download_stdlib, download_parser, download_stylesheet
 from parse import init_parser, init_gen_dir, gen_all_files
+from create_static import create_index_page
 
 
 def main(arguments: argparse.Namespace) -> None:
     """
     Program entry point.
     """
+    logging.basicConfig(level=arguments.log_level)
     download_stdlib(
         arguments.token, arguments.org, arguments.repo, arguments.br, arguments.dir
     )
     download_parser(arguments.parser_url, arguments.commit, arguments.parser)
     download_stylesheet(arguments.ide_br, arguments.style)
     parser = init_parser(arguments.parser)
     init_gen_dir(arguments.out, arguments.style)
-    gen_all_files(parser, arguments.std, arguments.out, arguments.style)
-    print("All done.")
+    gen_files = gen_all_files(parser, arguments.std, arguments.out, arguments.style)
+    create_index_page(arguments.out, arguments.index, gen_files)
+    logging.info("All done.")
 
 
 if __name__ == "__main__":
@@ -58,5 +62,9 @@ def main(arguments: argparse.Namespace) -> None:
     arg_parser.add_argument(
         "--parser_url", default=constants.PARSER_URL, help="URL to parser."
     )
+    arg_parser.add_argument(
+        "--index", default=constants.INDEX_FILE, help="Index page name."
+    )
+    arg_parser.add_argument("--log_level", default=logging.INFO, help="Logging level.")
     args = arg_parser.parse_args()
     main(args)