From 1670c65723bf77dd3677625a249bf725a6d27e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Miko=C5=82ajek?= Date: Wed, 3 Mar 2021 17:15:34 +0100 Subject: [PATCH] Generate index page from python code containing docs chooser. (#18) --- .github/workflows/python-app.yml | 6 ++ .pylintrc | 2 +- favicon.ico | Bin 0 -> 2067 bytes poetry.lock | 54 ++++++++++- pyproject.toml | 1 + src/constants.py | 8 +- src/create_static.py | 148 +++++++++++++++++++++++++++++++ src/download_helpers.py | 7 +- src/index.html | 20 ----- src/main.py | 12 ++- src/parse.py | 31 ++++--- 11 files changed, 251 insertions(+), 38 deletions(-) create mode 100644 favicon.ico create mode 100644 src/create_static.py delete mode 100644 src/index.html diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 785cf02a..9880b732 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -46,3 +46,9 @@ jobs: run: poetry install - name: Run app run: poetry run python src/main.py ${{ secrets.GITHUB_TOKEN }} + - name: Deploy to GH pages + if: github.ref == 'refs/heads/main' + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: gen diff --git a/.pylintrc b/.pylintrc index 75e991c0..629f1bc5 100644 --- a/.pylintrc +++ b/.pylintrc @@ -18,7 +18,7 @@ ignore-patterns= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). -#init-hook= +#init-hook='import sys; sys.path.append("/src/")' # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the # number of processors available to use. diff --git a/favicon.ico b/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..8b5cd5dc19de0283e2bf959f6a88ba0b7697e565 GIT binary patch literal 2067 zcmaJ?c~nzZ9uA_>U{Pc%RF-EVizr!G0*S0434}vJOo9Yk#7FWZAtWy*5AuR-Q4Auo z6cNO+0s{iJFbwYCfT&R%6|~k$JGE3rrC3KR4uhaZofj*1{ut(+bKkx9cYfdZ+rD#d ziBJ%4Z}*WM4u`YnC9p-t0X#LB1&JU~ zZrpSUiooG){go0aBIPGBWw4qIT7AfE^kT_o;k>#so3^_463Wz|N7=ao>Kmb+eX!J}plK7UFiJh%x3K4kg zf~X>izb7T-3xPOT2LYjEdZ3IJ91esr$h1&Kcu>eHAc#t%Q>Y9IEhLaiV^SGRDh+t| z5wU1G`9`LQ&3zXOyG0Te2%=?DCQE8i&IP zkE7GD#pN=%Ggvk(%T+@fWCkn$0~UK2ZY9n4K8*d|& zngp;C-`Jc_^7(gk#Zw|r!hF2_WU$SXkQn=hC-kRN{HQ;lcJx2^EAt0?p9{8Y`{qTd zn1k`3IOs)bGP8wxal;zq_J_N#DFb4*5ml5=;hTgd_a;a$3i&0~XZrCAm%J`^rY`Oa6IVIb z$B?>8{e)G!2V4}Rn|ws((jos_N0Nj59xeW%Yr8DMZAlV6;V$8O?McUByDK*0CgtG^ z89vv|B_WM?@pP}QCei2dg6A;@ioH%Q^{Q)-7~>DI`1U8ppO1Q!>X*uu`p?g?GOs^) zy8NosGl#X{Q4Oir?^9rS=e7E8D5MqN4!qKSsr|l+kg9Gw49Spy;X92Ddrp z4){)tO3SnsXTg_Z!iQkinPrD6CszCSjGwt&{h2A=)OE37+x_qn_ajF-(huFokGMBv zsND8Y;}*YqBfdAa&hm3C8aFLK`6y>PX}W~)#4qP8_qRO>FLxC4Cc&+OdB?k@xs~Gs z)2HTK%}(}SRMfI3sk2-#T&^0c^{jN*wt4I?clKv_cE8TwaUxYFxNBhf{8pUS_z5}8 zlSBw=X?9I*p*Tf7`_V4HGd;7Ytg5m%aJpolcFs8dCql+cMR|wWT#*2tbgFz|Y&&%` zuQ~Zxe$2GoB3Q_H#i$~h8l0kN*{Fp$~LjD>^4`lR-bDtm~v_z20VaIOTOO0 zcYlm@ItY%A-P`j~%y+JJ02cd-fQqM@}!qM+RSLm$Y-gC+nSyGudaBWm!OJnxk;JpO&T3GolZ8PWt1DM z&!@(*+;*A6>!C&J6yC>GI?o8CC}q51BIN1(RY8~VLm_G9C09&RpVFak(*6P5`eH7$ zj#Ek*V(`?$*Fl#9XzhLRvNx(N#Hy*nhnCF;_dPVL cZ+P3_arA%AYwoHhpw?%a#}Tmq%Gyx)A3pIi+W-In literal 0 HcmV?d00001 diff --git a/poetry.lock b/poetry.lock index 9eb78d7a..66e3dfb1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,19 @@ +[[package]] +name = "airium" +version = "0.2.2" +description = "Easy and quick html builder with natural syntax correspondence (python->html). No templates needed. Serves pure pythonic library with no dependencies." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +beautifulsoup4 = {version = ">=4.9,<5.0", optional = true, markers = "extra == \"parse\""} +requests = {version = ">=2.12,<3", optional = true, markers = "extra == \"parse\""} + +[package.extras] +dev = ["pdbpp (>=0.10.0,<0.11.0)", "pytest (>=5.4.0,<5.5.0)", "pytest-cov (>=2.10.0,<2.11.0)", "pytest-mock (>=3.2.0,<3.3.0)"] +parse = ["requests (>=2.12,<3)", "beautifulsoup4 (>=4.9,<5.0)"] + [[package]] name = "appdirs" version = "1.4.4" @@ -19,6 +35,21 @@ lazy-object-proxy = ">=1.4.0,<1.5.0" six = ">=1.12,<2.0" wrapt = ">=1.11,<2.0" +[[package]] +name = "beautifulsoup4" +version = "4.9.3" +description = "Screen-scraping library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""} + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "black" version = "20.8b1" @@ -245,6 +276,14 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "soupsieve" +version = "2.2" +description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "toml" version = "0.10.2" @@ -293,9 +332,13 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "b8d08582256720baff79a7213fb847da596ef80a0d287db3e01bb4d42235b72c" +content-hash = "510203c39754ff7feb23ca4aefdab4b2709e1553c48444568a01342ad2119f32" [metadata.files] +airium = [ + {file = "airium-0.2.2-py3-none-any.whl", hash = "sha256:5157e80a2366fd13bae657fbfe37dca90c5c279961a2772a7eccd4eaa2dea339"}, + {file = "airium-0.2.2.tar.gz", hash = "sha256:19015ad991ed226c9998ea3e516a1a77319a8859dab904b73187584c8d9353b6"}, +] appdirs = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, @@ -304,6 +347,11 @@ astroid = [ {file = "astroid-2.4.2-py3-none-any.whl", hash = "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386"}, {file = "astroid-2.4.2.tar.gz", hash = "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703"}, ] +beautifulsoup4 = [ + {file = "beautifulsoup4-4.9.3-py2-none-any.whl", hash = "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35"}, + {file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"}, + {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, +] black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] @@ -460,6 +508,10 @@ six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] +soupsieve = [ + {file = "soupsieve-2.2-py3-none-any.whl", hash = "sha256:d3a5ea5b350423f47d07639f74475afedad48cf41c0ad7a82ca13a3928af34f6"}, + {file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, diff --git a/pyproject.toml b/pyproject.toml index 9880f7f9..5732e7ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ black = "^20.8b1" mypy = "^0.800" PyExecJS = "^1.5.1" pylint = "^2.6.2" +airium = {extras = ["parse"], version = "^0.2.2"} [tool.poetry.dev-dependencies] diff --git a/src/constants.py b/src/constants.py index cbd306ed..a3c0b9a0 100644 --- a/src/constants.py +++ b/src/constants.py @@ -15,9 +15,13 @@ DIRECTORY: str = "distribution/std-lib" #: The commit from which scala parser will be obtained. -PARSER_COMMIT: str = "5c735b0ae7fc14bf72f5c0bf43b17c3fdf5d86b4" +PARSER_COMMIT: str = "43c6fde4ef5873c645aa1ff196d7b36864831468" #: The URL used to download scala parser package. PARSER_URL: str = "https://packages.luna-lang.org/parser-js/nightly/" +#: The method in scala parser used to generate documentation from AST. +PARSE_AST_METHOD: str = "$e_doc_parser_generate_html_source" +#: The method in scala parser used to generate documentation from doc code. +PARSE_PURE_METHOD: str = "$e_doc_parser_generate_html_from_doc" #: The URL leading to IDE repository. IDE_REPO_URL: str = "https://raw.githubusercontent.com/enso-org/ide/" #: The branch in the above repository from which stylesheet will be downloaded. @@ -40,3 +44,5 @@ PARSER_FILE: str = "scala-parser.js" #: The stylesheet file name. STYLE_FILE: str = "style.css" +#: The generated HTML index page. +INDEX_FILE: str = "index.html" diff --git a/src/create_static.py b/src/create_static.py new file mode 100644 index 00000000..3d8db258 --- /dev/null +++ b/src/create_static.py @@ -0,0 +1,148 @@ +""" +Methods for generating static HTML content. +""" +from typing import List +import logging +from airium import Airium + + +def create_index_page(out_dir: str, out_name: str, gen_files: List[str]) -> None: + """ + Generates index page. + """ + template = Airium() + logging.info("Generating: %s", out_name) + with template.html(): + with template.head(): + template.title(_t="Enso Reference") + template.link(href="style.css", rel="stylesheet") + template.link(href="favicon.ico", rel="icon") + template.style(_t="ul { padding-inline-start: 15px; }") + template.style( + _t="""ul, .section ul { + list-style: none; + padding: 0; + margin: 0; + word-wrap: initial; + } + + ul li { + padding: 5px 10px; + } + + .section ul { display: none; } + .section input:checked ~ ul { display: block; } + .section input[type=checkbox] { display: none; } + .section { + position: relative; + padding-left: 20px !important; + } + + .section label:after { + content: "+"; + position: absolute; + top: 0; left: 0; + padding: 0; + text-align: center; + font-size: 17px; + color: cornflowerblue; + transition: all 0.3s; + } + + .section input:checked ~ label:after { + color: cadetblue; + transform: rotate(45deg); + } + + @media only screen and (max-width: 1100px) { + #tree { + width: 30% !important; + } + } + """ + ) + template.script( + _t="""function set_frame_content(file) { + document.getElementById("frame").src = file + }""" + ) + with template.body(style="background-color: #333"): + template.h2( + style="""text-align: center; + padding: 15px; + margin: 0; + color: #fafafa""", + _t="Enso Reference", + ) + with template.div( + style="background-color: #fafafa; display: flex; height: 100%" + ): + with template.div( + id="tree", + style="""background-color: #efefef; + border-radius: 14px; + width: 20%; + margin: 15px; + padding-left: 20px; + overflow: scroll; + height: 90%;""", + ): + grouped_file_names = group_by_prefix(gen_files) + create_html_tree(template, "", grouped_file_names, gen_files) + template.iframe( + frameborder="0", + height="100%", + id="frame", + src="Base-Main.html", + width="100%", + target="_blank", + ) + + html_file = open(out_dir + "/" + out_name, "w") + html_file.write(str(template)) + html_file.close() + + +def create_html_tree( + template: Airium, curr_beg: str, ele, all_existing_files: List[str] +) -> None: + """ + Method used to create all of HTML tree chooser's branches and leaves. + """ + if len(ele) > 0: + with template.ul(): + for key, value in ele.items(): + file_name = curr_beg + "-" + key + action = "" + if file_name in all_existing_files: + action = "set_frame_content('" + file_name + ".html')" + if len(value) > 0: + klass = "section" + with template.li(klass=klass): + template.input(type="checkbox", id=file_name) + template.label(for_=file_name, _t=key, onclick=action) + beg = curr_beg + if len(curr_beg) == 0: + beg = key + else: + beg = beg + "-" + key + create_html_tree(template, beg, value, all_existing_files) + else: + template.li(onclick=action, _t=key) + + +def group_by_prefix(strings: List[str]) -> dict: + """ + Groups strings by common prefixes + """ + strings_by_prefix: dict = {} + for string in strings: + if len(string.split("-")) <= 1: + strings_by_prefix.setdefault(string, []) + continue + prefix, suffix = map(str.strip, string.split("-", 1)) + group = strings_by_prefix.setdefault(prefix, []) + group.append(suffix) + for key, string_group in strings_by_prefix.items(): + strings_by_prefix[key] = group_by_prefix(string_group) + return strings_by_prefix diff --git a/src/download_helpers.py b/src/download_helpers.py index 301acfec..2ec6b21b 100644 --- a/src/download_helpers.py +++ b/src/download_helpers.py @@ -2,6 +2,7 @@ Helper methods for downloading files and folders. """ import os +import logging import base64 import requests from github import Github @@ -35,7 +36,7 @@ def __download_directory(repository: Repository, sha: str, server_path: str) -> contents = repository.get_dir_contents(server_path, ref=sha) for content in contents: - print("Downloading: %s" % content.path) + logging.info("Downloading: %s", content.path) if content.type == "dir" and not content.path.endswith("THIRD-PARTY"): os.makedirs(content.path) __download_directory(repository, sha, content.path) @@ -49,7 +50,7 @@ def __download_directory(repository: Repository, sha: str, server_path: str) -> file_out.write(file_data) file_out.close() except (GithubException, IOError) as exc: - print("Error processing %s: %s", content.path, exc) + logging.info("Error processing %s: %s", content.path, exc) def download_from_git( @@ -70,5 +71,5 @@ def download_from_url(url: str, to_file: str) -> None: Downloads file from given URL. """ request = requests.get(url, allow_redirects=True) - print("Downloading: %s" % url) + logging.info("Downloading: %s", url) open(to_file, "wb").write(request.content) diff --git a/src/index.html b/src/index.html deleted file mode 100644 index 164e05b7..00000000 --- a/src/index.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - - -

- Enso Docs -

-
- -
- - diff --git a/src/main.py b/src/main.py index ed7ef4df..6fb0d4a8 100644 --- a/src/main.py +++ b/src/main.py @@ -2,15 +2,18 @@ Enso standard library documentation generator. """ import argparse +import logging import constants from downloaders import download_stdlib, download_parser, download_stylesheet from parse import init_parser, init_gen_dir, gen_all_files +from create_static import create_index_page def main(arguments: argparse.Namespace) -> None: """ Program entry point. """ + logging.basicConfig(level=arguments.log_level) download_stdlib( arguments.token, arguments.org, arguments.repo, arguments.br, arguments.dir ) @@ -18,8 +21,9 @@ def main(arguments: argparse.Namespace) -> None: download_stylesheet(arguments.ide_br, arguments.style) parser = init_parser(arguments.parser) init_gen_dir(arguments.out, arguments.style) - gen_all_files(parser, arguments.std, arguments.out, arguments.style) - print("All done.") + gen_files = gen_all_files(parser, arguments.std, arguments.out, arguments.style) + create_index_page(arguments.out, arguments.index, gen_files) + logging.info("All done.") if __name__ == "__main__": @@ -58,5 +62,9 @@ def main(arguments: argparse.Namespace) -> None: arg_parser.add_argument( "--parser_url", default=constants.PARSER_URL, help="URL to parser." ) + arg_parser.add_argument( + "--index", default=constants.INDEX_FILE, help="Index page name." + ) + arg_parser.add_argument("--log_level", default=logging.INFO, help="Logging level.") args = arg_parser.parse_args() main(args) diff --git a/src/parse.py b/src/parse.py index e8917fac..03fa4258 100644 --- a/src/parse.py +++ b/src/parse.py @@ -2,6 +2,8 @@ Creates `gen` directory with all necessary files. """ import glob +import logging +from typing import List import execjs import constants from copy_file import copy_file @@ -10,22 +12,28 @@ def gen_all_files( parser: execjs.ExternalRuntime, std_dir: str, out_dir: str, style_file: str -) -> None: +) -> List[str]: """ Recursively generates all doc files and puts them into the `gen` directory. """ + all_file_names: List[str] = [] + for filename in glob.iglob("**/*" + constants.FILE_EXT, recursive=True): out_file_name = ( filename.replace(std_dir + "/", "") .replace("/", "-") + .replace("-src-", "-") .replace(constants.FILE_EXT, ".html") ) - print("Generating: " + out_file_name) + logging.info("Generating: %s", out_file_name) try: __gen_file(parser, filename, out_file_name, out_dir, style_file) + all_file_names.append(out_file_name.replace(".html", "")) except execjs.Error as err: - print("Could not generate: " + out_file_name) - print("Got an exception: " + str(err)) + logging.info("Could not generate: %s", out_file_name) + logging.info("Got an exception: %s", str(err)) + + return all_file_names def __gen_file( @@ -40,12 +48,16 @@ def __gen_file( it as `out_name`. """ enso_file = open(path, "r") - js_method = "$e_doc_parser_generate_html_source" stylesheet_link = '' - parsed = parser.call(js_method, enso_file.read()) + parsed = parser.call(constants.PARSE_AST_METHOD, enso_file.read()) enso_file.close() html_file = open(out_dir + "/" + out_name, "w") - html_file.write(stylesheet_link + parsed) + if len(parsed) == 0: + parsed = parser.call( + constants.PARSE_PURE_METHOD, + "\n\n*Enso Reference Viewer.*\n\nNo documentation available for chosen source file.", + ) + html_file.write(stylesheet_link + parsed.replace("display: flex", "display: none")) html_file.close() @@ -55,10 +67,9 @@ def init_gen_dir(name: str, style_file: str) -> None: """ safe_create_directory(name) stylesheet_file: str = "/" + style_file - index_html_file: str = "/index.html" - source_directory: str = "src" + favicon_file: str = "favicon.ico" copy_file(constants.IN_DIR + stylesheet_file, name + stylesheet_file) - copy_file(source_directory + index_html_file, name + index_html_file) + copy_file(favicon_file, name + "/" + favicon_file) def init_parser(parser_file: str) -> execjs.ExternalRuntime: