Add a script to download std-lib and parser from engine (#10)

enso-org · Feb 18, 2021 · 632fbf2 · 632fbf2
1 parent 31beb2c
commit 632fbf2
Show file tree

Hide file tree

Showing 8 changed files with 214 additions and 6 deletions.
diff --git a/.gitignore b/.gitignore
@@ -39,5 +39,7 @@ Thumbs.db
 # Untracked files #
 ###################
 .vscode/
-node_modules/
-dist/
+distribution/
+.idea
+venv
+__pycache__
diff --git a/README.md b/README.md
@@ -1,4 +1,3 @@
-
 <p align="center">
   <br/>
   <a href="http://enso.org">
@@ -22,7 +21,15 @@
   <br/>
 </p>
 
-# Enso Standard Library Documentation Viewer
+# Enso Standard Library Documentation
+
+This repository contains scripts that generates HTML docs from Enso Std-Lib.
+Documentation is drawn from [here](https://github.com/enso-org/enso/tree/main/distribution/std-lib).
+
+### Running
 
-This site hosts the documentation for Enso's standard library.
-Documentation is drawn from [here](https://github.com/enso-org/enso/tree/main/distribution/std-lib).
+- Make sure You have Python 3 installed on your system.
+- Using pip get PyExecJS and PyGithub.
+- Go to GitHub, and create new access token. You'll need to provide it.
+- In terminal, go to this project and run script:
+  `python main.py --token=YOUR_GH_TOKEN`
diff --git a/__init__.py b/__init__.py
diff --git a/download_helpers.py b/download_helpers.py
@@ -0,0 +1,62 @@
+import os
+import base64
+import shutil
+import requests
+from github import Github
+from github import GithubException
+
+
+def get_sha_for_tag(repository, tag):
+    branches = repository.get_branches()
+    matched_branches = [match for match in branches if match.name == tag]
+    if matched_branches:
+        return matched_branches[0].commit.sha
+
+    tags = repository.get_tags()
+    matched_tags = [match for match in tags if match.name == tag]
+    if not matched_tags:
+        raise ValueError('No Tag or Branch exists with that name')
+    return matched_tags[0].commit.sha
+
+
+def download_directory(repository, sha, server_path):
+    if os.path.exists(server_path):
+        shutil.rmtree(server_path)
+
+    os.makedirs(server_path)
+    contents = repository.get_dir_contents(server_path, ref=sha)
+
+    for content in contents:
+        print("Downloading: %s" % content.path)
+        if content.type == 'dir' and not content.path.endswith("THIRD-PARTY"):
+            os.makedirs(content.path)
+            download_directory(repository, sha, content.path)
+        else:
+            try:
+                path = content.path
+                if path.endswith(".enso"):
+                    file_content = repository.get_contents(path, ref=sha)
+                    file_data = base64.b64decode(file_content.content)
+                    file_out = open(content.path, "wb+")
+                    file_out.write(file_data)
+                    file_out.close()
+            except (GithubException, IOError) as exc:
+                print('Error processing %s: %s', content.path, exc)
+
+
+def download_from_git(token: str,
+                      org: str,
+                      repo: str,
+                      branch: str,
+                      folder: str):
+    github = Github(token)
+    organization = github.get_organization(org)
+    repository = organization.get_repo(repo)
+    sha = get_sha_for_tag(repository, branch)
+    download_directory(repository, sha, folder)
+
+
+def download_from_url(url, to):
+    r = requests.get(url, allow_redirects=True)
+    print("Downloading: %s" % url)
+    open(to, 'wb').write(r.content)
diff --git a/index.html b/index.html
@@ -0,0 +1,12 @@
+<html>
+    <head>
+        <link href="../style.css" rel="stylesheet">
+    </head>
+    <body style="background-color: #333;">
+        <h2 style="text-align: center; padding: 15px; margin:0; color: #fafafa;"> Enso Docs </h2>
+        <div style="background-color: #fafafa;">
+            <iframe src="Meta.html" id="frame" width="100%"" height="100%" frameBorder="0">
+            </iframe>
+        </div>
+    </body>
+</html>
diff --git a/main.py b/main.py
@@ -0,0 +1,102 @@
+import sys
+import getopt
+import execjs
+from download_helpers import *
+from replace_all_occurences_in_file import *
+from safe_create_dir import *
+
+ORGANIZATION = "enso-org"
+REPO = "enso"
+BRANCH = "main"
+DIRECTORY = "distribution/std-lib"
+PARSER_COMMIT = "5e309bddcbec33cfbd150fcb8a16b45192cf5189"
+
+
+def main(argv):
+    download_stdlib(argv)
+    download_parser()
+    download_stylesheet()
+    parser = init_parser()
+    test_parse(parser)
+
+
+def test_parse(parser):
+    """
+        Test if generating docs work.
+        To be removed when created method to recursively generate all docfiles.
+    """
+    example = open('distribution/std-lib/Base/src/Math.enso', 'r').read()
+    parsed = parser.call("$e_doc_parser_generate_html_source", example)
+    out_dir = 'distribution/gen'
+    safe_create_directory(out_dir)
+    html_file = open(out_dir + '/Math.html', 'w')
+    html_file.write('<link rel="stylesheet" href="../style.css"/>' + parsed)
+    html_file.close()
+
+    example = open('distribution/std-lib/Base/src/Meta.enso', 'r').read()
+    parsed = parser.call("$e_doc_parser_generate_html_source", example)
+    html_file = open(out_dir + '/Meta.html', 'w')
+    html_file.write('<link rel="stylesheet" href="../style.css"/>' + parsed)
+    html_file.close()
+
+
+def init_parser():
+    """
+        Compiles JS parser to call from Python.
+    """
+    parser = open('distribution/parser.js', 'r').read()
+    parser = execjs.compile(parser)
+    return parser
+
+
+def download_stylesheet():
+    """
+        Downloads stylesheet for docs from IDE repository.
+    """
+    repo_url = "https://raw.githubusercontent.com/enso-org/ide/"
+    file_path = "develop/src/rust/ide/view/src/documentation/style.css"
+    url = repo_url + file_path
+    download_to = "distribution/temp-style.css"
+    download_from_url(url, download_to)
+    replace_all_occurrences_in_file(download_to,
+                                    'distribution/style.css',
+                                    '.docVis',
+                                    'body')
+
+
+def download_parser():
+    """
+        Downloads scala parser from Engine repository.
+    """
+    url = "https://packages.luna-lang.org/parser-js/nightly/"
+    url = url + PARSER_COMMIT + "/scala-parser.js"
+    download_to = "distribution/scala-parser.js"
+    download_from_url(url, download_to)
+    replace_all_occurrences_in_file(download_to,
+                                    'distribution/parser.js',
+                                    'export ',
+                                    '// export')
+
+
+def download_stdlib(argv):
+    """
+        Downloads Std-Lib from Engine repository.
+    """
+    token = ""
+    try:
+        opts, args = getopt.getopt(argv, "t:", ["token="])
+    except getopt.GetoptError as err:
+        print(str(err))
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt in ("-t", "--token"):
+            token = arg
+    download_from_git(token,
+                      org=ORGANIZATION,
+                      repo=REPO,
+                      branch=BRANCH,
+                      folder=DIRECTORY)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/replace_all_occurences_in_file.py b/replace_all_occurences_in_file.py
@@ -0,0 +1,13 @@
+import os
+
+
+def replace_all_occurrences_in_file(path_old, path_new, pat, to):
+    file1 = open(path_old, 'r')
+    file2 = open(path_new, 'w')
+
+    for line in file1.readlines():
+        new_line = line.replace(pat, to)
+        file2.write(new_line)
+    file1.close()
+    file2.close()
+    os.remove(path_old)
diff --git a/safe_create_dir.py b/safe_create_dir.py
@@ -0,0 +1,10 @@
+import os
+import errno
+
+
+def safe_create_directory(path):
+    try:
+        os.makedirs(path)
+    except OSError as exception:
+        if exception.errno != errno.EEXIST:
+            raise