Add Chapter 4 (#23)

* Add specified function definitions

Misc functions are to be imported after this

* Import type checking and error methods

Only some are used, but all of them are imported to keep this file up to
date with all the functions that were present before

* Remove duplicate error function

* Add metaCircularParser to Context

This Parser will be preserved for the lifetime of the Context, and this
ensures that it is reset together with the context as well.

* Add parse function

* Fix no typing error for misc

* Rename JSON.stringify -> stringify

* Remove unused Parser import

From rebasing away jison commits

* Add base parser file

* Format parser file

* Use .js for parser file

* Import and use parser for parse function

* Add parser jison source

* Add metacircular interpreter source

* Add doc to refer to source

* Fix tests not compiling

There was a problem with transpiling the files when testing, because of
the .js files used. I first added a transformer for the .js files (that
didnt have one at first, throwing an error at the `export` line in
parser.js). Then, I added the presets required, which were displayed as
missing presets when trying to test

* Update build to a script

This script will run the typescript build, and then copy over the
required .js file over. Note that to add more .js files, the build
script will need to be modified.

* Use && for failing tsc

* testing dev env

* Add .js files to interpreter folder

Oh my

* testing of martins dev setup for meta

* another try

* another try: getting really ugly

* getting uglier

* getting ready for prime time

* forgot removing tag

* trying to get => to work

* another attempt to fix arrow

* minor mods

Author: remo5000

package.json

@@ -22,7 +22,7 @@
   ],
   "scripts": {
     "prepublishOnly": "tsc",
-    "build": "tsc",
+    "build": "scripts/build.sh",
     "test": "jest",
     "test-coveralls": "jest --coverage --coverageReporters=text-lcov | coveralls"
   },
@@ -39,6 +39,8 @@
     "@types/invariant": "^2.2.29",
     "@types/jest": "^23.1.4",
     "@types/node": "^9.4.7",
+    "babel-preset-es2015": "^6.24.1",
+    "babel-preset-stage-2": "^6.24.1",
     "coveralls": "^3.0.1",
     "jest": "^23.3.0",
     "ts-jest": "^23.0.0",
@@ -50,7 +52,8 @@
       "js"
     ],
     "transform": {
-      "\\.ts$": "<rootDir>/node_modules/ts-jest/preprocessor.js"
+      "\\.ts$": "<rootDir>/node_modules/ts-jest/preprocessor.js",
+      "\\.js$": "<rootDir>/node_modules/babel-jest"
     },
     "testRegex": "/__tests__/.*\\.ts$",
     "testPathIgnorePatterns": [

scripts/build.sh

@@ -0,0 +1,6 @@
+# Compile all .ts files, then
+# copy the only .js file, parser.js, to dist.
+# This is required because it is not possible to both inlcude .js
+# files (allowJs = true) and export type declarations (declarations = true) 
+# in tsconfig. See https://github.com/Microsoft/TypeScript/issues/7546
+tsc && cp src/stdlib/parser.js dist/stdlib

src/createContext.ts

@@ -3,6 +3,10 @@ import * as misc from './stdlib/misc'
 import { Context, CustomBuiltIns, Value } from './types'
 import { toString } from '.';
 
+/** Import meta-circular parser */
+const createParserModule = require('./stdlib/parser')
+const createParser = createParserModule.default
+
 const GLOBAL = typeof window === 'undefined' ? global : window
 
 const createEmptyCFG = () => ({
@@ -24,7 +28,8 @@ export const createEmptyContext = <T>(chapter: number, externalSymbols: string[]
   errors: [],
   externalContext,
   cfg: createEmptyCFG(),
-  runtime: createEmptyRuntime()
+  runtime: createEmptyRuntime(),
+  metaCircularParser: createParser()
 })
 
 export const ensureGlobalEnvironmentExist = (context: Context) => {
@@ -132,6 +137,34 @@ export const importBuiltins = (context: Context, externalBuiltIns: CustomBuiltIn
     defineSymbol(context, 'array_length', misc.array_length)
   }
 
+  if (context.chapter >= 4) {
+    defineSymbol(context, 'stringify', JSON.stringify)
+    defineSymbol(context, 'parse',
+      function () { 
+        return context.metaCircularParser
+          .parse.apply(context.metaCircularParser, arguments)
+      });
+    defineSymbol(context, 'apply_in_underlying_javascript', function(
+      fun: Function,
+      args: Value
+    ) {
+      const res = []
+      var i = 0
+      while (!(args.length === 0)) {
+        res[i] = args[0]
+        i = i + 1
+        args = args[1]
+      }
+      return fun.apply(fun, res)
+    })
+    defineSymbol(context, 'is_number', misc.is_number)
+    defineSymbol(context, 'is_array', misc.is_array)
+    defineSymbol(context, 'is_object', misc.is_object)
+    defineSymbol(context, 'is_string', misc.is_string)
+    defineSymbol(context, 'is_function', misc.is_function)
+    defineSymbol(context, 'is_boolean', misc.is_boolean)
+  }
+
   if (context.chapter >= Infinity) {
     // previously week 4
     defineSymbol(context, 'alert', alert)
@@ -140,7 +173,6 @@ export const importBuiltins = (context: Context, externalBuiltIns: CustomBuiltIn
     // previously week 5
     defineSymbol(context, 'assoc', list.assoc)
     // previously week 6
-    defineSymbol(context, 'is_number', misc.is_number)
   }
 }
 

src/stdlib/metacircular-interpreter/Makefile

@@ -0,0 +1,33 @@
+INTERPRETER_CORE = lib/exports.js.tpl \
+	lib/interpreter/interpreter.js \
+	lib/interpreter/json2.js \
+	lib/interpreter/parser.jison.tpl \
+	lib/util/io.js
+
+PARSER = lib/interpreter/parser.js
+
+.PHONY: environment academy
+
+all: environment
+
+# removed from dependencies for environment: ../list.js ../object.js ../stream.js
+environment: lib/interpreter/parser.js 
+
+clean:
+	rm -f jediscript-*.js
+
+distclean: clean
+	rm -f lib/*.js
+
+academy: environment
+	node generate.js --interpreter 5 --without_jquery
+	node generate.js --interpreter 8 --without_jquery
+	node generate.js --interpreter 13 --without_jquery --debug
+
+offline_zip: environment
+	node generate.js --interpreter 5
+	node generate.js --interpreter 8
+	node generate.js --interpreter 13
+
+$(PARSER): $(INTERPRETER_CORE)
+	node generate.js --environment

src/stdlib/metacircular-interpreter/README.md

@@ -0,0 +1,32 @@
+# JediScript Meta-Circular Evaluator
+####_vastly more up-to-date instructions, in nice Markdown format._
+
+This project comprises 4 main components:
+
+1. The parser, written in Jison
+2. The interpreter, written in legal JediScript
+3. The scratch project templates, which are a combination of JediScript, JavaScript, and Markup-JS (a templating engine)
+4. The project generator (`generate.js`)
+
+These all work together to form the functional evaluator.
+
+## Setting up the environment
+1. This project depends on node.js. Install [node.js](http://nodejs.org) for your platform.
+2. Using the shell for your platform, run `envsetup.{sh|bat}`. This will obtain the necessary installation dependencies and generate the development versions of the templates.
+   - Note that your Jison install will be patched to throw SyntaxError exceptions when parse errors occur. This may affect your other Jison dependents, if any.
+2 (alternative) . Just run `npm install` and all the dependencies should be magically installed.  
+3. If you have made changes to the templates, generate new development libraries by running `generate.js --environment`.
+4. `scratch.html` in the root directory of the project has got the evaluator-in-evaluator code being part of `test3`. Use that to test the interpreter interpreting itself.
+
+## Generating Student Scratch Projects
+In addition to generating the development libraries, the templates can be used in concert with the generator to produce scratch projects which only contain valid JediScript elements for any given week. Generate these by running `generate.js --week N`.
+
+If you are modifying the templates, note that:
+
+- When modifying the Jison template source, Jison blocks within Markup-JS blocks (i.e. Jison declarations within the `{{if}}` or other blocks) must have `$$` escaped to `$$$`. This is because Markup-JS will remove one `$`, leaving invalid Jison declarations
+- If new user-defined functions must be provided to the student, update `export.js`. `export.js` contains a list of key-value pairs, which will be registered in the interpreter so the student can access them from his script file.
+- By default, the generated scratch project will have the parser screen the input from the student's script tags and evaluate it. The student's code can still be directly evaluated by the browser's JavaScript engine by passing `--native` (`--no-native` will equivalently enable interpreted mode. But that's the default)
+- The generator will send all output through to Google's Closure compiler, both to shrink the codebase and also to optimise the code. Passing `--debug` while generating the package will suppress this, so that the generator (or the libraries) can be debugged.
+
+## Unit Testing
+I've coded unit tests for the interpreter. Open `test.html`, the test suite will run automatically. This uses the parser and other libraries generated by `generate.js --environment`, so  be sure to run it before any testing.

src/stdlib/metacircular-interpreter/count/count-tokens.js

@@ -0,0 +1,81 @@
+
+var jison = require("./lib/interpreter/parser-week-13");
+
+function parse(text) {
+	return new jison.Parser().parse(text);
+}
+
+function findSource(text, start_line, start_col, end_line, end_col) {
+	var lines = text.replace(/\r/g, '').split('\n');
+
+	--start_line, --end_line; // 0-based indexing for lines
+	// cols are already 0-based
+
+	var range = [];
+	for (var i=start_line; i<=end_line; i++) {
+		range.push(lines[i]);
+	}
+
+	if (range.length === 1) {
+		range[0] = range[0].slice(start_col, end_col+1);
+		return range[0];
+	} else {
+		range[0] = range[0].slice(start_col);
+		range[range.length-1] = range[range.length-1].slice(0, end_col+1);
+		return range.join('\n');
+	}
+}
+
+function listToArray(xs) {
+	var result = [];
+	while (xs.length !== 0) {
+		result.push(xs[0]);
+		xs = xs[1];
+	}
+	return result;
+}
+
+function findTopLevelFunctionDeclarations(text) {
+	var ast = listToArray(parse(text));
+	var declarations = ast.filter(function (node) {
+		return node.tag === 'var_definition'
+			&& node.value
+			&& node.value.tag === 'function_definition';
+	}).map(function (node) {
+		var loc = node.value.location;
+		return {
+			name: node.variable,
+			source: findSource(text, loc.start_line, loc.start_col, loc.end_line, loc.end_col)
+		};
+	});
+	return declarations;
+}
+
+// Invert symbol object so we can easily lookup symbol names
+
+var symbols = new jison.Parser().symbols_;
+var symbolName = {};
+Object.keys(symbols).forEach(function (key) {
+	symbolName[symbols[key]] = key;
+});
+
+function lex(input) {
+	var lexer = new jison.Parser().lexer;
+	lexer.setInput(input);
+	var token;
+	var tokens = [];
+	while (symbolName[token = lexer.lex()] !== 'EOF') {
+		tokens.push(symbolName[token]);
+	}
+	return tokens;
+}
+
+module.exports = {
+	count: function(program) {
+		var declarations = findTopLevelFunctionDeclarations(program);
+		declarations.forEach(function (item) {
+			item.count = lex(item.source).length;
+		});
+		return declarations;
+	}
+};

src/stdlib/metacircular-interpreter/count/example.js

@@ -0,0 +1,8 @@
+
+var fs = require('fs');
+var count = require('./count-tokens').count;
+
+var input = "function x()\n{return 1;}\nfunction y()\n{return 1;}\nfunction z()\n{return 10;}";
+
+console.log(count(input));
+console.log(count(fs.readFileSync('shouldbe36.js', 'utf8')));