Upgrade antlr to 4.11 (#2733)

Author: jlopezpena

.circleci/config.yml

@@ -100,7 +100,7 @@ commands:
           name: Preparing environment - system
           command: |
             choco install -y --no-progress miniconda3
-            choco install -y --no-progress openssl javaruntime
+            choco install -y --no-progress openssl openjdk11jre
             C:\tools\miniconda3\Scripts\conda.exe init powershell
       - run:
           name: Preparing environment - Hydra

.gitignore

@@ -17,3 +17,4 @@ pip-wheel-metadata
 .ipynb_checkpoints
 /.dmypy.json
 TODO.txt
+/venv

build_helpers/bin/antlr-4.11.1-complete.jar

@@ -4,7 +4,7 @@ import subprocess
 import sys
 
 my_dir = os.path.realpath(os.path.dirname(__file__))
-antlr = "antlr-4.9.3-complete.jar"
+antlr = "antlr-4.11.1-complete.jar"
 args = ["java", "-jar", my_dir + "/" + antlr]
 args.extend(sys.argv[1:])
 subprocess.check_call(args)

build_helpers/bin/antlr-4.9.3-complete.jar

@@ -4,7 +4,7 @@ import subprocess
 import sys
 
 my_dir = os.path.realpath(os.path.dirname(__file__))
-antlr = "antlr-4.9.3-complete.jar"
+antlr = "antlr-4.11.1-complete.jar"
 args = ["java", "-cp", my_dir + "/" + antlr, "org.antlr.v4.gui.TestRig"]
 args.extend(sys.argv[1:])
 subprocess.check_call(args)

build_helpers/bin/antlr4

@@ -6,7 +6,9 @@
 import re
 import shutil
 import subprocess
+from functools import partial
 from os.path import abspath, basename, dirname, exists, isdir, join
+from pathlib import Path
 from typing import List, Optional
 
 from setuptools import Command
@@ -185,7 +187,7 @@ def run(self) -> None:
             command = [
                 "java",
                 "-jar",
-                join(root_dir, "bin/antlr-4.9.3-complete.jar"),
+                join(root_dir, "bin/antlr-4.11.1-complete.jar"),
                 "-Dlanguage=Python3",
                 "-o",
                 join(project_root, "hydra/grammar/gen/"),
@@ -198,8 +200,37 @@ def run(self) -> None:
 
             subprocess.check_call(command)
 
+            log.info("Replacing imports of antlr4 in generated parsers")
+            self._fix_imports()
+
     def initialize_options(self) -> None:
         pass
 
     def finalize_options(self) -> None:
         pass
+
+    def _fix_imports(self) -> None:
+        """Fix imports from the generated parsers to use the vendored antlr4 instead"""
+        build_dir = Path(__file__).parent.absolute()
+        project_root = build_dir.parent
+        lib = "antlr4"
+        pkgname = 'omegaconf.vendor'
+
+        replacements = [
+            partial(  # import antlr4 -> import omegaconf.vendor.antlr4
+                re.compile(r'(^\s*)import {}\n'.format(lib), flags=re.M).sub,
+                r'\1from {} import {}\n'.format(pkgname, lib)
+            ),
+            partial(  # from antlr4 -> from fomegaconf.vendor.antlr4
+                re.compile(r'(^\s*)from {}(\.|\s+)'.format(lib), flags=re.M).sub,
+                r'\1from {}.{}\2'.format(pkgname, lib)
+            ),
+        ]
+
+        path = project_root / "hydra" / "grammar" / "gen"
+        for item in path.iterdir():
+            if item.is_file() and item.name.endswith(".py"):
+                text = item.read_text('utf8')
+                for replacement in replacements:
+                    text = replacement(text)
+                item.write_text(text, 'utf8')

build_helpers/bin/grun

@@ -2,7 +2,10 @@
 import sys
 from typing import Any, List, Optional
 
-from antlr4.error.Errors import LexerNoViableAltException, RecognitionException
+from omegaconf.vendor.antlr4.error.Errors import (
+    LexerNoViableAltException,
+    RecognitionException,
+)
 
 from hydra._internal.grammar import grammar_functions
 from hydra._internal.grammar.functions import Functions

build_helpers/build_helpers.py

@@ -3,9 +3,13 @@
 import warnings
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-from antlr4 import ParserRuleContext, TerminalNode, Token
-from antlr4.error.ErrorListener import ErrorListener
-from antlr4.tree.Tree import TerminalNodeImpl
+from omegaconf.vendor.antlr4 import (  # type: ignore[attr-defined]
+    ParserRuleContext,
+    TerminalNode,
+    Token,
+)
+from omegaconf.vendor.antlr4.error.ErrorListener import ErrorListener
+from omegaconf.vendor.antlr4.tree.Tree import TerminalNodeImpl
 
 from hydra._internal.grammar.functions import FunctionCall, Functions
 from hydra._internal.grammar.utils import _ESC_QUOTED_STR
@@ -162,7 +166,7 @@ def visitOverride(self, ctx: OverrideParser.OverrideContext) -> Override:
         if (
             override_type == OverrideType.DEL
             and isinstance(eq_node, TerminalNode)
-            and eq_node.symbol.type == Token.EOF
+            and eq_node.symbol.type == Token.EOF  # type: ignore[attr-defined]
         ):
             value = None
             value_type = None
@@ -244,26 +248,26 @@ def visitFunction(self, ctx: OverrideParser.FunctionContext) -> Any:
             ) from e
 
     def _createPrimitive(
-        self, ctx: ParserRuleContext
+        self, ctx: ParserRuleContext  # type: ignore[valid-type]
     ) -> Optional[Union[QuotedString, int, bool, float, str]]:
         ret: Optional[Union[int, bool, float, str]]
         first_idx = 0
-        last_idx = ctx.getChildCount()
+        last_idx = ctx.getChildCount()  # type: ignore[attr-defined]
         # skip first if whitespace
-        if self.is_ws(ctx.getChild(0)):
+        if self.is_ws(ctx.getChild(0)):  # type: ignore[attr-defined]
             if last_idx == 1:
                 # Only whitespaces => this is not allowed.
                 raise HydraException(
                     "Trying to parse a primitive that is all whitespaces"
                 )
             first_idx = 1
-        if self.is_ws(ctx.getChild(-1)):
+        if self.is_ws(ctx.getChild(-1)):  # type: ignore[attr-defined]
             last_idx = last_idx - 1
         num = last_idx - first_idx
         if num > 1:
             # Concatenate, while un-escaping as needed.
             tokens = []
-            for i, n in enumerate(ctx.getChildren()):
+            for i, n in enumerate(ctx.getChildren()):  # type: ignore[attr-defined]
                 if n.symbol.type == OverrideLexer.WS and (
                     i < first_idx or i >= last_idx
                 ):
@@ -276,7 +280,7 @@ def _createPrimitive(
                 )
             ret = "".join(tokens)
         else:
-            node = ctx.getChild(first_idx)
+            node = ctx.getChild(first_idx)  # type: ignore[attr-defined]
             if node.symbol.type == OverrideLexer.QUOTED_VALUE:
                 text = node.getText()
                 qc = text[0]
@@ -360,7 +364,7 @@ def _unescape_quoted_string(self, text: str) -> str:
         return "".join(tokens)
 
 
-class HydraErrorListener(ErrorListener):  # type: ignore
+class HydraErrorListener(ErrorListener):
     def syntaxError(
         self,
         recognizer: Any,

hydra/core/override_parser/overrides_parser.py

@@ -187,13 +187,14 @@ def _scan_all_plugins(
                             assert m is not None
                             loaded_mod = m.load_module(modname)
                         else:
-                            spec = importer.find_spec(modname)
+                            spec = importer.find_spec(modname)  # type: ignore[call-arg]
                             assert spec is not None
                             if modname in sys.modules:
                                 loaded_mod = sys.modules[modname]
                             else:
                                 loaded_mod = importlib.util.module_from_spec(spec)
                             if loaded_mod is not None:
+                                assert spec.loader is not None
                                 spec.loader.exec_module(loaded_mod)
                                 sys.modules[modname] = loaded_mod