Revise tools scripts to be python3 compatible

We introduce setup_stdio function to setup stdout/stderr properly.
For python <-> python pipe, we always use 'utf8'/'ignore' encoding for not lost
characters.
For tty <-> python, we using native encoding with xmlcharrefreplace to encode, to
preserve maximal information.
For python <-> native program, we use naive encoding with 'ignore' to not cause error

Fixes #4854

JerryScript-DCO-1.0-Signed-off-by: Yonggang Luo [email protected]

Author: lygstate

tools/run-tests.py

@@ -498,6 +498,7 @@ def run_buildoption_test(options):
 Check = collections.namedtuple('Check', ['enabled', 'runner', 'arg'])
 
 def main(options):
+    util.setup_stdio()
     checks = [
         Check(options.check_signed_off, run_check, [settings.SIGNED_OFF_SCRIPT]
               + {'tolerant': ['--tolerant'], 'gh-actions': ['--gh-actions']}.get(options.check_signed_off, [])),

tools/runners/run-test-suite-test262.py

@@ -174,6 +174,7 @@ def update_exclude_list(args):
 
 
 def main(args):
+    util.setup_stdio()
     return_code = prepare_test262_test_suite(args)
     if return_code:
         return return_code

tools/runners/run-test-suite.py

@@ -86,6 +86,7 @@ def execute_test_command(test_cmd):
 
 
 def main(args):
+    util.setup_stdio()
     tests = get_tests(args.test_dir, args.test_list, args.skip_list)
     total = len(tests)
     if total == 0:

tools/runners/run-unittests.py

@@ -50,6 +50,7 @@ def get_unittests(path):
 
 
 def main(args):
+    util.setup_stdio()
     unittests = get_unittests(args.path)
     total = len(unittests)
     if total == 0:

tools/runners/test262-harness.py

@@ -58,6 +58,8 @@
 import threading
 import multiprocessing
 
+import util
+
 #######################################################################
 # based on _monkeyYaml.py
 #######################################################################
@@ -917,6 +919,7 @@ def list_includes(self, tests):
 
 
 def main():
+    util.setup_stdio()
     code = 0
     parser = build_options()
     (options, args) = parser.parse_args()

tools/runners/util.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from __future__ import print_function
+import codecs
 import signal
 import subprocess
 import sys
@@ -45,6 +46,17 @@ def set_sighdl_to_reset_timezone(timezone):
     signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(timezone))
 
 
+def setup_stdio():
+    (out_stream, err_stream) = (sys.stdout, sys.stderr)
+    if sys.version_info.major >= 3:
+        (out_stream, err_stream) = (sys.stdout.buffer, sys.stderr.buffer)
+    # For tty using native encoding, otherwise (pipe) use 'utf-8'
+    encoding = sys.stdout.encoding if sys.stdout.isatty() else 'utf-8'
+    # Always override it to anvoid encode error
+    sys.stdout = codecs.getwriter(encoding)(out_stream, 'xmlcharrefreplace')
+    sys.stderr = codecs.getwriter(encoding)(err_stream, 'xmlcharrefreplace')
+
+
 def print_test_summary(summary_string, total, passed, failed):
     print("\n[summary] %s\n" % summary_string)
     print("TOTAL: %d" % total)