Revise tools scripts to be python3 compatible on win32

We introduce setup_stdio function to setup stdout/stderr properly.
For python <-> python pipe, we always use 'utf8'/'ignore' encoding for not lost
characters.
For tty <-> python, we using native encoding with xmlcharrefreplace to encode, to
preserve maximal information.
For python <-> native program, we use naive encoding with 'ignore' to not cause error

update_exclude_list with binary mode so that on win32 would not generate \r\n

run-test-suite.py: Handling skiplist properly on win32

Fixes jerryscript-project#4854

Fixes test262-harness.py complain cannot use a string pattern on a bytes-like object with running test262 with python3

For reading/writing to file, we use 'utf8' /'ignore' encoding for not lost characters.
For decoding from process stdout, using native encoding with decoding error ignored for not lost data.
Execute commands also ignore errors
Fixes jerryscript-project#4853

Fixes running test262-esnext failed with installed python3.9 on win32 with space in path
Fixes jerryscript-project#4852

JerryScript-DCO-1.0-Signed-off-by: Yonggang Luo luoyonggang@gmail.com

tools/run-tests.py

@@ -251,7 +251,8 @@ def create_binary(job, options):
         subprocess.check_output(build_cmd)
         ret = 0
     except subprocess.CalledProcessError as err:
-        print(err.output.decode("utf8"))
+        # For python <-> native program, we use default encoding with error='ignore' to not lost data
+        print(err.output.decode(errors="ignore"))
         ret = err.returncode
 
     BINARY_CACHE[binary_key] = (ret, build_dir_path)
@@ -445,6 +446,7 @@ def run_buildoption_test(options):
 Check = collections.namedtuple('Check', ['enabled', 'runner', 'arg'])
 
 def main(options):
+    util.setup_stdio()
     checks = [
         Check(options.check_signed_off, run_check, [settings.SIGNED_OFF_SCRIPT]
               + {'tolerant': ['--tolerant'], 'gh-actions': ['--gh-actions']}.get(options.check_signed_off, [])),

tools/runners/run-test-suite-test262.py

@@ -22,12 +22,6 @@
 
 import util
 
-def get_platform_cmd_prefix():
-    if sys.platform == 'win32':
-        return ['cmd', '/S', '/C']
-    return ['python3']
-
-
 def get_arguments():
     execution_runtime = os.environ.get('RUNTIME', '')
     parser = argparse.ArgumentParser()
@@ -91,14 +85,14 @@ def update_exclude_list(args):
     # Tests pass in strict-mode but fail in non-strict-mode (or vice versa) should be considered as failures
     passing_tests = passing_tests - failing_tests
 
-    with open(args.excludelist_path, 'r+', encoding='utf8') as exclude_file:
+    with open(args.excludelist_path, 'rb+') as exclude_file:
         lines = exclude_file.readlines()
         exclude_file.seek(0)
         exclude_file.truncate()
 
         # Skip the last line "</excludeList>" to be able to insert new failing tests.
         for line in lines[:-1]:
-            match = re.match(r"  <test id=\"(\S*)\">", line)
+            match = re.match(r"  <test id=\"(\S*)\">", line.decode('utf-8', 'ignore'))
             if match:
                 test = match.group(1)
                 if test in failing_tests:
@@ -114,11 +108,12 @@ def update_exclude_list(args):
         if failing_tests:
             print("New failing tests added to the excludelist")
             for test in sorted(failing_tests):
-                exclude_file.write('  <test id="' + test + '"><reason></reason></test>\n')
+                line_added = '  <test id="' + test + '"><reason></reason></test>\n'
+                exclude_file.write(line_added.encode('utf-8'))
                 print("  " + test)
             print("")
 
-        exclude_file.write('</excludeList>\n')
+        exclude_file.write('</excludeList>\n'.encode('utf-8'))
 
     if new_passing_tests:
         print("New passing tests removed from the excludelist")
@@ -135,6 +130,7 @@ def update_exclude_list(args):
 
 
 def main(args):
+    util.setup_stdio()
     return_code = prepare_test262_test_suite(args)
     if return_code:
         return return_code
@@ -154,7 +150,7 @@ def main(args):
 
     test262_harness_path = os.path.join(args.test262_harness_dir, 'test262-harness.py')
 
-    test262_command = get_platform_cmd_prefix() + \
+    test262_command = util.get_python_cmd_prefix() + \
                       [test262_harness_path,
                        '--command', command,
                        '--tests', args.test_dir,

tools/runners/run-test-suite.py

@@ -66,6 +66,7 @@ def get_tests(test_dir, test_list, skip_list):
     tests.sort()
 
     def filter_tests(test):
+        test = test.replace('\\', '/')
         for skipped in skip_list:
             if skipped in test:
                 return False
@@ -78,13 +79,15 @@ def execute_test_command(test_cmd):
     kwargs = {}
     if sys.version_info.major >= 3:
         kwargs['encoding'] = 'unicode_escape'
+        kwargs['errors'] = 'ignore'
     with subprocess.Popen(test_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                           universal_newlines=True, **kwargs) as process:
         stdout, _ = process.communicate()
         return process.returncode, stdout
 
 
 def main(args):
+    util.setup_stdio()
     tests = get_tests(args.test_dir, args.test_list, args.skip_list)
     total = len(tests)
     if total == 0:

tools/runners/run-unittests.py

@@ -49,6 +49,7 @@ def get_unittests(path):
 
 
 def main(args):
+    util.setup_stdio()
     unittests = get_unittests(args.path)
     total = len(unittests)
     if total == 0:

tools/runners/test262-harness.py

@@ -40,6 +40,7 @@
 # This code is governed by the BSD license found in the LICENSE file.
 
 
+import codecs
 import logging
 import argparse
 import os
@@ -55,6 +56,8 @@
 import signal
 import multiprocessing
 
+import util
+
 #######################################################################
 # based on _monkeyYaml.py
 #######################################################################
@@ -400,11 +403,13 @@ def open_file(self):
             text=self.text)
 
     def write(self, string):
-        os.write(self.file_desc, string.encode('utf8'))
+        os.write(self.file_desc, string.encode('utf8', 'ignore'))
 
     def read(self):
-        with open(self.name, "r", newline='', encoding='utf8') as file_desc:
-            return file_desc.read()
+        file_desc = open(self.name, 'rb')
+        result = file_desc.read()
+        file_desc.close()
+        return result.decode('utf8', 'ignore')
 
     def close(self):
         if not self.is_closed:
@@ -490,7 +495,7 @@ def __init__(self, suite, name, full_path, strict_mode, command_template, module
         self.name = name
         self.full_path = full_path
         self.strict_mode = strict_mode
-        with open(self.full_path, "r", newline='', encoding='utf8') as file_desc:
+        with open(self.full_path, "r", newline='', encoding='utf8', errors='ignore') as file_desc:
             self.contents = file_desc.read()
         test_record = parse_test_record(self.contents, name)
         self.test = test_record["test"]
@@ -750,8 +755,8 @@ def get_include(self, name):
         if not name in self.include_cache:
             static = path.join(self.lib_root, name)
             if path.exists(static):
-                with open(static, encoding='utf8') as file_desc:
-                    contents = file_desc.read()
+                with open(static, 'rb') as file_desc:
+                    contents = file_desc.read().decode('utf8', 'ignore')
                     contents = re.sub(r'\r\n', '\n', contents)
                     self.include_cache[name] = contents + "\n"
             else:
@@ -839,7 +844,7 @@ def run(self, command_template, tests, print_summary, full_summary, logname, job
             report_error("No tests to run")
         progress = ProgressIndicator(len(cases))
         if logname:
-            self.logf = open(logname, "w", encoding='utf8')  # pylint: disable=consider-using-with
+            self.logf = codecs.open(logname, "w", encoding='utf8', errors='ignore')  # pylint: disable=consider-using-with
 
         if job_count == 1:
             for case in cases:
@@ -901,6 +906,7 @@ def list_includes(self, tests):
 
 
 def main():
+    util.setup_stdio()
     code = 0
     parser = build_options()
     options = parser.parse_args()

tools/runners/util.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import signal
 import subprocess
 import sys
@@ -42,6 +43,17 @@ def set_sighdl_to_reset_timezone(timezone):
     signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(timezone))
 
 
+def setup_stdio():
+    (out_stream, err_stream) = (sys.stdout, sys.stderr)
+    if sys.version_info.major >= 3:
+        (out_stream, err_stream) = (sys.stdout.buffer, sys.stderr.buffer)
+    # For tty using native encoding, otherwise (pipe) use 'utf-8'
+    encoding = sys.stdout.encoding if sys.stdout.isatty() else 'utf-8'
+    # Always override it to anvoid encode error
+    sys.stdout = codecs.getwriter(encoding)(out_stream, 'xmlcharrefreplace')
+    sys.stderr = codecs.getwriter(encoding)(err_stream, 'xmlcharrefreplace')
+
+
 def print_test_summary(summary_string, total, passed, failed):
     print(f"\n[summary] {summary_string}\n")
     print(f"TOTAL: {total}")
@@ -72,4 +84,4 @@ def get_platform_cmd_prefix():
 
 def get_python_cmd_prefix():
     # python script doesn't have execute permission on github actions windows runner
-    return get_platform_cmd_prefix() + [sys.executable or 'python']
+    return [sys.executable or 'python']