Revise tools scripts to be python3 compatible on win32

We introduce setup_stdio function to setup stdout/stderr properly.
For python <-> python pipe, we always use 'utf8'/'ignore' encoding for not lost
characters.
For tty <-> python, we using native encoding with xmlcharrefreplace to encode, to
preserve maximal information.
For python <-> native program, we use naive encoding with 'ignore' to not cause error

update_exclude_list with binary mode so that on win32 would not generate \r\n

run-test-suite.py: Handling skiplist properly on win32

Fixes jerryscript-project#4854

Fixes test262-harness.py complain cannot use a string pattern on a bytes-like object with running test262 with python3

For reading/writing to file, we use 'utf8' /'ignore' encoding for not lost characters.
For decoding from process stdout, using native encoding with decoding error ignored for not lost data.
Execute commands also ignore errors
Fixes jerryscript-project#4853

Fixes running test262-esnext failed with installed python3.9 on win32 with space in path
Fixes jerryscript-project#4852

JerryScript-DCO-1.0-Signed-off-by: Yonggang Luo luoyonggang@gmail.com

tools/run-tests.py

@@ -251,7 +251,8 @@ def create_binary(job, options):
         subprocess.check_output(build_cmd)
         ret = 0
     except subprocess.CalledProcessError as err:
-        print(err.output.decode("utf8"))
+        # For python <-> native program, we use default encoding with error='ignore' to not lost data
+        print(err.output.decode(errors="ignore"))
         ret = err.returncode
 
     BINARY_CACHE[binary_key] = (ret, build_dir_path)
@@ -445,6 +446,7 @@ def run_buildoption_test(options):
 Check = collections.namedtuple('Check', ['enabled', 'runner', 'arg'])
 
 def main(options):
+    util.setup_stdio()
     checks = [
         Check(options.check_signed_off, run_check, [settings.SIGNED_OFF_SCRIPT]
               + {'tolerant': ['--tolerant'], 'gh-actions': ['--gh-actions']}.get(options.check_signed_off, [])),

tools/runners/run-test-suite-test262.py

@@ -91,14 +91,14 @@ def update_exclude_list(args):
     # Tests pass in strict-mode but fail in non-strict-mode (or vice versa) should be considered as failures
     passing_tests = passing_tests - failing_tests
 
-    with open(args.excludelist_path, 'r+', encoding='utf8') as exclude_file:
+    with open(args.excludelist_path, 'rb+') as exclude_file:
         lines = exclude_file.readlines()
         exclude_file.seek(0)
         exclude_file.truncate()
 
         # Skip the last line "</excludeList>" to be able to insert new failing tests.
         for line in lines[:-1]:
-            match = re.match(r"  <test id=\"(\S*)\">", line)
+            match = re.match(r"  <test id=\"(\S*)\">", line.decode('utf-8', 'ignore'))
             if match:
                 test = match.group(1)
                 if test in failing_tests:
@@ -114,11 +114,12 @@ def update_exclude_list(args):
         if failing_tests:
             print("New failing tests added to the excludelist")
             for test in sorted(failing_tests):
-                exclude_file.write('  <test id="' + test + '"><reason></reason></test>\n')
+                line_added = '  <test id="' + test + '"><reason></reason></test>\n'
+                exclude_file.write(line_added.encode('utf-8'))
                 print("  " + test)
             print("")
 
-        exclude_file.write('</excludeList>\n')
+        exclude_file.write('</excludeList>\n'.encode('utf-8'))
 
     if new_passing_tests:
         print("New passing tests removed from the excludelist")
@@ -135,6 +136,7 @@ def update_exclude_list(args):
 
 
 def main(args):
+    util.setup_stdio()
     return_code = prepare_test262_test_suite(args)
     if return_code:
         return return_code

tools/runners/run-test-suite.py

@@ -66,6 +66,7 @@ def get_tests(test_dir, test_list, skip_list):
     tests.sort()
 
     def filter_tests(test):
+        test = test.replace('\\', '/')
         for skipped in skip_list:
             if skipped in test:
                 return False
@@ -78,13 +79,15 @@ def execute_test_command(test_cmd):
     kwargs = {}
     if sys.version_info.major >= 3:
         kwargs['encoding'] = 'unicode_escape'
+        kwargs['errors'] = 'ignore'
     with subprocess.Popen(test_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                           universal_newlines=True, **kwargs) as process:
         stdout, _ = process.communicate()
         return process.returncode, stdout
 
 
 def main(args):
+    util.setup_stdio()
     tests = get_tests(args.test_dir, args.test_list, args.skip_list)
     total = len(tests)
     if total == 0:

tools/runners/run-unittests.py

@@ -49,6 +49,7 @@ def get_unittests(path):
 
 
 def main(args):
+    util.setup_stdio()
     unittests = get_unittests(args.path)
     total = len(unittests)
     if total == 0:

tools/runners/test262-harness.py

@@ -40,6 +40,7 @@
 # This code is governed by the BSD license found in the LICENSE file.
 
 
+import codecs
 import logging
 import argparse
 import os
@@ -55,6 +56,8 @@
 import signal
 import multiprocessing
 
+import util
+
 #######################################################################
 # based on _monkeyYaml.py
 #######################################################################
@@ -490,8 +493,8 @@ def __init__(self, suite, name, full_path, strict_mode, command_template, module
         self.name = name
         self.full_path = full_path
         self.strict_mode = strict_mode
-        with open(self.full_path, "r", newline='', encoding='utf8') as file_desc:
-            self.contents = file_desc.read()
+        with open(self.full_path, "rb") as file_desc:
+            self.contents = file_desc.read().decode("utf8", "ignore")
         test_record = parse_test_record(self.contents, name)
         self.test = test_record["test"]
         del test_record["test"]
@@ -901,6 +904,7 @@ def list_includes(self, tests):
 
 
 def main():
+    util.setup_stdio()
     code = 0
     parser = build_options()
     options = parser.parse_args()

tools/runners/util.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import signal
 import subprocess
 import sys
@@ -42,6 +43,17 @@ def set_sighdl_to_reset_timezone(timezone):
     signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(timezone))
 
 
+def setup_stdio():
+    (out_stream, err_stream) = (sys.stdout, sys.stderr)
+    if sys.version_info.major >= 3:
+        (out_stream, err_stream) = (sys.stdout.buffer, sys.stderr.buffer)
+    # For tty using native encoding, otherwise (pipe) use 'utf-8'
+    encoding = sys.stdout.encoding if sys.stdout.isatty() else 'utf-8'
+    # Always override it to anvoid encode error
+    sys.stdout = codecs.getwriter(encoding)(out_stream, 'xmlcharrefreplace')
+    sys.stderr = codecs.getwriter(encoding)(err_stream, 'xmlcharrefreplace')
+
+
 def print_test_summary(summary_string, total, passed, failed):
     print(f"\n[summary] {summary_string}\n")
     print(f"TOTAL: {total}")
@@ -72,4 +84,4 @@ def get_platform_cmd_prefix():
 
 def get_python_cmd_prefix():
     # python script doesn't have execute permission on github actions windows runner
-    return get_platform_cmd_prefix() + [sys.executable or 'python']
+    return [sys.executable or 'python']