Revise tools scripts to be python3 compatible on win32

We introduce setup_stdio function to setup stdout/stderr properly.
For python <-> python pipe, we always use 'utf8'/'ignore' encoding for not lost
characters.
For tty <-> python, we using native encoding with xmlcharrefreplace to encode, to
preserve maximal information.
For python <-> native program, we use naive encoding with 'ignore' to not cause error

update_exclude_list with binary mode so that on win32 would not generate \r\n

run-test-suite.py: Handling skiplist properly on win32

Fixes jerryscript-project#4854

Fixes test262-harness.py complain cannot use a string pattern on a bytes-like object with running test262 with python3

For reading/writing to file, we use 'utf8' /'ignore' encoding for not lost characters.
For decoding from process stdout, using native encoding with decoding error ignored for not lost data.
Execute commands also ignore errors
Fixes jerryscript-project#4853

Fixes running test262-esnext failed with installed python3.9 on win32 with space in path
Fixes jerryscript-project#4852

```
support both / \ in --test262-test-list arg

On win32.
python tools/run-tests.py  --test262-es2015=update --test262-test-list=built-ins/decodeURI/
python tools/run-tests.py  --test262-es2015=update --test262-test-list=built-ins\decodeURI\
should be both valid,
currently only --test262-test-list=built-ins\decodeURI\ are valid.
```

```
Support snapshot-tests-skiplist.txt on win32 by use os.path.normpath
```

Guard run-tests.py with timer.

All run-tests.py are finished in 30 minutes in normal situation.
May increase the timeout future.

wait JERRY_CHECK_TIMEOUT

```

Move Windows CI to github actions

Convert run-debugger-test.sh to run-debugger-test.py

After this change, run-debugger-test.py could running on Win32 and OSX

Define TERM colors for win32 properly

```

```

flush stderr.write stdout.write

On CI, the stderr are redirect to stdout, and if we don't flush stderr and stdout,
The output from stderr/stdout would out of sync.

```

`Testing new Date(-8640000000000000) and fixes date for win32`
So that the CI can passed

JerryScript-DCO-1.0-Signed-off-by: Yonggang Luo luoyonggang@gmail.com

.github/workflows/gh-actions.yml

@@ -56,6 +56,34 @@ jobs:
       - run: $RUNNER -q --jerry-tests --buildoptions=--compile-flag=-m32,--cpointer-32bit=on
       - run: $RUNNER -q --jerry-tests --buildoptions=--compile-flag=-m32,--cpointer-32bit=on --build-debug
 
+  Win_x86-64_Conformance_Tests_ESNext:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: python $env:RUNNER --test262 update
+
+  Win_x86-64_Conformance_Tests_ESNext_Debug:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: python $env:RUNNER --test262 update --build-debug
+
+  Win_x86-64_Tests:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: python $env:RUNNER -q --jerry-tests
+      - run: python $env:RUNNER -q --unittests
+      - run: python $env:RUNNER -q --buildoption-test
+
+  Win_x86-64_Tests_Debug:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: python $env:RUNNER -q --jerry-tests --build-debug
+      - run: python $env:RUNNER -q --unittests --build-debug
+      - run: python $env:RUNNER -q --buildoption-test --build-debug
+
   OSX_x86-64_Build_Correctness_Unit_Tests:
     runs-on: macos-13
     steps:

README.md

@@ -2,7 +2,6 @@
 # JerryScript: JavaScript engine for the Internet of Things
 [![License](https://img.shields.io/badge/licence-Apache%202.0-brightgreen.svg?style=flat)](LICENSE)
 [![GitHub Actions Status](https://github.com/jerryscript-project/jerryscript/workflows/JerryScript%20CI/badge.svg)](https://github.com/jerryscript-project/jerryscript/actions)
-[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/ct8reap35u2vooa5/branch/master?svg=true)](https://ci.appveyor.com/project/jerryscript-project/jerryscript/branch/master)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgh.neting.cc%2Fjerryscript-project%2Fjerryscript.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgh.neting.cc%2Fjerryscript-project%2Fjerryscript?ref=badge_shield)
 [![IRC Channel](https://img.shields.io/badge/chat-on%20freenode-brightgreen.svg)](https://kiwiirc.com/client/irc.freenode.net/#jerryscript)
 

jerry-debugger/jerry_client.py

@@ -326,6 +326,8 @@ def main():
             break
         if res_type == result.PROMPT:
             prompt.cmdloop()
+            sys.stdout.flush()
+            sys.stderr.flush()
         elif res_type == result.TEXT:
             write(result.get_text())
         continue

jerry-port/win/jerry-port-win-date.c

@@ -23,8 +23,8 @@
 #include <winbase.h>
 #include <winnt.h>
 
-#define UNIX_EPOCH_IN_TICKS 116444736000000000ull /* difference between 1970 and 1601 */
-#define TICKS_PER_MS        10000ull /* 1 tick is 100 nanoseconds */
+#define UNIX_EPOCH_IN_TICKS 116444736000000000LL /* difference between 1970 and 1601 */
+#define TICKS_PER_MS        10000LL /* 1 tick is 100 nanoseconds */
 
 /*
  * If you take the limit of SYSTEMTIME (last millisecond in 30827) then you end up with
@@ -44,15 +44,9 @@
  * https://support.microsoft.com/en-us/help/167296/how-to-convert-a-unix-time-t-to-a-win32-filetime-or-systemtime
  */
 static void
-unix_time_to_filetime (double t, LPFILETIME ft_p)
+unix_time_to_filetime (LONGLONG t, LPFILETIME ft_p)
 {
-  LONGLONG ll = (LONGLONG) t * TICKS_PER_MS + UNIX_EPOCH_IN_TICKS;
-
-  /* FILETIME values before the epoch are invalid. */
-  if (ll < 0)
-  {
-    ll = 0;
-  }
+  LONGLONG ll = t * TICKS_PER_MS + UNIX_EPOCH_IN_TICKS;
 
   ft_p->dwLowDateTime = (DWORD) ll;
   ft_p->dwHighDateTime = (DWORD) (ll >> 32);
@@ -63,13 +57,15 @@ unix_time_to_filetime (double t, LPFILETIME ft_p)
  *
  * @return unix time
  */
-static double
+static LONGLONG
 filetime_to_unix_time (LPFILETIME ft_p)
 {
   ULARGE_INTEGER date;
+  LONGLONG ll;
   date.HighPart = ft_p->dwHighDateTime;
   date.LowPart = ft_p->dwLowDateTime;
-  return (double) (((LONGLONG) date.QuadPart - UNIX_EPOCH_IN_TICKS) / TICKS_PER_MS);
+  ll = date.QuadPart - UNIX_EPOCH_IN_TICKS;
+  return ll / TICKS_PER_MS;
 } /* filetime_to_unix_time */
 
 /**
@@ -85,6 +81,7 @@ jerry_port_local_tza (double unix_ms)
   FILETIME local;
   SYSTEMTIME utc_sys;
   SYSTEMTIME local_sys;
+  LONGLONG t = (LONGLONG) (unix_ms);
 
   /*
    * If the time is earlier than the date 1601-01-02, then always using date 1601-01-02 to
@@ -93,23 +90,23 @@ jerry_port_local_tza (double unix_ms)
    * after converting between local time and utc time, the time may be earlier than 1601-01-01
    * in UTC time, that exceeds the FILETIME representation range.
    */
-  if (unix_ms < (double) UNIX_EPOCH_DATE_1601_01_02)
+  if (t < UNIX_EPOCH_DATE_1601_01_02)
   {
-    unix_ms = (double) UNIX_EPOCH_DATE_1601_01_02;
+    t = UNIX_EPOCH_DATE_1601_01_02;
   }
 
   /* Like above, do not use the last supported day */
-  if (unix_ms > (double) UNIX_EPOCH_DATE_30827_12_29)
+  if (t > UNIX_EPOCH_DATE_30827_12_29)
   {
-    unix_ms = (double) UNIX_EPOCH_DATE_30827_12_29;
+    t = UNIX_EPOCH_DATE_30827_12_29;
   }
-  unix_time_to_filetime (unix_ms, &utc);
+  unix_time_to_filetime (t, &utc);
 
   if (FileTimeToSystemTime (&utc, &utc_sys) && SystemTimeToTzSpecificLocalTime (NULL, &utc_sys, &local_sys)
       && SystemTimeToFileTime (&local_sys, &local))
   {
-    double unix_local = filetime_to_unix_time (&local);
-    return (int32_t) (unix_local - unix_ms);
+    LONGLONG unix_local = filetime_to_unix_time (&local);
+    return (int32_t) (unix_local - t);
   }
 
   return 0;
@@ -125,7 +122,7 @@ jerry_port_current_time (void)
 {
   FILETIME ft;
   GetSystemTimeAsFileTime (&ft);
-  return filetime_to_unix_time (&ft);
+  return (double) filetime_to_unix_time (&ft);
 } /* jerry_port_current_time */
 
 #endif /* defined(_WIN32) */

tests/jerry/date-getters.js

@@ -110,8 +110,16 @@ assert (new Date(-1, -1, -1, -1, -1, -1, -1, -1).getMilliseconds() === 999);
 assert (isNaN(new Date(20000000, 0).getFullYear()));
 assert (new Date(0, 0).getFullYear() === 1900);
 assert (new Date(1.2, 0).getFullYear() === 1901);
+
+/* 7. test case */
+/* A Number can exactly represent all integers from -9,007,199,254,740,992 to 9,007,199,254,740,992 (21.1.2.8 and 21.1.2.6).
+   A time value supports a slightly smaller range of -8,640,000,000,000,000 to 8,640,000,000,000,000 milliseconds. */
 assert((new Date(8640000000000000).getFullYear()) == 275760);
 assert(isNaN(new Date(8640000000000001).getFullYear()));
+assert((new Date(-8640000000000000).getFullYear()) == -271821);
+assert(isNaN(new Date(-8640000000000001).getFullYear()));
+
+/* 8. test case */
 assert((new Date(-271821, 3, 21).getFullYear()) == -271821);
 assert(isNaN(new Date(1970, 0, -100000000).getFullYear())); 
 assert(new Date(1970, 0, -100000000 + 1).getFullYear() == -271821);

tools/run-tests.py

@@ -25,12 +25,16 @@
 
 if sys.version_info.major >= 3:
     from runners import util
+    from runners.util import TERM_NORMAL, TERM_YELLOW, TERM_BLUE, TERM_RED
 else:
     sys.path.append(os.path.dirname(os.path.realpath(__file__)) + '/runners')
     import util
 
 OUTPUT_DIR = os.path.join(settings.PROJECT_DIR, 'build', 'tests')
 
+# The saved result of get_arguments()
+options = None
+
 Options = collections.namedtuple('Options', ['name', 'build_args', 'test_args', 'skip'])
 Options.__new__.__defaults__ = ([], [], False)
 
@@ -170,6 +174,8 @@ def get_arguments():
                         help='Run "magic string source code generator should be executed" check')
     parser.add_argument('--build-debug', action='store_true',
                         help='Build debug version jerryscript')
+    parser.add_argument('--run-check-timeout', action='store_const', const=30 * 60,
+                        help='Specify run_check timeout, default to 30 minutes, unit: second')
     parser.add_argument('--jerry-debugger', action='store_true',
                         help='Run jerry-debugger tests')
     parser.add_argument('--jerry-tests', action='store_true',
@@ -197,11 +203,6 @@ def get_arguments():
 
 BINARY_CACHE = {}
 
-TERM_NORMAL = '\033[0m'
-TERM_YELLOW = '\033[1;33m'
-TERM_BLUE = '\033[1;34m'
-TERM_RED = '\033[1;31m'
-
 def report_command(cmd_type, cmd, env=None):
     sys.stderr.write(f'{TERM_BLUE}{cmd_type}{TERM_NORMAL}\n')
     if env is not None:
@@ -210,12 +211,14 @@ def report_command(cmd_type, cmd, env=None):
     sys.stderr.write(f"{TERM_BLUE}" +
                      f" \\{TERM_NORMAL}\n\t{TERM_BLUE}".join(cmd) +
                      f"{TERM_NORMAL}\n")
+    sys.stderr.flush()
 
 def report_skip(job):
     sys.stderr.write(f'{TERM_YELLOW}Skipping: {job.name}')
     if job.skip:
         sys.stderr.write(f' ({job.skip})')
     sys.stderr.write(f'{TERM_NORMAL}\n')
+    sys.stderr.flush()
 
 def create_binary(job, options):
     build_args = job.build_args[:]
@@ -246,13 +249,17 @@ def create_binary(job, options):
     if binary_key in BINARY_CACHE:
         ret, build_dir_path = BINARY_CACHE[binary_key]
         sys.stderr.write(f'(skipping: already built at {build_dir_path} with returncode {ret})\n')
+        sys.stderr.flush()
         return ret, build_dir_path
 
     try:
         subprocess.check_output(build_cmd)
         ret = 0
     except subprocess.CalledProcessError as err:
-        print(err.output.decode("utf8"))
+        # As on Win32, the output encoding of 'cmd.exe' is 'native' such as 'gbk' not 'utf8';
+        # For Linux, the output encoding of bash is 'utf8' by default, and native is 'utf8 in most case
+        # So it's better not specify the encoding when call native program with subprocess.check_output
+        print(err.output.decode(errors="ignore"))
         ret = err.returncode
 
     BINARY_CACHE[binary_key] = (ret, build_dir_path)
@@ -283,6 +290,7 @@ def iterate_test_runner_jobs(jobs, options):
 
         if build_dir_path in tested_paths:
             sys.stderr.write(f'(skipping: already tested with {build_dir_path})\n')
+            sys.stderr.flush()
             continue
         tested_paths.add(build_dir_path)
 
@@ -291,6 +299,7 @@ def iterate_test_runner_jobs(jobs, options):
 
         if bin_hash in tested_hashes:
             sys.stderr.write(f'(skipping: already tested with equivalent {tested_hashes[bin_hash]})\n')
+            sys.stderr.flush()
             continue
         tested_hashes[bin_hash] = build_dir_path
 
@@ -308,7 +317,7 @@ def run_check(runnable, env=None):
         env = full_env
 
     with subprocess.Popen(runnable, env=env) as proc:
-        proc.wait()
+        proc.wait(timeout=options.run_check_timeout)
         return proc.returncode
 
 def run_jerry_debugger_tests(options):
@@ -324,7 +333,7 @@ def run_jerry_debugger_tests(options):
                 if test_file.endswith(".cmd"):
                     test_case, _ = os.path.splitext(test_file)
                     test_case_path = os.path.join(settings.DEBUGGER_TESTS_DIR, test_case)
-                    test_cmd = [
+                    test_cmd = util.get_python_cmd_prefix() + [
                         settings.DEBUGGER_TEST_RUNNER_SCRIPT,
                         get_binary_path(build_dir_path),
                         channel,
@@ -445,7 +454,8 @@ def run_buildoption_test(options):
 
 Check = collections.namedtuple('Check', ['enabled', 'runner', 'arg'])
 
-def main(options):
+def main():
+    util.setup_stdio()
     checks = [
         Check(options.check_signed_off, run_check, [settings.SIGNED_OFF_SCRIPT]
               + {'tolerant': ['--tolerant'], 'gh-actions': ['--gh-actions']}.get(options.check_signed_off, [])),
@@ -469,4 +479,5 @@ def main(options):
                 sys.exit(ret)
 
 if __name__ == "__main__":
-    main(get_arguments())
+    options = get_arguments()
+    main()