From 5a95ed1bb16aaf4c02b48bcabe80d64976b06383 Mon Sep 17 00:00:00 2001 From: Seonghyun Kim Date: Thu, 29 Jun 2023 18:14:01 +0900 Subject: [PATCH] Update source and test driver for running test262 on windows Signed-off-by: Seonghyun Kim --- .github/workflows/es-actions.yml | 89 +++++++------- src/runtime/DateObject.cpp | 1 + src/util/SpinLock.h | 3 +- third_party/GCutil | 2 +- tools/run-tests.py | 145 +++++++++++------------ tools/test/test262/make_excludelist.py | 2 +- tools/test/test262/monkeyYaml.py | 29 +++-- tools/test/test262/parseTestRecord.py | 6 +- tools/test/test262/test262.py | 155 +++++++++++++++---------- 9 files changed, 240 insertions(+), 192 deletions(-) diff --git a/.github/workflows/es-actions.yml b/.github/workflows/es-actions.yml index 3fa379920..30363414c 100644 --- a/.github/workflows/es-actions.yml +++ b/.github/workflows/es-actions.yml @@ -43,45 +43,6 @@ jobs: ninja -Cout/mac/x64 ./tools/run-tests.py --engine="./out/mac/x64/escargot" new-es - build-on-windows: - runs-on: windows-2022 - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - uses: lukka/get-cmake@latest - - uses: GuillaumeFalourd/setup-windows10-sdk-action@v1.11 - with: - sdk-version: 20348 - - uses: ilammy/msvc-dev-cmd@v1.12.1 - with: - arch: x86 - sdk: "10.0.20348.0" - - name: Install msvc redist package - run: | - (new-object System.Net.WebClient).DownloadFile('https://github.com/abbodi1406/vcredist/releases/download/v0.73.0/VisualCppRedist_AIO_x86_x64.exe','VisualCppRedist_AIO_x86_x64.exe') - .\VisualCppRedist_AIO_x86_x64.exe /y - - name: Copy octane - run: | - copy test\octane\*.js - dir - - name: Build Win32 Release - run: | - CMake -G "Visual Studio 16 2019" -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION:STRING="10.0" -DCMAKE_SYSTEM_PROCESSOR=x86 -DESCARGOT_ARCH=x86 -DESCARGOT_MODE=release -Bout/win32_release_ninja/ -DESCARGOT_HOST=windows -DESCARGOT_OUTPUT=shell -DESCARGOT_LIBICU_SUPPORT=ON -DESCARGOT_LIBICU_SUPPORT_WITH_DLOPEN=OFF -DESCARGOT_TEST=ON -G Ninja -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl -DCMAKE_BUILD_TYPE=release - CMake --build out/win32_release_ninja/ --config Release - .\out\win32_release_ninja\escargot.exe run.js - - uses: ilammy/msvc-dev-cmd@v1.12.1 - with: - arch: x64 - sdk: "10.0.20348.0" - - name: Build Win64 Release - run: | - CMake -G "Visual Studio 16 2019" -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION:STRING="10.0" -DCMAKE_SYSTEM_PROCESSOR=x64 -DESCARGOT_ARCH=x64 -DESCARGOT_MODE=release -Bout/win64_release_ninja/ -DESCARGOT_HOST=windows -DESCARGOT_OUTPUT=shell -DESCARGOT_LIBICU_SUPPORT=ON -DESCARGOT_LIBICU_SUPPORT_WITH_DLOPEN=OFF -DESCARGOT_TEST=ON -G Ninja -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl -DCMAKE_BUILD_TYPE=release - CMake --build out/win64_release_ninja/ --config Release - .\out\win64_release_ninja\escargot.exe run.js - - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 15 build-test-on-android: runs-on: macos-12 strategy: @@ -214,6 +175,54 @@ jobs: ninja -Cout/release python3 ./tools/run-tests.py --engine="./out/release/escargot" new-es + test-on-windows: + runs-on: windows-2022 + strategy: + matrix: + arch: [x86, x64] + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - uses: szenius/set-timezone@v1.2 + with: + timezoneWindows: "Pacific Standard Time" + - uses: lukka/get-cmake@latest + - uses: GuillaumeFalourd/setup-windows10-sdk-action@v1.11 + with: + sdk-version: 20348 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install msvc redist package + run: | + (new-object System.Net.WebClient).DownloadFile('https://github.com/abbodi1406/vcredist/releases/download/v0.73.0/VisualCppRedist_AIO_x86_x64.exe','VisualCppRedist_AIO_x86_x64.exe') + .\VisualCppRedist_AIO_x86_x64.exe /y + - uses: ilammy/msvc-dev-cmd@v1.12.1 + with: + arch: ${{ matrix.arch }} + sdk: "10.0.20348.0" + - name: Build ${{ matrix.arch }} Release + run: | + CMake -G "Visual Studio 16 2019" -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION:STRING="10.0" -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.arch }} -DESCARGOT_ARCH=${{ matrix.arch }} -DESCARGOT_MODE=release -Bout/ -DESCARGOT_HOST=windows -DESCARGOT_OUTPUT=shell -DESCARGOT_LIBICU_SUPPORT=ON -DESCARGOT_LIBICU_SUPPORT_WITH_DLOPEN=OFF -DESCARGOT_THREADING=ON -DESCARGOT_TCO=ON -DESCARGOT_TEST=ON -G Ninja -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl -DCMAKE_BUILD_TYPE=release + CMake --build out/ --config Release + # windows internal ICU doesn't support Temporal and intl402 well + # github action windows runner only have 2 CPUs. that's why I disable Atomics(timeout occured with some tests) + - name: Run test262 + run: | + set GC_FREE_SPACE_DIVISOR=1 + pip install chardet + python tools\run-tests.py --engine=%cd%\out\escargot.exe test262 --extra-arg="--skip Temporal --skip intl402 --skip Atomics" + shell: cmd + - name: Run octane + run: | + copy test\octane\*.js + dir + .\out\escargot.exe run.js + - if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 15 build-test-on-x86-release: runs-on: ubuntu-latest strategy: @@ -454,4 +463,4 @@ jobs: - name: Run x86 test run: $RUNNER --arch=x86 --engine="$GITHUB_WORKSPACE/out/wasm/x86/escargot" wasm-js - name: Run x64 test - run: $RUNNER --arch=x86_64 --engine="$GITHUB_WORKSPACE/out/wasm/x64/escargot" wasm-js + run: $RUNNER --arch=x86_64 --engine="$GITHUB_WORKSPACE/out/wasm/x64/escargot" wasm-js \ No newline at end of file diff --git a/src/runtime/DateObject.cpp b/src/runtime/DateObject.cpp index ce73c53a6..1446a7df9 100644 --- a/src/runtime/DateObject.cpp +++ b/src/runtime/DateObject.cpp @@ -1317,6 +1317,7 @@ String* DateObject::toTimeString(ExecutionState& state) snprintf(buffer, sizeof(buffer), "%02d:%02d:%02d GMT%s%04d", getHours(state), getMinutes(state), getSeconds(state), (tzOffsetAsMin < 0) ? "-" : "+", std::abs(tzOffsetHour + tzOffsetMin)); StringBuilder sb; sb.appendString(buffer); + sb.appendChar(' '); sb.appendChar('('); sb.appendString(String::fromUTF8(timeZoneName.data(), timeZoneName.length())); sb.appendChar(')'); diff --git a/src/util/SpinLock.h b/src/util/SpinLock.h index d2daf73c5..1a8df3d90 100644 --- a/src/util/SpinLock.h +++ b/src/util/SpinLock.h @@ -31,8 +31,9 @@ class SpinLock { public: SpinLock() - : m_locked(ATOMIC_FLAG_INIT) + : m_locked() { + m_locked.clear(); } void lock() diff --git a/third_party/GCutil b/third_party/GCutil index 0d6fe0e6d..c4804a317 160000 --- a/third_party/GCutil +++ b/third_party/GCutil @@ -1 +1 @@ -Subproject commit 0d6fe0e6da3ef58b60554d425bf141f63997bc03 +Subproject commit c4804a317fa3cc481279f1724ede5c53439b06dc diff --git a/tools/run-tests.py b/tools/run-tests.py index 8c732241c..73625c7ec 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -102,7 +102,7 @@ def readfile(filename): @runner('sunspider') -def run_sunspider(engine, arch): +def run_sunspider(engine, arch, extra_arg): run([join('.', 'sunspider'), '--shell', engine, '--suite', 'sunspider-1.0.2'], @@ -110,12 +110,12 @@ def run_sunspider(engine, arch): @runner('sunspider-js', default=True) -def run_sunspider_js(engine, arch): +def run_sunspider_js(engine, arch, extra_arg): run([engine] + sorted(glob(join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'SunSpider', 'tests', 'sunspider-1.0.2', '*.js')))) @runner('octane', default=True) -def run_octane(engine, arch): +def run_octane(engine, arch, extra_arg): max_retry_count = 5 try_count = 0 last_error = None @@ -149,7 +149,7 @@ def run_octane(engine, arch): @runner('octane-loading', default=True) -def run_octane_loading(engine, arch): +def run_octane_loading(engine, arch, extra_arg): OCTANE_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'octane') OCTANE_DIR = join(PROJECT_SOURCE_DIR, 'test', 'octane') copy(join(OCTANE_OVERRIDE_DIR, 'runLoading.js'), join(OCTANE_DIR, 'runLoading.js')) @@ -159,7 +159,7 @@ def run_octane_loading(engine, arch): @runner('modifiedVendorTest', default=True) -def run_internal_test(engine, arch): +def run_internal_test(engine, arch, extra_arg): INTERNAL_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'ModifiedVendorTest') INTERNAL_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'ModifiedVendorTest') @@ -169,9 +169,7 @@ def run_internal_test(engine, arch): run(['python', 'driver.py', engine, 'internal-test-cases.txt'], cwd=INTERNAL_DIR) - -@runner('test262', default=True) -def run_test262(engine, arch): +def copy_test262_files(): TEST262_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'test262') TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262') @@ -182,9 +180,16 @@ def run_test262(engine, arch): copy(join(TEST262_OVERRIDE_DIR, 'parseTestRecord.py'), join(TEST262_DIR, 'tools', 'packaging', 'parseTestRecord.py')) copy(join(TEST262_OVERRIDE_DIR, 'test262.py'), join(TEST262_DIR, 'tools', 'packaging', 'test262.py')) # for parallel running (we should re-implement this for es6 suite) - stdout = run(['pypy', join('tools', 'packaging', 'test262.py'), +@runner('test262', default=True) +def run_test262(engine, arch, extra_arg): + copy_test262_files() + TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262') + args = ['python3', join('tools', 'packaging', 'test262.py'), '--command', engine, - '--summary'], + '--summary'] + if len(extra_arg): + args.extend(extra_arg.split(" ")) + stdout = run(args, cwd=TEST262_DIR, env={'TZ': 'US/Pacific'}, stdout=PIPE) @@ -193,21 +198,20 @@ def run_test262(engine, arch): if summary.find('- All tests succeeded') < 0: raise Exception('test262 failed') print('test262: All tests passed') - -@runner('test262-strict', default=True) -def run_test262_strict(engine, arch): - TEST262_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'test262') + +@runner('test262-strict', default=False) +def run_test262_strict(engine, arch, extra_arg): + copy_test262_files() TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262') - - copy(join(TEST262_OVERRIDE_DIR, 'excludelist.orig.xml'), join(TEST262_DIR, 'excludelist.xml')) - copy(join(TEST262_OVERRIDE_DIR, 'test262.py'), join(TEST262_DIR, 'tools', 'packaging', 'test262.py')) # for parallel running (we should re-implement this for es6 suite) - out = open('test262-strict_out', 'w') - run(['pypy', join('tools', 'packaging', 'test262.py'), + args = ['python3', join('tools', 'packaging', 'test262.py'), '--command', engine, - '--full-summary', + '--full-summary', '--strict_only'], + if len(extra_arg): + args.extend(extra_arg.split(" ")) + run(args, cwd=TEST262_DIR, env={'TZ': 'US/Pacific'}, stdout=out, @@ -225,20 +229,19 @@ def run_test262_strict(engine, arch): print('test262-strict: All tests passed') -@runner('test262-nonstrict', default=True) -def run_test262_nonstrict(engine, arch): - TEST262_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'test262') +@runner('test262-nonstrict', default=False) +def run_test262_nonstrict(engine, arch, extra_arg): + copy_test262_files() TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262') - - copy(join(TEST262_OVERRIDE_DIR, 'excludelist.orig.xml'), join(TEST262_DIR, 'excludelist.xml')) - copy(join(TEST262_OVERRIDE_DIR, 'test262.py'), join(TEST262_DIR, 'tools', 'packaging', 'test262.py')) # for parallel running (we should re-implement this for es6 suite) - out = open('test262-nonstrict_out', 'w') - run(['pypy', join('tools', 'packaging', 'test262.py'), + args = ['python3', join('tools', 'packaging', 'test262.py'), '--command', engine, - '--full-summary', + '--full-summary', '--non_strict_only'], + if len(extra_arg): + args.extend(extra_arg.split(" ")) + run(args, cwd=TEST262_DIR, env={'TZ': 'US/Pacific'}, stdout=out, @@ -261,20 +264,16 @@ def compile_test_data_runner(): stdout=PIPE) @runner('test262-dump', default=False) -def run_test262_dump(engine, arch): - TEST262_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'test262') +def run_test262_dump(engine, arch, extra_arg): + copy_test262_files() TEST262_DIR = join(PROJECT_SOURCE_DIR, 'test', 'test262') - copy(join(TEST262_OVERRIDE_DIR, 'excludelist.orig.xml'), join(TEST262_DIR, 'excludelist.xml')) - copy(join(TEST262_OVERRIDE_DIR, 'cth.js'), join(TEST262_DIR, 'harness', 'cth.js')) - copy(join(TEST262_OVERRIDE_DIR, 'testIntl.js'), join(TEST262_DIR, 'harness', 'testIntl.js')) - - copy(join(TEST262_OVERRIDE_DIR, 'parseTestRecord.py'), join(TEST262_DIR, 'tools', 'packaging', 'parseTestRecord.py')) - copy(join(TEST262_OVERRIDE_DIR, 'test262.py'), join(TEST262_DIR, 'tools', 'packaging', 'test262.py')) # for parallel running (we should re-implement this for es6 suite) - - stdout = run(['pypy', join('tools', 'packaging', 'test262.py'), + args = ['python3', join('tools', 'packaging', 'test262.py'), '--command', engine, - '--summary'], + '--summary'] + if len(extra_arg): + args.extend(extra_arg.split(" ")) + stdout = run(args, cwd=TEST262_DIR, env={'TZ': 'US/Pacific', 'ESCARGOT_DUMP262DATA': '1'}, stdout=PIPE) @@ -285,7 +284,7 @@ def run_test262_dump(engine, arch): print('test262: All tests passed and dumped') @runner('test262-dump-run', default=False) -def run_test262_dump_run(engine, arch): +def run_test262_dump_run(engine, arch, extra_arg): compile_test_data_runner() stdout = run([PROJECT_SOURCE_DIR + '/tools/test/test-data-runner/test-data-runner', "--shell", engine, @@ -300,7 +299,7 @@ def run_test262_dump_run(engine, arch): print('test262-dump-passed') @runner('spidermonkey', default=True) -def run_spidermonkey(engine, arch): +def run_spidermonkey(engine, arch, extra_arg): SPIDERMONKEY_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'spidermonkey') SPIDERMONKEY_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'SpiderMonkey') @@ -329,7 +328,7 @@ def run_spidermonkey(engine, arch): raise Exception('failure files differ') @runner('spidermonkey-dump', default=False) -def run_spidermonkey_dump(engine, arch): +def run_spidermonkey_dump(engine, arch, extra_arg): SPIDERMONKEY_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'spidermonkey') SPIDERMONKEY_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'SpiderMonkey') @@ -358,7 +357,7 @@ def run_spidermonkey_dump(engine, arch): output.write("\n") @runner('spidermonkey-dump-run', default=False) -def run_spidermonkey_dump_run(engine, arch): +def run_spidermonkey_dump_run(engine, arch, extra_arg): compile_test_data_runner() SPIDERMONKEY_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'spidermonkey') @@ -375,7 +374,7 @@ def run_spidermonkey_dump_run(engine, arch): @runner('jsc-stress', default=True) -def run_jsc_stress(engine, arch): +def run_jsc_stress(engine, arch, extra_arg): JSC_DIR = join('test', 'vendortest', 'driver') run([join(JSC_DIR, 'driver.py'), @@ -386,7 +385,7 @@ def run_jsc_stress(engine, arch): env={'PYTHONPATH': '.'}) @runner('jsc-stress-dump', default=False) -def run_jsc_stress_dump(engine, arch): +def run_jsc_stress_dump(engine, arch, extra_arg): JSC_DIR = join('test', 'vendortest', 'driver') log_path = join(JSC_DIR, 'jsc.stress.%s.gen.txt' % arch) @@ -404,7 +403,7 @@ def run_jsc_stress_dump(engine, arch): output.write("\n") @runner('jsc-stress-dump-run', default=False) -def run_jsc_stress_dump_run(engine, arch): +def run_jsc_stress_dump_run(engine, arch, extra_arg): JSC_DIR = join('test', 'vendortest', 'driver') data_path = join(JSC_DIR, 'jsc.stress.%s.data.txt' % arch) @@ -439,7 +438,7 @@ def _run_regression_tests(engine, assert_js, files, is_fail): @runner('regression-tests', default=True) -def run_regression_tests(engine, arch): +def run_regression_tests(engine, arch, extra_arg): REGRESSION_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'regression-tests') REGRESSION_XFAIL_DIR = join(REGRESSION_DIR, 'xfail') REGRESSION_ASSERT_JS = join(REGRESSION_DIR, 'assert.js') @@ -485,42 +484,42 @@ def _run_jetstream(engine, target_test): @runner('jetstream-only-simple-parallel-1') -def run_jetstream_only_simple_parallel_1(engine, arch): +def run_jetstream_only_simple_parallel_1(engine, arch, extra_arg): _run_jetstream(engine, 'simple-1') @runner('jetstream-only-simple-parallel-2') -def run_jetstream_only_simple_parallel_2(engine, arch): +def run_jetstream_only_simple_parallel_2(engine, arch, extra_arg): _run_jetstream(engine, 'simple-2') @runner('jetstream-only-simple-parallel-3') -def run_jetstream_only_simple_parallel_3(engine, arch): +def run_jetstream_only_simple_parallel_3(engine, arch, extra_arg): _run_jetstream(engine, 'simple-3') @runner('jetstream-only-simple', default=True) -def run_jetstream_only_simple(engine, arch): +def run_jetstream_only_simple(engine, arch, extra_arg): _run_jetstream(engine, 'simple') @runner('jetstream-only-cdjs', default=True) -def run_jetstream_only_cdjs(engine, arch): +def run_jetstream_only_cdjs(engine, arch, extra_arg): _run_jetstream(engine, 'cdjs') @runner('jetstream-only-sunspider') -def run_jetstream_only_sunspider(engine, arch): +def run_jetstream_only_sunspider(engine, arch, extra_arg): _run_jetstream(engine, 'sunspider') @runner('jetstream-only-octane') -def run_jetstream_only_octane(engine, arch): +def run_jetstream_only_octane(engine, arch, extra_arg): _run_jetstream(engine, 'octane') @runner('chakracore', default=True) -def run_chakracore(engine, arch): +def run_chakracore(engine, arch, extra_arg): CHAKRACORE_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'chakracore') CHAKRACORE_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'ChakraCore') @@ -545,7 +544,7 @@ def run_chakracore(engine, arch): join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'driver', 'chakracore.%s.gen.txt' % arch)]) @runner('chakracore-dump', default=False) -def run_chakracore_dump(engine, arch): +def run_chakracore_dump(engine, arch, extra_arg): CHAKRACORE_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'chakracore') CHAKRACORE_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'ChakraCore') @@ -578,7 +577,7 @@ def run_chakracore_dump(engine, arch): output.write(baseline) @runner('chakracore-dump-run', default=False) -def run_chakracore_dump_run(engine, arch): +def run_chakracore_dump_run(engine, arch, extra_arg): compile_test_data_runner() CHAKRACORE_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'chakracore') @@ -596,7 +595,7 @@ def run_chakracore_dump_run(engine, arch): print('chakracore-dump-passed') @runner('v8', default=True) -def run_v8(engine, arch): +def run_v8(engine, arch, extra_arg): V8_OVERRIDE_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'v8') V8_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'v8') @@ -644,7 +643,7 @@ def run_v8(engine, arch): @runner('v8-dump', default=False) -def run_v8_dump(engine, arch): +def run_v8_dump(engine, arch, extra_arg): stdout = run_v8(engine, arch) stdout = stdout.decode("utf-8").split("\n") @@ -674,7 +673,7 @@ def run_v8_dump(engine, arch): print("SKIP " + casename) @runner('v8-dump-run', default=False) -def run_v8_dump_run(engine, arch): +def run_v8_dump_run(engine, arch, extra_arg): compile_test_data_runner() TOOL_V8_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'test', 'v8') @@ -690,7 +689,7 @@ def run_v8_dump_run(engine, arch): print('v8-dump-passed') @runner('new-es', default=True) -def run_new_es(engine, arch): +def run_new_es(engine, arch, extra_arg): NEW_ES_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'new-es') NEW_ES_ASSERT_JS = join(NEW_ES_DIR, 'assert.js') @@ -718,7 +717,7 @@ def run_new_es(engine, arch): raise Exception('new-es tests failed') @runner('intl', default=True) -def run_intl(engine, arch): +def run_intl(engine, arch, extra_arg): INTL_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'intl') INTL_ASSERT_JS = join(INTL_DIR, 'assert.js') @@ -741,7 +740,7 @@ def run_intl(engine, arch): raise Exception('Intl tests failed') @runner('escargot-test-dump', default=False) -def run_escargot_test_dump(engine, arch): +def run_escargot_test_dump(engine, arch, extra_arg): NEW_ES_DIR = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'Escargot', 'new-es') NEW_ES_ASSERT_JS = join(NEW_ES_DIR, 'assert.js') @@ -786,7 +785,7 @@ def run_escargot_test_dump(engine, arch): output.write("\n") @runner('escargot-test-dump-run', default=False) -def run_escargot_test_dump_run(engine, arch): +def run_escargot_test_dump_run(engine, arch, extra_arg): compile_test_data_runner() stdout = run([PROJECT_SOURCE_DIR + '/tools/test/test-data-runner/test-data-runner', "--shell", engine, @@ -801,7 +800,7 @@ def run_escargot_test_dump_run(engine, arch): print('escargot-dump-passed') @runner('wasm-js', default=False) -def run_wasm_js(engine, arch): +def run_wasm_js(engine, arch, extra_arg): WASM_TEST_ROOT = join(PROJECT_SOURCE_DIR, 'test', 'vendortest', 'wasm-js') WASM_TEST_DIR = join(WASM_TEST_ROOT, 'tests') WASM_TEST_MJS = join(WASM_TEST_ROOT, 'mjsunit.js') @@ -864,7 +863,7 @@ def run_wasm_js(engine, arch): raise Exception('new-es tests failed') @runner('cctest', default=False) -def run_cctest(engine, arch): +def run_cctest(engine, arch, extra_arg): proc = Popen([engine], stdout=PIPE) out, _ = proc.communicate() @@ -875,7 +874,7 @@ def run_cctest(engine, arch): raise Exception('Not all tests succeeded') @runner('debugger-server-source', default=True) -def run_escargot_debugger(engine, arch): +def run_escargot_debugger(engine, arch, extra_arg): ESCARGOT_DEBUGGER_TEST_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'tests') ESCARGOT_DEBUGGER_CLIENT = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'debugger.py') ESCARGOT_DEBUGGER_TESTER = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'debugger_tester.sh') @@ -897,7 +896,7 @@ def run_escargot_debugger(engine, arch): raise Exception('Escargot-Debugger-Server-Source tests failed') @runner('debugger-client-source', default=True) -def run_escargot_debugger2(engine, arch): +def run_escargot_debugger2(engine, arch, extra_arg): ESCARGOT_DEBUGGER_TEST_DIR = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'tests') ESCARGOT_DEBUGGER_CLIENT = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'debugger.py') ESCARGOT_DEBUGGER_TESTER = join(PROJECT_SOURCE_DIR, 'tools', 'debugger', 'debugger_tester.sh') @@ -919,13 +918,13 @@ def run_escargot_debugger2(engine, arch): raise Exception('Escargot-Debugger-Client-Source tests failed') @runner('dump-all', default=False) -def run_dump_all(engine, arch): +def run_dump_all(engine, arch, extra_arg): for test in RUNNERS: if test.endswith("-dump"): RUNNERS[test](engine, arch) @runner('dump-run-all', default=False) -def run_dump_run_all(engine, arch): +def run_dump_run_all(engine, arch, extra_arg): for test in RUNNERS: if test.endswith("-dump-run"): RUNNERS[test](engine, arch) @@ -936,6 +935,8 @@ def main(): help='path to the engine to be tested (default: %(default)s)') parser.add_argument('--arch', metavar='NAME', choices=['x86', 'x86_64'], default='x86_64', help='architecture the engine was built for (%(choices)s; default: %(default)s)') + parser.add_argument('--extra-arg', default='', + help='extra argument variable to drivers') parser.add_argument('suite', metavar='SUITE', nargs='*', default=sorted(DEFAULT_RUNNERS), help='test suite to run (%s; default: %s)' % (', '.join(sorted(RUNNERS.keys())), ' '.join(sorted(DEFAULT_RUNNERS)))) args = parser.parse_args() @@ -949,7 +950,7 @@ def main(): for suite in args.suite: print(COLOR_PURPLE + 'running test suite: ' + suite + COLOR_RESET) try: - RUNNERS[suite](args.engine, args.arch) + RUNNERS[suite](args.engine, args.arch, args.extra_arg) success += [suite] except Exception as e: print('\n'.join(COLOR_YELLOW + line + COLOR_RESET for line in traceback.format_exc().splitlines())) diff --git a/tools/test/test262/make_excludelist.py b/tools/test/test262/make_excludelist.py index ee50d16ec..c8db62416 100755 --- a/tools/test/test262/make_excludelist.py +++ b/tools/test/test262/make_excludelist.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + import os import re diff --git a/tools/test/test262/monkeyYaml.py b/tools/test/test262/monkeyYaml.py index 26ac194b4..7125899f6 100644 --- a/tools/test/test262/monkeyYaml.py +++ b/tools/test/test262/monkeyYaml.py @@ -14,12 +14,16 @@ mYamlMultilineList = re.compile(r"^ *- (.*)$") def load(str): + return myReadDict(str.splitlines())[1] + +def myReadDict(lines, indent=""): dict = None key = None emptyLines = 0 - - lines = str.splitlines() while lines: + if not lines[0].startswith(indent): + break + line = lines.pop(0) if myIsAllSpaces(line): emptyLines += 1 @@ -31,7 +35,7 @@ def load(str): dict = {} key = result.group(1).strip() value = result.group(2).strip() - (lines, value) = myReadValue(lines, value) + (lines, value) = myReadValue(lines, value, indent) dict[key] = value else: if dict and key and key in dict: @@ -40,17 +44,20 @@ def load(str): else: raise Exception("monkeyYaml is confused at " + line) emptyLines = 0 - return dict + return lines, dict -def myReadValue(lines, value): - if value == ">": +def myReadValue(lines, value, indent): + if value == ">" or value == "|": (lines, value) = myMultiline(lines, value) value = value + "\n" return (lines, value) - if lines and not value and myMaybeList(lines[0]): - return myMultilineList(lines, value) - else: - return lines, myReadOneLine(value) + if lines and not value: + if myMaybeList(lines[0]): + return myMultilineList(lines, value) + indentMatch = re.match("(" + indent + r"\s+)", lines[0]) + if indentMatch: + return myReadDict(lines, indentMatch.group(1)) + return lines, myReadOneLine(value) def myMaybeList(value): return mYamlMultilineList.match(value) @@ -64,7 +71,7 @@ def myMultilineList(lines, value): leading = myLeadingSpaces(line) if myIsAllSpaces(line): pass - elif leading < indent: + elif indent is not None and leading < indent: lines.insert(0, line) break; else: diff --git a/tools/test/test262/parseTestRecord.py b/tools/test/test262/parseTestRecord.py index 4fe7c9789..b33003a8d 100644 --- a/tools/test/test262/parseTestRecord.py +++ b/tools/test/test262/parseTestRecord.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2011 by Google, Inc. All rights reserved. # This code is governed by the BSD license found in the LICENSE file. @@ -81,7 +81,7 @@ def yamlAttrParser(testRecord, attrs, name): parsed = yamlLoad(body) if (parsed is None): - print "Failed to parse yaml in name %s"%(name) + print("Failed to parse yaml in name %s"%(name)) return for key in parsed: @@ -101,7 +101,7 @@ def parseTestRecord(src, name): # we need to skip hashbang before copyright comment if len(src) > 1 and (src[0] == '#' or src[0] == '\\'): src = src[src.find("// Copyright"):] - elif src[:25].rfind("#!") is not -1: + elif src[:25].rfind("#!") != -1: src = src[src.find("// Copyright"):] match = matchParts(src, name) diff --git a/tools/test/test262/test262.py b/tools/test/test262/test262.py index 1f4e5cae2..5e91aa64c 100755 --- a/tools/test/test262/test262.py +++ b/tools/test/test262/test262.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2009 the Sputnik authors. All rights reserved. # This code is governed by the BSD license found in the LICENSE file. @@ -23,24 +23,17 @@ import shutil import json import stat -import signal import xml.etree.ElementTree as xmlj import unicodedata +import chardet from collections import Counter +is_windows = sys.platform.startswith('win') from parseTestRecord import parseTestRecord, stripHeader from packagerConfig import * -class Alarm(Exception): - pass - -def alarm_handler(signum, frame): - raise Alarm - -signal.signal(signal.SIGALRM, alarm_handler) - class Test262Error(Exception): def __init__(self, message): self.message = message @@ -48,11 +41,9 @@ def __init__(self, message): def ReportError(s): raise Test262Error(s) - - if not os.path.exists(EXCLUDED_FILENAME): - print "Cannot generate (JSON) test262 tests without a file," + \ - " %s, showing which tests have been disabled!" % EXCLUDED_FILENAME + print("Cannot generate (JSON) test262 tests without a file," + \ + " %s, showing which tests have been disabled!" % EXCLUDED_FILENAME) sys.exit(1) EXCLUDE_LIST = xml.dom.minidom.parse(EXCLUDED_FILENAME) EXCLUDE_REASON = EXCLUDE_LIST.getElementsByTagName("reason") @@ -95,6 +86,7 @@ def BuildOptions(): result.add_option("--print-handle", default="print", help="Command to print from console") result.add_option("--list-includes", default=False, action="store_true", help="List includes required by tests") + result.add_option("--skip", default=[], action="append", help="skip testcase pattern") return result @@ -122,20 +114,30 @@ def __init__(self, suffix="", prefix="tmp", text=False): self.fd = None self.name = None self.is_closed = False - self.Open() + self.open() - def Open(self): + def open(self): (self.fd, self.name) = tempfile.mkstemp( suffix = self.suffix, prefix = self.prefix, text = self.text) def Write(self, str): - os.write(self.fd, str) + os.write(self.fd, str.encode("utf-8")) def Read(self): - f = file(self.name) - result = f.read() + f = open(self.name, encoding="utf-8", newline='') + try: + result = f.read() + except UnicodeDecodeError: + f.close() + t = open(self.name, "rb") + rawdata = t.read() + t.close() + result = chardet.detect(rawdata) + charenc = result['encoding'] + f = open(self.name, encoding=charenc, newline='') + result = f.read() f.close() return result @@ -148,7 +150,7 @@ def Dispose(self): try: self.Close() os.unlink(self.name) - except OSError, e: + except OSError as e: logging.error("Error disposing temp file: %s", str(e)) @@ -165,20 +167,20 @@ def ReportOutcome(self, long_format): mode = self.case.GetMode() if self.HasUnexpectedOutcome(): if self.case.IsNegative(): - print "=== %s was expected to fail in %s, but didn't ===" % (name, mode) - print "--- expected error: %s ---\n" % self.case.GetNegative() + print("=== %s was expected to fail in %s, but didn't ===" % (name, mode)) + print("--- expected error: %s ---\n" % self.case.GetNegative()) else: if long_format: - print "=== %s failed in %s ===" % (name, mode) + print("=== %s failed in %s ===" % (name, mode)) else: - print "%s in %s: " % (name, mode) + print("%s in %s: " % (name, mode)) self.WriteOutput(sys.stdout) if long_format: - print "===" + print("===") elif self.case.IsNegative(): - print "%s failed in %s as expected" % (name, mode) + print("%s failed in %s as expected" % (name, mode)) else: - print "%s passed in %s" % (name, mode) + print("%s passed in %s" % (name, mode)) def WriteOutput(self, target): out = self.stdout.strip() @@ -192,8 +194,8 @@ def WriteOutput(self, target): def SafeFormat(self, msg): try: msg = msg.encode(encoding='ascii', errors='strict') - msg = msg.replace('\u000Bx', '?') - msg = msg.replace('\u000Cx', '?') + msg = msg.replace('\\u000Bx', '?') + msg = msg.replace('\\u000Cx', '?') except: return 'Output contained invalid characters' @@ -254,7 +256,7 @@ def __init__(self, suite, index, name, full_path, strict_mode): self.name = name self.full_path = full_path self.strict_mode = strict_mode - f = open(self.full_path) + f = open(self.full_path, encoding="utf-8", newline='') self.contents = f.read() self.source = self.contents f.close() @@ -310,6 +312,11 @@ def GetIncludeList(self): return self.testRecord['includes'] return [] + def GetFeatureList(self): + if self.testRecord.get('features'): + return self.testRecord['features'] + return [] + def GetAdditionalIncludes(self): return '\n'.join([self.suite.GetInclude(include) for include in self.GetIncludeList()]) @@ -376,19 +383,19 @@ def Execute(self, command): env = my_env ) - signal.alarm(120) # raise Alarm in 2 minutes - + is_timeout_expired = False try: - code = process.wait() - signal.alarm(0) # reset the alarm - except Alarm: + code = process.wait(60 * 10) + except subprocess.TimeoutExpired: + is_timeout_expired = true process.kill() process.wait() code = -1 - # code = process.wait() out = stdout.Read() err = stderr.Read() + if is_timeout_expired: + err = err + "(TimeoutExpired)" finally: stdout.Dispose() stderr.Dispose() @@ -428,7 +435,22 @@ def RunTestIn(self, command_template, driver_tmp, tmp): 'can_block_is_false': can_block_is_false }) - (code, out, err) = self.Execute(command) + code = 0 + out = "" + err = "" + retry_count = 1 + count = 0 + + # timeouts does not work properly on some cases + if "Atomics" in self.GetFeatureList() or "SharedArrayBuffer" in self.GetFeatureList(): + retry_count = 10 + + while count < retry_count: + (code, out, err) = self.Execute(command) + result = TestResult(code, out, err, self) + if not result.HasUnexpectedOutcome(): + break + count = count + 1 if len(ESCARGOT_DUMP262DATA): self.escargot_data = dict() @@ -454,7 +476,7 @@ def Run(self, command_template): return result def Print(self): - print self.GetSource() + print(self.GetSource()) def validate(self): flags = self.testRecord.get("flags") @@ -509,7 +531,7 @@ def CaseRunner(case): class TestSuite(object): - def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle): + def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle, skip_patterns): # TODO: derive from packagerConfig.py self.test_root = path.join(root, 'test') if len(ESCARGOT_DUMP262DATA): @@ -523,6 +545,7 @@ def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_h self.print_handle = print_handle self.include_cache = { } self.total_test_number = 0 + self.skip_patterns = skip_patterns def Validate(self): if not path.exists(self.test_root): @@ -577,12 +600,17 @@ def EnumerateTests(self, tests): basename = path.basename(full_path)[:-3] name = rel_path.split(path.sep)[:-1] + [basename] file_path = rel_path[0:rel_path.rindex('.')] + if is_windows: + file_path = file_path.replace("\\", "/") if file_path in EXCLUDE_LIST: - #print 'Excluded: ' + rel_path skip = True elif EXCLUDE_LIST.count(basename) >= 1: - #print 'Excluded: ' + basename skip = True + else: + for s in self.skip_patterns: + if s in full_path: + skip = True + break if not self.non_strict_only: strict_case = TestCase(self, len(cases), name, full_path, True) @@ -609,9 +637,9 @@ def PrintSummary(self, progress, logfile): def write(s): if logfile: self.logf.write(s + "\n") - print s + print(s) - print + print() write("=== Test262 Summary ==="); count = progress.count succeeded = progress.succeeded @@ -626,13 +654,13 @@ def write(s): positive = [c for c in progress.failed_tests if not c.case.IsNegative()] negative = [c for c in progress.failed_tests if c.case.IsNegative()] if len(positive) > 0: - print + print() write("Test262 Failed Tests") for result in positive: write(" %s in %s" % (result.case.GetName(), result.case.GetMode())) write("Failed Tests End") if len(negative) > 0: - print + print() write("Test262 Expected to fail but passed") for result in negative: write(" %s in %s" % (result.case.GetName(), result.case.GetMode())) @@ -643,9 +671,9 @@ def PrintFull(self, progress, logfile): def write(s): if logfile: self.logf.write(s + "\n") - print s + print(s) - print + print() write("=== Test262 Summary ==="); count = progress.count succeeded = progress.succeeded @@ -658,13 +686,13 @@ def write(s): positive = [c for c in progress.succeeded_tests if not c.case.IsNegative()] negative = [c for c in progress.succeeded_tests if c.case.IsNegative()] if len(positive) > 0: - print + print() write("Test262 Succeeded Tests") for result in positive: write(" %s in %s (pass)" % (result.case.GetName(), result.case.GetMode())) write("Succeeded Tests End") if len(negative) > 0: - print + print() write("Test262 Expected to fail") for result in negative: write(" %s in %s (pass)" % (result.case.GetName(), result.case.GetMode())) @@ -673,13 +701,13 @@ def write(s): positive = [c for c in progress.failed_tests if not c.case.IsNegative()] negative = [c for c in progress.failed_tests if c.case.IsNegative()] if len(positive) > 0: - print + print() write("Test262 Failed Tests") for result in positive: write(" %s in %s (fail)" % (result.case.GetName(), result.case.GetMode())) write("Failed Tests End") if len(negative) > 0: - print + print() write("Test262 Expected to fail but passed") for result in negative: write(" %s in %s (fail)" % (result.case.GetName(), result.case.GetMode())) @@ -689,7 +717,7 @@ def PrintFailureOutput(self, progress, logfile): for result in progress.failed_tests: if logfile: self.WriteLog(result) - print + print() result.ReportOutcome(False) def Run(self, command_template, tests, print_summary, print_full, logname, junitfile): @@ -700,20 +728,20 @@ def Run(self, command_template, tests, print_summary, print_full, logname, junit ReportError("No tests to run") progress = ProgressIndicator(len(cases)) if logname: - self.logf = open(logname, "w") + self.logf = open(logname, "w", encoding="utf-8") if junitfile: - self.outfile = open(junitfile, "w") + self.outfile = open(junitfile, "w", encoding="utf-8") TestSuitesElement = xmlj.Element("testsuites") TestSuiteElement = xmlj.Element("testsuite") TestSuitesElement.append(TestSuiteElement) TestSuiteElement.attrib["name "] = "test262" for x in range(len(EXCLUDE_LIST)): - if self.ShouldRun (unicode(EXCLUDE_LIST[x].encode('utf-8','ignore')), tests): + if self.ShouldRun (str(EXCLUDE_LIST[x].encode('utf-8','ignore')), tests): SkipCaseElement = xmlj.Element("testcase") - SkipCaseElement.attrib["classname"] = unicode(EXCLUDE_LIST[x]).encode('utf-8','ignore') - SkipCaseElement.attrib["name"] = unicode(EXCLUDE_LIST[x]).encode('utf-8','ignore') + SkipCaseElement.attrib["classname"] = str(EXCLUDE_LIST[x]).encode('utf-8','ignore') + SkipCaseElement.attrib["name"] = str(EXCLUDE_LIST[x]).encode('utf-8','ignore') SkipElement = xmlj.Element("skipped") - SkipElement.attrib["message"] = unicode(EXCLUDE_REASON[x].firstChild.nodeValue) + SkipElement.attrib["message"] = str(EXCLUDE_REASON[x].firstChild.nodeValue) SkipCaseElement.append(SkipElement) TestSuiteElement.append(SkipCaseElement) @@ -769,7 +797,7 @@ def Run(self, command_template, tests, print_summary, print_full, logname, junit print print "Use --full-summary to see output from failed tests" ''' - print + print() return progress.failed def WriteLog(self, result): @@ -801,7 +829,7 @@ def ListIncludes(self, tests): includes = case.GetIncludeList() includes_dict.update(includes) - print includes_dict + print(includes_dict) def Main(): @@ -813,7 +841,8 @@ def Main(): options.strict_only, options.non_strict_only, options.unmarked_default, - options.print_handle) + options.print_handle, + options.skip) test_suite.Validate() if options.loglevel == 'debug': logging.basicConfig(level=logging.DEBUG) @@ -841,6 +870,6 @@ def Main(): try: code = Main() sys.exit(code) - except Test262Error, e: - print "Error: %s" % e.message + except Test262Error as e: + print("Error: %s" % e.message) sys.exit(1)