Feat(kclvm-windows): Add konfig test for windows. (#414)

* Fix(kclvm-windows): rm all the '\\?\' path prefix in windows.
issue #379.

* Feat(kclvm-windows): add konfig test for kclvm-windows.
issue #379.

* add windows ci

* fix typo

Author: zong-zhe

.github/workflows/windows_test.yaml

@@ -26,7 +26,23 @@ jobs:
 
       - run: echo "C:/LLVM/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
-      - run: .\build_and_test_win.bat
+      # Build kclvm-cli
+      - run: .\scripts\build-windows\build.bat
+
+      # Set kclvm-cli path
+      - run: echo ";$(pwd)\scripts\build-windows\_output\kclvm-windows\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+        working-directory: .
+
+      # Rust unit test
+      - run: cargo test -p kclvm-*
+        working-directory: ./kclvm
+
+      # Rust runtime test
+      - run: python3 -m pip install --upgrade pip && python3 -m pip install pytest && kclvm -m pytest -vv
+        working-directory: ./kclvm/tests/test_units
+
+      # Rust konfig test
+      - run: .\test\integration\test_konfig.bat
         working-directory: .
 
       - uses: actions/upload-artifact@v3

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "plugins"]
 	path = plugins
 	url = https://github.com/KusionStack/kcl-plugin
-[submodule "test/konfig"]
-	path = test/konfig
+[submodule "test/integration/konfig"]
+	path = test/integration/konfig
 	url = https://github.com/KusionStack/konfig.git

build_and_test_win.bat

@@ -13,3 +13,7 @@ cd %~dp0
 @REM rust runtime test
 cd .\\kclvm\\tests\\test_units
 kclvm -m pytest -vv
+cd %~dp0
+
+@REM konfig test
+call .\\test\\integration\\test_konfig.bat

kclvm/runner/src/tests.rs

@@ -80,6 +80,7 @@ fn multi_file_test_cases() -> Vec<String> {
             .join("..")
             .join("..")
             .join("test")
+            .join("integration")
             .join("konfig")
             .join("base")
             .join("examples")

test/integration/konfig

@@ -0,0 +1,5 @@
+cd %~dp0
+
+python3 -m pip install --upgrade pip
+python3 -m pip install pytest pytest-xdist
+python3 -m pytest -vv -n 10

test/integration/test_konfig.bat

@@ -0,0 +1,110 @@
+"""
+this testing framework is developed based on pytest.
+see quick start of pytest: https://docs.pytest.org/en/latest/example/simple.html
+
+"""
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+from ruamel.yaml import YAML
+from collections.abc import Mapping, Sequence
+
+TEST_FILE = "kcl.yaml"
+CI_TEST_DIR = "ci-test"
+STDOUT_GOLDEN = "stdout.golden.yaml"
+SETTINGS_FILE = "settings.yaml"
+
+ROOT_STR = "konfig"
+ROOT = str(Path(__file__).parent.joinpath(ROOT_STR))
+
+yaml = YAML(typ="unsafe", pure=True)
+
+
+def find_test_dirs():
+    result = []
+    root_dirs = [ROOT]
+    for root_dir in root_dirs:
+        for root, _, files in os.walk(root_dir):
+            for name in files:
+                if name == TEST_FILE:
+                    result.append(root)
+    return result
+
+
+def compare_results(result, golden_result):
+    # Convert result and golden_result string to string lines with line ending stripped, then compare.
+
+    assert compare_unordered_yaml_objects(list(yaml.load_all(result)), list(yaml.load_all(golden_result)))
+
+# Comparing the contents of two YAML objects for equality in an unordered manner
+def compare_unordered_yaml_objects(result, golden_result):
+    if isinstance(result, Mapping) and isinstance(golden_result, Mapping):
+        if result.keys() != golden_result.keys():
+            return False
+        for key in result.keys():
+            if not compare_unordered_yaml_objects(result[key], golden_result[key]):
+                return False
+
+        return True
+    elif isinstance(result, Sequence) and isinstance(golden_result, Sequence):
+        if len(result) != len(golden_result):
+            return False
+        for item in result:
+            if item not in golden_result:
+                return False
+        for item in golden_result:
+            if item not in result:
+                return False
+        return True
+    else:
+        return result == golden_result
+
+def has_settings_file(directory):
+    settings_file = directory / SETTINGS_FILE
+    return settings_file.is_file()
+
+
+print("##### K Language Grammar Test Suite #####")
+test_dirs = find_test_dirs()
+pwd = str(Path(__file__).parent.parent.parent)
+os.environ["PYTHONPATH"] = pwd
+
+
+@pytest.mark.parametrize("test_dir", test_dirs)
+def test_konfigs(test_dir):
+    print(f"Testing {test_dir}")
+    test_dir = Path(test_dir)
+    kcl_file_name = test_dir / TEST_FILE
+    ci_test_dir = test_dir / CI_TEST_DIR
+    if not ci_test_dir.is_dir():
+        # Skip invalid test cases
+        return
+    golden_file = ci_test_dir / STDOUT_GOLDEN
+    if not golden_file.is_file():
+        # Skip invalid test cases
+        return
+    kcl_command = ["kcl"]
+    if has_settings_file(ci_test_dir):
+        kcl_command.append("-Y")
+        kcl_command.append(f"{CI_TEST_DIR}/{SETTINGS_FILE}")
+        kcl_command.append(f"kcl.yaml")
+    else:
+        kcl_command.append(f"{TEST_FILE}")
+    kcl_command.append("--target")
+    kcl_command.append("native")
+    process = subprocess.run(
+        kcl_command, capture_output=True, cwd=test_dir, env=dict(os.environ)
+    )
+    stdout, stderr = process.stdout, process.stderr
+    print(f"STDOUT:\n{stdout.decode()}")
+    assert (
+        process.returncode == 0 and len(stderr) == 0
+    ), f"Error executing file {kcl_file_name}.\nexit code = {process.returncode}\nstderr = {stderr}"
+    if process.returncode == 0 and len(stderr) == 0:
+        try:
+            with open(golden_file, "r") as golden:
+                compare_results(stdout.decode(), golden)
+        except FileNotFoundError:
+            raise Exception(f"Error reading expected result from file {golden_file}")