Keep adapters independent

This commit removes test runner imports from the adapters, allowing
them to be developed independently. Additionally this commit
introduces the concept of `proposals` in the configuration, which
are passed to the configuration in the form of `List[str]`, through
the `proposals`  the adapters can derive the specific flags needed to
run the test to completion.

As a last change, this commit also migrates the http-response test to
use the new test configuration using the `http` proposal. Without
this change it is not straight forward to derive "base proposals" for
tests with the legacy configuration that wish to use specific proposals. We
could certainly introduce other heuristics, at the  cost of more
complexity and seemingly not many advantages.

Author: saulecabrera

adapters/pywasm.py

@@ -7,12 +7,6 @@
 import importlib
 
 
-_test_runner_path = Path(__file__).parent.parent / "test-runner"
-if str(_test_runner_path) not in sys.path:
-    sys.path.insert(0, str(_test_runner_path))
-
-_test_case_module = importlib.import_module('wasi_test_runner.test_case')
-
 # shlex.split() splits according to shell quoting rules
 RUN_PYWASM = Path(__file__).parent.parent / "tools" / "run-pywasm"
 
@@ -34,17 +28,21 @@ def get_wasi_versions() -> List[str]:
     return ["wasm32-wasip1"]
 
 def compute_argv(test_path: str,
-                 config: _test_case_module.Config,
+                 args_env_dirs: Tuple[List[str], Dict[str, str], List[Tuple[Path, str]]],
+                 proposals: List[str],
                  wasi_version: str) -> List[str]:
+
     argv = []
-    for op in config.operations:
-        match op:
-            case _test_case_module.Run(args, env, dirs):
-                argv += [str(RUN_PYWASM)]
-                for k, v in env.items():
-                    argv += ["--env", f"{k}={v}"]
-                for host, guest in dirs:
-                    argv += ["--dir", f"{host}::{guest}"]  # noqa: E231
-                argv += [test_path]
-                argv += args
+    argv += [str(RUN_PYWASM)]
+    args, env, dirs = args_env_dirs
+
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+
+    for host, guest in dirs:
+        argv += ["--dir", f"{host}::{guest}"]  # noqa: E231
+
+    argv += [test_path]
+
+    argv += args
     return argv

adapters/wasm-micro-runtime.py

@@ -7,12 +7,6 @@
 import importlib
 
 
-_test_runner_path = Path(__file__).parent.parent / "test-runner"
-if str(_test_runner_path) not in sys.path:
-    sys.path.insert(0, str(_test_runner_path))
-
-_test_case_module = importlib.import_module('wasi_test_runner.test_case')
-
 # shlex.split() splits according to shell quoting rules
 IWASM = shlex.split(os.getenv("IWASM", "iwasm"))
 
@@ -35,17 +29,21 @@ def get_wasi_versions() -> List[str]:
 
 
 def compute_argv(test_path: str,
-                 config: _test_case_module.Config,
+                 args_env_dirs: Tuple[List[str], Dict[str, str], List[Tuple[Path, str]]],
+                 proposals: List[str],
                  wasi_version: str) -> List[str]:
+
     argv = []
-    for op in config.operations:
-        match op:
-            case _test_case_module.Run(args, env, dirs):
-                argv += [IWASM]
-                for k, v in env.items():
-                    argv += ["--env", f"{k}={v}"]
-                for host, guest in dirs:
-                    argv += ["--map-dir", f"{host}::{guest}"]  # noqa: E231
-                argv += [test_path]
-                argv += args
+    argv += IWASM
+    args, env, dirs = args_env_dirs
+
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+
+    for host, guest in dirs:
+        argv += ["--map-dir", f"{host}::{guest}"]  # noqa: E231
+
+    argv += [test_path]
+
+    argv += args
     return argv

adapters/wasmedge.py

@@ -7,12 +7,6 @@
 import importlib
 
 
-_test_runner_path = Path(__file__).parent.parent / "test-runner"
-if str(_test_runner_path) not in sys.path:
-    sys.path.insert(0, str(_test_runner_path))
-
-_test_case_module = importlib.import_module('wasi_test_runner.test_case')
-
 # shlex.split() splits according to shell quoting rules
 WASMEDGE = shlex.split(os.getenv("WASMEDGE", "wasmedge"))
 
@@ -35,17 +29,21 @@ def get_wasi_versions() -> List[str]:
 
 
 def compute_argv(test_path: str,
-                 config: _test_case_module.Config,
+                 args_env_dirs: Tuple[List[str], Dict[str, str], List[Tuple[Path, str]]],
+                 proposals: List[str],
                  wasi_version: str) -> List[str]:
+
     argv = []
-    for op in config.operations:
-        match op:
-            case _test_case_module.Run(args, env, dirs):
-                argv += [WASMEDGE]
-                for k, v in env.items():
-                    argv += ["--env", f"{k}={v}"]
-                for host, guest in dirs:
-                    argv += ["--dir", f"{guest}:{host}"]
-                argv += [test_path]
-                argv += args
+    argv += WASMEDGE
+    args, env, dirs = args_env_dirs
+
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+
+    for host, guest in dirs:
+        argv += ["--dir", f"{guest}:{host}"]
+
+    argv += [test_path]
+
+    argv += args
     return argv

adapters/wasmtime.py

@@ -7,12 +7,6 @@
 import importlib
 
 
-_test_runner_path = Path(__file__).parent.parent / "test-runner"
-if str(_test_runner_path) not in sys.path:
-    sys.path.insert(0, str(_test_runner_path))
-
-_test_case_module = importlib.import_module('wasi_test_runner.test_case')
-
 # shlex.split() splits according to shell quoting rules
 WASMTIME = shlex.split(os.getenv("WASMTIME", "wasmtime"))
 
@@ -35,38 +29,46 @@ def get_wasi_versions() -> List[str]:
 
 
 def compute_argv(test_path: str,
-                 config: _test_case_module.Config,
+                 args_env_dirs: Tuple[List[str], Dict[str, str], List[Tuple[Path, str]]],
+                 proposals: List[str],
                  wasi_version: str) -> List[str]:
 
     argv = []
-    for op in config.operations:
-        match op:
-            case _test_case_module.Run(args, env, dirs):
-                argv += WASMTIME
-                for k, v in env.items():
-                    argv += ["--env", f"{k}={v}"]
-                for host, guest in dirs:
-                    argv += ["--dir", f"{host}::{guest}"]  # noqa: E231
-                argv += [test_path]
-                argv += args
-                _add_wasi_version_options(argv, wasi_version)
-            case _:
-                pass
+    argv += WASMTIME
+    args, env, dirs = args_env_dirs
+
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+
+    for host, guest in dirs:
+        argv += ["--dir", f"{host}::{guest}"]  # noqa: E231
+
+    argv += [test_path]
+
+    argv += args
+    _add_wasi_version_options(argv, wasi_version, proposals)
     return argv
 
 
 # The user might provide WASMTIME="wasmtime --option -Sfoo".  Let's
 # insert the options to choose the WASI version before the user's
 # options, so that the user can override our choices.
-def _add_wasi_version_options(argv: List[str], wasi_version: str) -> None:
+def _add_wasi_version_options(argv: List[str], wasi_version: str, proposals: List[str]) -> None:
     splice_pos = len(WASMTIME)
     while splice_pos > 1 and args[splice_pos-1].startswith("-"):
         splice_pos -= 1
     match wasi_version:
         case "wasm32-wasip1":
             pass
         case "wasm32-wasip3":
+            flags_from_proposals = ""
+            if "http" in proposals:
+                flags_from_proposals += ",http"
+            if "sockets" in proposals:
+                flags_from_proposals += ",inherit-network"
+
             argv[splice_pos:splice_pos] = ["-Wcomponent-model-async",
-                                           "-Sp3,http,inherit-network"]
+                                           f"-Sp3{flags_from_proposals}"]
+
         case _:
             pass

adapters/wazero.py

@@ -7,12 +7,6 @@
 import importlib
 
 
-_test_runner_path = Path(__file__).parent.parent / "test-runner"
-if str(_test_runner_path) not in sys.path:
-    sys.path.insert(0, str(_test_runner_path))
-
-_test_case_module = importlib.import_module('wasi_test_runner.test_case')
-
 # shlex.split() splits according to shell quoting rules
 WAZERO = shlex.split(os.getenv("WAZERO", "wazero"))
 
@@ -37,17 +31,22 @@ def get_wasi_versions() -> List[str]:
 
 
 def compute_argv(test_path: str,
-                 config: _test_case_module.Config,
+                 args_env_dirs: Tuple[List[str], Dict[str, str], List[Tuple[Path, str]]],
+                 proposals: List[str],
                  wasi_version: str) -> List[str]:
+
     argv = []
-    for op in config.operations:
-        match op:
-            case _test_case_module.Run(args, env, dirs):
-                argv += [WAZERO, "run", "-hostlogging=filesystem"]
-                for k, v in env.items():
-                    argv += [f"-env={k}={v}"]
-                for host, guest in dirs:
-                    argv += [f"-mount={host}:{guest}"]
-                argv += [test_path]
-                argv += args
+    argv += WAZERO
+    argv += ["run", "-hostlogging=filesystem"]
+    args, env, dirs = args_env_dirs
+
+    for k, v in env.items():
+        argv += [f"-env={k}={v}"]
+
+    for host, guest in dirs:
+        argv += [f"-mount={host}:{guest}"]
+
+    argv += [test_path]
+
+    argv += args
     return argv

adapters/wizard.py

@@ -7,12 +7,6 @@
 import importlib
 
 
-_test_runner_path = Path(__file__).parent.parent / "test-runner"
-if str(_test_runner_path) not in sys.path:
-    sys.path.insert(0, str(_test_runner_path))
-
-_test_case_module = importlib.import_module('wasi_test_runner.test_case')
-
 # shlex.split() splits according to shell quoting rules
 WIZARD = shlex.split(os.getenv("WIZARD", "wizeng.x86-64-linux"))
 
@@ -43,18 +37,22 @@ def get_wasi_versions() -> List[str]:
 
 
 def compute_argv(test_path: str,
-                 config: _test_case_module.Config,
+                 args_env_dirs: Tuple[List[str], Dict[str, str], List[Tuple[Path, str]]],
+                 proposals: List[str],
                  wasi_version: str) -> List[str]:
+
     argv = []
-    for op in config.operations:
-        match op:
-            case _test_case_module.Run(args, env, dirs):
-                argv += [WIZARD]
-                for k, v in env.items():
-                    argv += [f"--env={k}={v}"]
-                for host, guest in dirs:
-                    # FIXME: https://github.com/titzer/wizard-engine/issues/482
-                    argv += [f"--dir={host}"]
-                argv += [test_path]
-                argv += args
+    argv += WIZARD
+    args, env, dirs = args_env_dirs
+
+    for k, v in env.items():
+        argv += [f"--env={k}={v}"]
+
+    for host, guest in dirs:
+        # FIXME: https://github.com/titzer/wizard-engine/issues/482
+        argv += [f"--dir={host}"]
+
+    argv += [test_path]
+
+    argv += args
     return argv

test-runner/tests/test_test_case.py

@@ -5,7 +5,7 @@
 import pytest
 
 from wasi_test_runner.test_case import (
-    Config, Failure, Result, Run, Wait, Read, Connect, Send, Recv, ProtocolType
+    Config, Failure, Result, Run, Wait, Read, Connect, Send, Recv, ProtocolType, WasiProposal
 )
 
 
@@ -69,13 +69,13 @@ def test_test_config_should_fail_when_mixing_config_styles(_mock_file: Mock) ->
 
 
 def test_test_results_should_mark_failed_if_multiple_failures() -> None:
-    results = Result([], True, [Failure("type", "message")])
+    results = Result(True, [Failure("type", "message")])
 
     assert results.failed is True
 
 
 def test_test_results_should_not_mark_failed_if_no_failure() -> None:
-    results = Result([], True, [])
+    results = Result(True, [])
 
     assert results.failed is False
 
@@ -179,3 +179,45 @@ def test_recv_from_config_with_default_payload() -> None:
 
     assert recv.id == "conn1"
     assert recv.payload == ""
+
+
+@patch("builtins.open", new_callable=mock_open, read_data="{}")
+def test_legacy_config_should_have_http_proposal_by_default(_mock_file: Mock) -> None:
+    config = Config.from_file("file")
+
+    assert len(config.proposals) == 1
+    assert config.proposals[0] == WasiProposal.HTTP
+
+
+@patch(
+    "builtins.open",
+    new_callable=mock_open,
+    read_data='{"operations": [{"type": "run"}, {"type": "wait"}], "proposals": []}',
+)
+def test_new_config_with_empty_proposals(_mock_file: Mock) -> None:
+    config = Config.from_file("file")
+
+    assert len(config.proposals) == 0
+
+
+@patch(
+    "builtins.open",
+    new_callable=mock_open,
+    read_data='{"operations": [{"type": "run"}], "proposals": ["http", "sockets"]}',
+)
+def test_new_config_with_multiple_proposals(_mock_file: Mock) -> None:
+    config = Config.from_file("file")
+
+    assert len(config.proposals) == 2
+    assert config.proposals[0] == WasiProposal.HTTP
+    assert config.proposals[1] == WasiProposal.SOCKETS
+
+
+@patch(
+    "builtins.open",
+    new_callable=mock_open,
+    read_data='{"operations": [{"type": "run"}], "proposals": ["invalid"]}',
+)
+def test_new_config_should_fail_with_invalid_proposal(_mock_file: Mock) -> None:
+    with pytest.raises(ValueError):
+        Config.from_file("file")

test-runner/tests/test_test_suite.py

@@ -11,7 +11,7 @@ def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCa
         name,
         ["test-runtime-exe", name],
         tc.Config(),
-        tc.Result(["foo", "bar"], is_executed, failures),
+        tc.Result(is_executed, failures),
         1.0,
     )