From 582bbf639f8e8f2b41deed87fa45f6aaafd29917 Mon Sep 17 00:00:00 2001 From: Guillaume ARM Date: Mon, 8 Jun 2026 04:52:19 +0200 Subject: [PATCH] test(craftos): report verbose case statuses --- DEVELOPMENT.md | 2 +- Justfile | 35 ++++- apis/libtest.lua | 132 ++++++++++-------- docs/adrs/adr-0005-craftos-pc-harness.md | 4 +- .../adr-0007-use-libtest-for-craftos-tests.md | 2 +- docs/install-craftos-pc.md | 2 +- 6 files changed, 109 insertions(+), 68 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 2bf575b..b453caf 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -16,6 +16,6 @@ This creates `.env` from `.env.sample` when needed and installs the local Git ho `just ci` is the local verification entry point. Today it verifies that `craftos --version` reports v2.8.3 or newer, runs `just check` for `luacheck`, and runs `just test` for CraftOS-PC headless tests. Use `just craftos` to launch CraftOS-PC with repo-local save data under `.craftos` and read-only mounts for `/trapos`, `/apis`, `/programs`, `/servers`, and `/startup`. `just repl` opens the same environment with `--cli` for human interactive use only; LLM agents must not run it. Use the CraftOS-PC glossary when adjusting `--headless`, `--exec`, `--script`, `--rom`, or `--mount-*` usage. -Tests live under `tests/` and run inside CraftOS-PC through `just test`. API-level tests should use `require('/apis/libtest')({ ... })`, register cases with `testlib.test(name, fn)`, and call `testlib.run()` at the end. `libtest` prints `__TRAPOS_TEST_OK__` only when every case passes, which is the contract consumed by the shell harness. Pass `--verbose` to `just test` to list each test script; `libtest` also accepts `--verbose` and prints each case when script stdout is inspected. +Tests live under `tests/` and run inside CraftOS-PC through `just test`. API-level tests should use `require('/apis/libtest')({ ... })`, register cases with `testlib.test(name, fn)`, and call `testlib.run()` at the end. `libtest` prints `__TRAPOS_TEST_OK__` only when every case passes, which is the contract consumed by the shell harness. Pass `--verbose` to `just test` to list each test script and each `libtest` case with colored `[OK]`/`[KO]` statuses. Each CraftOS-PC test process is killed if it does not finish within `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`, defaulting to `3`. Override this in `.env` for slower local probes. diff --git a/Justfile b/Justfile index 0e2fa8c..72c516c 100644 --- a/Justfile +++ b/Justfile @@ -101,26 +101,27 @@ repl: ci *args: check-craftos check @just test {{args}} -# Run CraftOS-PC headless smoke tests. Pass `--verbose` to list each test. +# Run CraftOS-PC headless integration tests. Pass `--verbose` to list each case. test *args: @if [ -f .env ]; then set -a; . ./.env; set +a; fi; \ verbose=0; \ timeout_seconds="${TRAP_CCLIBS_TEST_TIMEOUT_SECONDS:-3}"; \ case "$timeout_seconds" in ''|*[!0-9]*) printf '%s\n' 'TRAP_CCLIBS_TEST_TIMEOUT_SECONDS must be a positive integer' >&2; exit 1 ;; esac; \ if [ "$timeout_seconds" -lt 1 ]; then printf '%s\n' 'TRAP_CCLIBS_TEST_TIMEOUT_SECONDS must be >= 1' >&2; exit 1; fi; \ - for a in {{args}}; do [ "$a" = "--verbose" ] && verbose=1; done; \ - script_args=""; \ - if [ "$verbose" -eq 1 ]; then script_args="--verbose"; fi; \ + for a in {{args}}; do case "$a" in --verbose|-v) verbose=1 ;; esac; done; \ rom_arg=""; \ if [ "$(uname -s)" = "Darwin" ]; then \ rom_arg="--rom /Applications/CraftOS-PC.app/Contents/Resources"; \ fi; \ repo='{{justfile_directory()}}'; \ - mount_arg="--mount-ro /apis=$repo/apis"; \ + mount_arg="--mount-ro /apis=$repo/apis --mount-ro /tests=$repo/tests"; \ green=$(printf '\033[32m'); reset=$(printf '\033[0m'); red=$(printf '\033[31m'); \ for script in tests/boot.lua tests/ready.lua tests/eventloop.lua; do \ tmp="$(mktemp)"; \ - craftos --headless $rom_arg $mount_arg --script "$script" $script_args >"$tmp" 2>&1 & \ + data_dir="$(mktemp -d)"; \ + exec_code="shell.run('/$script')"; \ + if [ "$verbose" -eq 1 ]; then exec_code="shell.run('/$script', '--verbose')"; fi; \ + craftos --directory "$data_dir" --headless $rom_arg $mount_arg --exec "$exec_code" >"$tmp" 2>&1 & \ pid="$!"; \ ( sleep "$timeout_seconds"; kill -TERM "$pid" >/dev/null 2>&1 ) & \ watchdog="$!"; \ @@ -129,7 +130,17 @@ test *args: kill "$watchdog" >/dev/null 2>&1 || true; \ wait "$watchdog" >/dev/null 2>&1 || true; \ if grep -q __TRAPOS_TEST_OK__ "$tmp"; then \ + if [ "$verbose" -eq 1 ]; then \ + report="$data_dir/computer/0/trapos-test-report"; \ + if [ -f "$report" ]; then while IFS= read -r line; do \ + case "$line" in \ + 'OK '*) name="${line#OK }"; printf '%s\n' "${green}[OK]${reset} $name" ;; \ + 'KO '*) name="${line#KO }"; printf '%s\n' "${red}[KO]${reset} $name" ;; \ + esac; \ + done <"$report"; fi; \ + fi; \ rm -f "$tmp"; \ + rm -rf "$data_dir"; \ [ "$verbose" -eq 1 ] && printf '%s\n' "${green}PASS${reset} $script"; \ else \ if [ "$status" -eq 143 ]; then \ @@ -137,12 +148,22 @@ test *args: else \ printf '%s\n' "${red}FAIL${reset} $script did not print __TRAPOS_TEST_OK__" >&2; \ fi; \ + if [ "$verbose" -eq 1 ]; then \ + report="$data_dir/computer/0/trapos-test-report"; \ + if [ -f "$report" ]; then while IFS= read -r line; do \ + case "$line" in \ + 'OK '*) name="${line#OK }"; printf '%s\n' "${green}[OK]${reset} $name" >&2 ;; \ + 'KO '*) name="${line#KO }"; printf '%s\n' "${red}[KO]${reset} $name" >&2 ;; \ + esac; \ + done <"$report"; fi; \ + fi; \ cat "$tmp" >&2; \ rm -f "$tmp"; \ + rm -rf "$data_dir"; \ exit 1; \ fi; \ done; \ - printf '%s\n' 'OK: smoke tests passed' + printf '%s\n' 'OK: CraftOS integration tests passed' # Lint all Lua source with luacheck. check: check-luacheck diff --git a/apis/libtest.lua b/apis/libtest.lua index 2abe024..efd1caa 100644 --- a/apis/libtest.lua +++ b/apis/libtest.lua @@ -1,72 +1,92 @@ -local _VERSION = '1.1.0'; +local _VERSION = "1.3.0" local function createLibTest(args) - local api = {}; - local tests = {}; - local verbose = false; + local api = {} + local tests = {} + local verbose = false + local reportPath = "/trapos-test-report" - for _, arg in ipairs(args or {}) do - if arg == '--verbose' then - verbose = true; - end - end + for _, arg in ipairs(args or {}) do + if arg == "--verbose" then + verbose = true + end + end - local function fail(message) - error(message, 2); - end + local function fail(message) + error(message, 2) + end - function api.test(name, fn) - assert(type(name) == 'string', 'bad argument #1 (string expected)'); - assert(type(fn) == 'function', 'bad argument #2 (function expected)'); + local function writeReport(line) + if not verbose then + return + end - tests[#tests + 1] = { name = name, fn = fn }; - end + local file = fs.open(reportPath, "a") + if file then + file.writeLine(line) + file.close() + end + end - function api.assertEquals(actual, expected, message) - if actual ~= expected then - fail((message or 'assertEquals failed') .. ': expected ' .. tostring(expected) .. ', got ' .. tostring(actual)); - end - end + function api.test(name, fn) + assert(type(name) == "string", "bad argument #1 (string expected)") + assert(type(fn) == "function", "bad argument #2 (function expected)") - function api.assertTrue(value, message) - if not value then - fail(message or 'assertTrue failed'); - end - end + tests[#tests + 1] = { name = name, fn = fn } + end - function api.assertErrors(fn, expected) - assert(type(fn) == 'function', 'bad argument #1 (function expected)'); + function api.assertEquals(actual, expected, message) + if actual ~= expected then + fail( + (message or "assertEquals failed") + .. ": expected " + .. tostring(expected) + .. ", got " + .. tostring(actual) + ) + end + end - local ok, err = pcall(fn); - if ok then - fail('expected error'); - end - if expected and not string.find(tostring(err), expected, 1, true) then - fail('expected error containing ' .. expected .. ', got ' .. tostring(err)); - end - end + function api.assertTrue(value, message) + if not value then + fail(message or "assertTrue failed") + end + end - function api.run() - for _, t in ipairs(tests) do - if verbose then - print('RUN ' .. t.name); - end - local ok, err = pcall(t.fn); - if not ok then - print('FAIL ' .. t.name .. ': ' .. tostring(err)); - os.shutdown(); - end - end + function api.assertErrors(fn, expected) + assert(type(fn) == "function", "bad argument #1 (function expected)") - print('__TRAPOS_TEST_OK__'); - os.shutdown(); - end + local ok, err = pcall(fn) + if ok then + fail("expected error") + end + if expected and not string.find(tostring(err), expected, 1, true) then + fail("expected error containing " .. expected .. ", got " .. tostring(err)) + end + end - function api.version() - return _VERSION; - end + function api.run() + for _, t in ipairs(tests) do + local ok, err = pcall(t.fn) + if not ok then + writeReport("KO " .. t.name .. ": " .. tostring(err)) + if not verbose then + print("FAIL " .. t.name .. ": " .. tostring(err)) + end + os.shutdown() + end + writeReport("OK " .. t.name) + end - return api; + print("__TRAPOS_TEST_OK__") + os.shutdown() + end + + function api.version() + return _VERSION + end + + return api end -return createLibTest; +return createLibTest diff --git a/docs/adrs/adr-0005-craftos-pc-harness.md b/docs/adrs/adr-0005-craftos-pc-harness.md index 4ee702b..3e74155 100644 --- a/docs/adrs/adr-0005-craftos-pc-harness.md +++ b/docs/adrs/adr-0005-craftos-pc-harness.md @@ -28,7 +28,7 @@ Treat CraftOS-PC as a first-class local development dependency. - **Documented upstream navigation** in [`docs/craftos_pc_glossary.md`](../craftos_pc_glossary.md), covering CLI flags, mounts, `periphemu`, save data, and troubleshooting pages. - **Verified by `just install`** via `check-install`, which checks `craftos`, `jq`, and `luacheck`. `check-craftos` runs `craftos --version` and requires v2.8.3 or newer. Failure prints a one-line pointer to the install guide instead of a long stack trace. - **Repository-local launch recipe.** `just craftos` runs CraftOS-PC with `--directory .craftos`, keeps the macOS `--rom /Applications/CraftOS-PC.app/Contents/Resources` workaround, mounts the repository root read-only at `/trapos`, and mounts each manifest top-level directory read-only at its ComputerCraft root path. -- **`just ci` is the local verification entry point.** Today it runs `check-craftos`, `check`, and `test`. The installed pre-commit hook invokes `just ci` directly, and `just test` owns CraftOS-PC-driven smoke tests. +- **`just ci` is the local verification entry point.** Today it runs `check-craftos`, `check`, and `test`. The installed pre-commit hook invokes `just ci` directly, and `just test` owns CraftOS-PC-driven integration tests. The existing `CLAUDE.md` constraint ("Do not run Lua locally or add a test harness unless asked") is reframed rather than removed: there is still no standalone Lua harness, and we are not adding a Busted-style test runner. The harness *is* CraftOS-PC, invoked deliberately. @@ -36,7 +36,7 @@ The existing `CLAUDE.md` constraint ("Do not run Lua locally or add a test harne - Contributors must install CraftOS-PC before `just install` succeeds. The install guide makes this a 4-step copy/paste on macOS. - Contributors should use the local CraftOS-PC glossary first when researching emulator behavior, then follow the linked upstream pages for details. -- Headless tests live under `tests/` and are driven by `just test`. Smoke tests still prove CraftOS-PC boots and the event queue works, and API behavior tests use `/apis/libtest.lua` for named cases and assertions. Each script is invoked as `craftos --headless --script ` and its stdout is grepped for `__TRAPOS_TEST_OK__`. Adding another test means dropping a Lua file in `tests/` and adding it to the loop in the `test:` recipe. +- Headless tests live under `tests/` and are driven by `just test`. Basic boot tests still prove CraftOS-PC boots and the event queue works, and API behavior tests use `/apis/libtest.lua` for named cases and assertions. Tests are mounted into CraftOS-PC and invoked with `shell.run`, and stdout is grepped for `__TRAPOS_TEST_OK__`. Adding another test means dropping a Lua file in `tests/` and adding it to the loop in the `test:` recipe. - Each test process is guarded by `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`, defaulting to `3`, so a blocked ComputerCraft event loop fails quickly and prints captured output. - The macOS install symlinks the binary into `/usr/local/bin`, which makes CraftOS-PC unable to auto-discover the ROM that ships inside the `.app` bundle (`Could not mount ROM`). The `test:` recipe works around this by passing `--rom /Applications/CraftOS-PC.app/Contents/Resources` on Darwin. Linux (AppImage) and Windows (installer) auto-discover correctly, so no flag is passed there. - `just craftos` uses repository-local save data under `.craftos/config/` and `.craftos/computer/`. This keeps emulator state out of `~/Library/Application Support/CraftOS-PC` during repository work and keeps repo files visible through read-only mounts instead of copying them into the VM save. diff --git a/docs/adrs/adr-0007-use-libtest-for-craftos-tests.md b/docs/adrs/adr-0007-use-libtest-for-craftos-tests.md index dc73b4b..45639e8 100644 --- a/docs/adrs/adr-0007-use-libtest-for-craftos-tests.md +++ b/docs/adrs/adr-0007-use-libtest-for-craftos-tests.md @@ -37,7 +37,7 @@ end); testlib.run(); ``` -`libtest` intentionally stays small. It provides named cases, `assertEquals`, `assertTrue`, `assertErrors`, verbose `RUN ` output, failure reporting, the `__TRAPOS_TEST_OK__` success marker, and `os.shutdown()` at process end. +`libtest` intentionally stays small. It provides named cases, `assertEquals`, `assertTrue`, `assertErrors`, verbose case-status report output consumed by the shell harness, failure reporting, the `__TRAPOS_TEST_OK__` success marker, and `os.shutdown()` at process end. The shell harness keeps ownership of process-level concerns: CraftOS-PC launch flags, read-only mounts, stdout capture, and timeout enforcement through `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`. diff --git a/docs/install-craftos-pc.md b/docs/install-craftos-pc.md index f331c0f..2d46bd9 100644 --- a/docs/install-craftos-pc.md +++ b/docs/install-craftos-pc.md @@ -82,7 +82,7 @@ On macOS, use the `--rom` form shown above if the command fails with `Could not ## Running tests -`just test` runs the headless smoke tests in `tests/` through CraftOS-PC. On macOS the recipe passes `--rom /Applications/CraftOS-PC.app/Contents/Resources` because the `/usr/local/bin/craftos` symlink loses ROM auto-discovery; on Linux and Windows no flag is needed. `just ci` runs the same tests after `luacheck`. +`just test` runs the headless CraftOS integration tests in `tests/` through CraftOS-PC. On macOS the recipe passes `--rom /Applications/CraftOS-PC.app/Contents/Resources` because the `/usr/local/bin/craftos` symlink loses ROM auto-discovery; on Linux and Windows no flag is needed. `just ci` runs the same tests after `luacheck`. ## Repository-local launches