test(craftos): report verbose case statuses
This commit is contained in:
Guillaume ARM
parent
1010e0d844
commit
582bbf639f
@ -16,6 +16,6 @@ This creates `.env` from `.env.sample` when needed and installs the local Git ho
|
||||
|
||||
`just ci` is the local verification entry point. Today it verifies that `craftos --version` reports v2.8.3 or newer, runs `just check` for `luacheck`, and runs `just test` for CraftOS-PC headless tests. Use `just craftos` to launch CraftOS-PC with repo-local save data under `.craftos` and read-only mounts for `/trapos`, `/apis`, `/programs`, `/servers`, and `/startup`. `just repl` opens the same environment with `--cli` for human interactive use only; LLM agents must not run it. Use the CraftOS-PC glossary when adjusting `--headless`, `--exec`, `--script`, `--rom`, or `--mount-*` usage.
|
||||
|
||||
Tests live under `tests/` and run inside CraftOS-PC through `just test`. API-level tests should use `require('/apis/libtest')({ ... })`, register cases with `testlib.test(name, fn)`, and call `testlib.run()` at the end. `libtest` prints `__TRAPOS_TEST_OK__` only when every case passes, which is the contract consumed by the shell harness. Pass `--verbose` to `just test` to list each test script; `libtest` also accepts `--verbose` and prints each case when script stdout is inspected.
|
||||
Tests live under `tests/` and run inside CraftOS-PC through `just test`. API-level tests should use `require('/apis/libtest')({ ... })`, register cases with `testlib.test(name, fn)`, and call `testlib.run()` at the end. `libtest` prints `__TRAPOS_TEST_OK__` only when every case passes, which is the contract consumed by the shell harness. Pass `--verbose` to `just test` to list each test script and each `libtest` case with colored `[OK]`/`[KO]` statuses.
|
||||
|
||||
Each CraftOS-PC test process is killed if it does not finish within `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`, defaulting to `3`. Override this in `.env` for slower local probes.
|
||||
|
||||
35
Justfile
35
Justfile
@ -101,26 +101,27 @@ repl:
|
||||
ci *args: check-craftos check
|
||||
@just test {{args}}
|
||||
|
||||
# Run CraftOS-PC headless smoke tests. Pass `--verbose` to list each test.
|
||||
# Run CraftOS-PC headless integration tests. Pass `--verbose` to list each case.
|
||||
test *args:
|
||||
@if [ -f .env ]; then set -a; . ./.env; set +a; fi; \
|
||||
verbose=0; \
|
||||
timeout_seconds="${TRAP_CCLIBS_TEST_TIMEOUT_SECONDS:-3}"; \
|
||||
case "$timeout_seconds" in ''|*[!0-9]*) printf '%s\n' 'TRAP_CCLIBS_TEST_TIMEOUT_SECONDS must be a positive integer' >&2; exit 1 ;; esac; \
|
||||
if [ "$timeout_seconds" -lt 1 ]; then printf '%s\n' 'TRAP_CCLIBS_TEST_TIMEOUT_SECONDS must be >= 1' >&2; exit 1; fi; \
|
||||
for a in {{args}}; do [ "$a" = "--verbose" ] && verbose=1; done; \
|
||||
script_args=""; \
|
||||
if [ "$verbose" -eq 1 ]; then script_args="--verbose"; fi; \
|
||||
for a in {{args}}; do case "$a" in --verbose|-v) verbose=1 ;; esac; done; \
|
||||
rom_arg=""; \
|
||||
if [ "$(uname -s)" = "Darwin" ]; then \
|
||||
rom_arg="--rom /Applications/CraftOS-PC.app/Contents/Resources"; \
|
||||
fi; \
|
||||
repo='{{justfile_directory()}}'; \
|
||||
mount_arg="--mount-ro /apis=$repo/apis"; \
|
||||
mount_arg="--mount-ro /apis=$repo/apis --mount-ro /tests=$repo/tests"; \
|
||||
green=$(printf '\033[32m'); reset=$(printf '\033[0m'); red=$(printf '\033[31m'); \
|
||||
for script in tests/boot.lua tests/ready.lua tests/eventloop.lua; do \
|
||||
tmp="$(mktemp)"; \
|
||||
craftos --headless $rom_arg $mount_arg --script "$script" $script_args >"$tmp" 2>&1 & \
|
||||
data_dir="$(mktemp -d)"; \
|
||||
exec_code="shell.run('/$script')"; \
|
||||
if [ "$verbose" -eq 1 ]; then exec_code="shell.run('/$script', '--verbose')"; fi; \
|
||||
craftos --directory "$data_dir" --headless $rom_arg $mount_arg --exec "$exec_code" >"$tmp" 2>&1 & \
|
||||
pid="$!"; \
|
||||
( sleep "$timeout_seconds"; kill -TERM "$pid" >/dev/null 2>&1 ) & \
|
||||
watchdog="$!"; \
|
||||
@ -129,7 +130,17 @@ test *args:
|
||||
kill "$watchdog" >/dev/null 2>&1 || true; \
|
||||
wait "$watchdog" >/dev/null 2>&1 || true; \
|
||||
if grep -q __TRAPOS_TEST_OK__ "$tmp"; then \
|
||||
if [ "$verbose" -eq 1 ]; then \
|
||||
report="$data_dir/computer/0/trapos-test-report"; \
|
||||
if [ -f "$report" ]; then while IFS= read -r line; do \
|
||||
case "$line" in \
|
||||
'OK '*) name="${line#OK }"; printf '%s\n' "${green}[OK]${reset} $name" ;; \
|
||||
'KO '*) name="${line#KO }"; printf '%s\n' "${red}[KO]${reset} $name" ;; \
|
||||
esac; \
|
||||
done <"$report"; fi; \
|
||||
fi; \
|
||||
rm -f "$tmp"; \
|
||||
rm -rf "$data_dir"; \
|
||||
[ "$verbose" -eq 1 ] && printf '%s\n' "${green}PASS${reset} $script"; \
|
||||
else \
|
||||
if [ "$status" -eq 143 ]; then \
|
||||
@ -137,12 +148,22 @@ test *args:
|
||||
else \
|
||||
printf '%s\n' "${red}FAIL${reset} $script did not print __TRAPOS_TEST_OK__" >&2; \
|
||||
fi; \
|
||||
if [ "$verbose" -eq 1 ]; then \
|
||||
report="$data_dir/computer/0/trapos-test-report"; \
|
||||
if [ -f "$report" ]; then while IFS= read -r line; do \
|
||||
case "$line" in \
|
||||
'OK '*) name="${line#OK }"; printf '%s\n' "${green}[OK]${reset} $name" >&2 ;; \
|
||||
'KO '*) name="${line#KO }"; printf '%s\n' "${red}[KO]${reset} $name" >&2 ;; \
|
||||
esac; \
|
||||
done <"$report"; fi; \
|
||||
fi; \
|
||||
cat "$tmp" >&2; \
|
||||
rm -f "$tmp"; \
|
||||
rm -rf "$data_dir"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
done; \
|
||||
printf '%s\n' 'OK: smoke tests passed'
|
||||
printf '%s\n' 'OK: CraftOS integration tests passed'
|
||||
|
||||
# Lint all Lua source with luacheck.
|
||||
check: check-luacheck
|
||||
|
||||
132
apis/libtest.lua
132
apis/libtest.lua
@ -1,72 +1,92 @@
|
||||
local _VERSION = '1.1.0';
|
||||
local _VERSION = "1.3.0"
|
||||
|
||||
local function createLibTest(args)
|
||||
local api = {};
|
||||
local tests = {};
|
||||
local verbose = false;
|
||||
local api = {}
|
||||
local tests = {}
|
||||
local verbose = false
|
||||
local reportPath = "/trapos-test-report"
|
||||
|
||||
for _, arg in ipairs(args or {}) do
|
||||
if arg == '--verbose' then
|
||||
verbose = true;
|
||||
end
|
||||
end
|
||||
for _, arg in ipairs(args or {}) do
|
||||
if arg == "--verbose" then
|
||||
verbose = true
|
||||
end
|
||||
end
|
||||
|
||||
local function fail(message)
|
||||
error(message, 2);
|
||||
end
|
||||
local function fail(message)
|
||||
error(message, 2)
|
||||
end
|
||||
|
||||
function api.test(name, fn)
|
||||
assert(type(name) == 'string', 'bad argument #1 (string expected)');
|
||||
assert(type(fn) == 'function', 'bad argument #2 (function expected)');
|
||||
local function writeReport(line)
|
||||
if not verbose then
|
||||
return
|
||||
end
|
||||
|
||||
tests[#tests + 1] = { name = name, fn = fn };
|
||||
end
|
||||
local file = fs.open(reportPath, "a")
|
||||
if file then
|
||||
file.writeLine(line)
|
||||
file.close()
|
||||
end
|
||||
end
|
||||
|
||||
function api.assertEquals(actual, expected, message)
|
||||
if actual ~= expected then
|
||||
fail((message or 'assertEquals failed') .. ': expected ' .. tostring(expected) .. ', got ' .. tostring(actual));
|
||||
end
|
||||
end
|
||||
function api.test(name, fn)
|
||||
assert(type(name) == "string", "bad argument #1 (string expected)")
|
||||
assert(type(fn) == "function", "bad argument #2 (function expected)")
|
||||
|
||||
function api.assertTrue(value, message)
|
||||
if not value then
|
||||
fail(message or 'assertTrue failed');
|
||||
end
|
||||
end
|
||||
tests[#tests + 1] = { name = name, fn = fn }
|
||||
end
|
||||
|
||||
function api.assertErrors(fn, expected)
|
||||
assert(type(fn) == 'function', 'bad argument #1 (function expected)');
|
||||
function api.assertEquals(actual, expected, message)
|
||||
if actual ~= expected then
|
||||
fail(
|
||||
(message or "assertEquals failed")
|
||||
.. ": expected "
|
||||
.. tostring(expected)
|
||||
.. ", got "
|
||||
.. tostring(actual)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
local ok, err = pcall(fn);
|
||||
if ok then
|
||||
fail('expected error');
|
||||
end
|
||||
if expected and not string.find(tostring(err), expected, 1, true) then
|
||||
fail('expected error containing ' .. expected .. ', got ' .. tostring(err));
|
||||
end
|
||||
end
|
||||
function api.assertTrue(value, message)
|
||||
if not value then
|
||||
fail(message or "assertTrue failed")
|
||||
end
|
||||
end
|
||||
|
||||
function api.run()
|
||||
for _, t in ipairs(tests) do
|
||||
if verbose then
|
||||
print('RUN ' .. t.name);
|
||||
end
|
||||
local ok, err = pcall(t.fn);
|
||||
if not ok then
|
||||
print('FAIL ' .. t.name .. ': ' .. tostring(err));
|
||||
os.shutdown();
|
||||
end
|
||||
end
|
||||
function api.assertErrors(fn, expected)
|
||||
assert(type(fn) == "function", "bad argument #1 (function expected)")
|
||||
|
||||
print('__TRAPOS_TEST_OK__');
|
||||
os.shutdown();
|
||||
end
|
||||
local ok, err = pcall(fn)
|
||||
if ok then
|
||||
fail("expected error")
|
||||
end
|
||||
if expected and not string.find(tostring(err), expected, 1, true) then
|
||||
fail("expected error containing " .. expected .. ", got " .. tostring(err))
|
||||
end
|
||||
end
|
||||
|
||||
function api.version()
|
||||
return _VERSION;
|
||||
end
|
||||
function api.run()
|
||||
for _, t in ipairs(tests) do
|
||||
local ok, err = pcall(t.fn)
|
||||
if not ok then
|
||||
writeReport("KO " .. t.name .. ": " .. tostring(err))
|
||||
if not verbose then
|
||||
print("FAIL " .. t.name .. ": " .. tostring(err))
|
||||
end
|
||||
os.shutdown()
|
||||
end
|
||||
writeReport("OK " .. t.name)
|
||||
end
|
||||
|
||||
return api;
|
||||
print("__TRAPOS_TEST_OK__")
|
||||
os.shutdown()
|
||||
end
|
||||
|
||||
function api.version()
|
||||
return _VERSION
|
||||
end
|
||||
|
||||
return api
|
||||
end
|
||||
|
||||
return createLibTest;
|
||||
return createLibTest
|
||||
|
||||
@ -28,7 +28,7 @@ Treat CraftOS-PC as a first-class local development dependency.
|
||||
- **Documented upstream navigation** in [`docs/craftos_pc_glossary.md`](../craftos_pc_glossary.md), covering CLI flags, mounts, `periphemu`, save data, and troubleshooting pages.
|
||||
- **Verified by `just install`** via `check-install`, which checks `craftos`, `jq`, and `luacheck`. `check-craftos` runs `craftos --version` and requires v2.8.3 or newer. Failure prints a one-line pointer to the install guide instead of a long stack trace.
|
||||
- **Repository-local launch recipe.** `just craftos` runs CraftOS-PC with `--directory .craftos`, keeps the macOS `--rom /Applications/CraftOS-PC.app/Contents/Resources` workaround, mounts the repository root read-only at `/trapos`, and mounts each manifest top-level directory read-only at its ComputerCraft root path.
|
||||
- **`just ci` is the local verification entry point.** Today it runs `check-craftos`, `check`, and `test`. The installed pre-commit hook invokes `just ci` directly, and `just test` owns CraftOS-PC-driven smoke tests.
|
||||
- **`just ci` is the local verification entry point.** Today it runs `check-craftos`, `check`, and `test`. The installed pre-commit hook invokes `just ci` directly, and `just test` owns CraftOS-PC-driven integration tests.
|
||||
|
||||
The existing `CLAUDE.md` constraint ("Do not run Lua locally or add a test harness unless asked") is reframed rather than removed: there is still no standalone Lua harness, and we are not adding a Busted-style test runner. The harness *is* CraftOS-PC, invoked deliberately.
|
||||
|
||||
@ -36,7 +36,7 @@ The existing `CLAUDE.md` constraint ("Do not run Lua locally or add a test harne
|
||||
|
||||
- Contributors must install CraftOS-PC before `just install` succeeds. The install guide makes this a 4-step copy/paste on macOS.
|
||||
- Contributors should use the local CraftOS-PC glossary first when researching emulator behavior, then follow the linked upstream pages for details.
|
||||
- Headless tests live under `tests/` and are driven by `just test`. Smoke tests still prove CraftOS-PC boots and the event queue works, and API behavior tests use `/apis/libtest.lua` for named cases and assertions. Each script is invoked as `craftos --headless --script <file>` and its stdout is grepped for `__TRAPOS_TEST_OK__`. Adding another test means dropping a Lua file in `tests/` and adding it to the loop in the `test:` recipe.
|
||||
- Headless tests live under `tests/` and are driven by `just test`. Basic boot tests still prove CraftOS-PC boots and the event queue works, and API behavior tests use `/apis/libtest.lua` for named cases and assertions. Tests are mounted into CraftOS-PC and invoked with `shell.run`, and stdout is grepped for `__TRAPOS_TEST_OK__`. Adding another test means dropping a Lua file in `tests/` and adding it to the loop in the `test:` recipe.
|
||||
- Each test process is guarded by `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`, defaulting to `3`, so a blocked ComputerCraft event loop fails quickly and prints captured output.
|
||||
- The macOS install symlinks the binary into `/usr/local/bin`, which makes CraftOS-PC unable to auto-discover the ROM that ships inside the `.app` bundle (`Could not mount ROM`). The `test:` recipe works around this by passing `--rom /Applications/CraftOS-PC.app/Contents/Resources` on Darwin. Linux (AppImage) and Windows (installer) auto-discover correctly, so no flag is passed there.
|
||||
- `just craftos` uses repository-local save data under `.craftos/config/` and `.craftos/computer/`. This keeps emulator state out of `~/Library/Application Support/CraftOS-PC` during repository work and keeps repo files visible through read-only mounts instead of copying them into the VM save.
|
||||
|
||||
@ -37,7 +37,7 @@ end);
|
||||
testlib.run();
|
||||
```
|
||||
|
||||
`libtest` intentionally stays small. It provides named cases, `assertEquals`, `assertTrue`, `assertErrors`, verbose `RUN <name>` output, failure reporting, the `__TRAPOS_TEST_OK__` success marker, and `os.shutdown()` at process end.
|
||||
`libtest` intentionally stays small. It provides named cases, `assertEquals`, `assertTrue`, `assertErrors`, verbose case-status report output consumed by the shell harness, failure reporting, the `__TRAPOS_TEST_OK__` success marker, and `os.shutdown()` at process end.
|
||||
|
||||
The shell harness keeps ownership of process-level concerns: CraftOS-PC launch flags, read-only mounts, stdout capture, and timeout enforcement through `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`.
|
||||
|
||||
|
||||
@ -82,7 +82,7 @@ On macOS, use the `--rom` form shown above if the command fails with `Could not
|
||||
|
||||
## Running tests
|
||||
|
||||
`just test` runs the headless smoke tests in `tests/` through CraftOS-PC. On macOS the recipe passes `--rom /Applications/CraftOS-PC.app/Contents/Resources` because the `/usr/local/bin/craftos` symlink loses ROM auto-discovery; on Linux and Windows no flag is needed. `just ci` runs the same tests after `luacheck`.
|
||||
`just test` runs the headless CraftOS integration tests in `tests/` through CraftOS-PC. On macOS the recipe passes `--rom /Applications/CraftOS-PC.app/Contents/Resources` because the `/usr/local/bin/craftos` symlink loses ROM auto-discovery; on Linux and Windows no flag is needed. `just ci` runs the same tests after `luacheck`.
|
||||
|
||||
## Repository-local launches
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user