From 3450d6e258a7b6ffba37b2f667a20df1c17d8b81 Mon Sep 17 00:00:00 2001 From: Guillaume ARM Date: Mon, 8 Jun 2026 04:23:31 +0200 Subject: [PATCH] test(craftos): add libtest helper --- CLAUDE.md | 3 + DEVELOPMENT.md | 8 +- apis/libtest.lua | 72 ++++++++++++ docs/adrs/README.md | 1 + docs/adrs/adr-0005-craftos-pc-harness.md | 5 +- .../adr-0007-use-libtest-for-craftos-tests.md | 56 +++++++++ manifest.json | 1 + tests/eventloop.lua | 107 +++++------------- 8 files changed, 171 insertions(+), 82 deletions(-) create mode 100644 apis/libtest.lua create mode 100644 docs/adrs/adr-0007-use-libtest-for-craftos-tests.md diff --git a/CLAUDE.md b/CLAUDE.md index c3018ce..c2c6efc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,6 +12,8 @@ Use `docs/README.md` as the entrypoint for CC:Tweaked, CraftOS-PC, Advanced Peri - Do not add a standalone Lua test harness unless asked. Local execution happens through the CraftOS-PC harness (see `docs/install-craftos-pc.md`, `docs/craftos_pc_glossary.md`, and ADR-0005); code otherwise executes in-game. - Do not run `just repl` as an LLM agent; it is a human-only interactive CraftOS-PC wrapper. Use `just craftos --headless ...` for automated probes. +- When changing behavior, add as many useful CraftOS-PC tests as practical. It is acceptable to skip tests that require human-only validation, such as complex turtle motion, in-game UX feel, or visual approval, but still add unit-style non-regression tests for deterministic parts when possible. +- Use `/apis/libtest.lua` for CraftOS-PC test scripts under `tests/`; scripts must print `__READY__` only after all assertions pass. - After editing Lua, run `just check` and fix all `luacheck` warnings. - Use 2-space indent, semicolons, and `local function`. - `require` paths are absolute ComputerCraft paths, for example `require('/apis/net')()`. @@ -20,6 +22,7 @@ Use `docs/README.md` as the entrypoint for CC:Tweaked, CraftOS-PC, Advanced Peri ## Architecture - `apis/eventloop.lua` is the single-threaded event loop around `os.pullEventRaw`; consider using it everywhere async behavior is needed. A handler that returns `api.STOP` auto-unregisters. +- `apis/libtest.lua` is the lightweight CraftOS-PC test helper used by scripts under `tests/`; it provides assertions, verbose case output, failure reporting, and the `__READY__` success marker. - `apis/net.lua` builds modem packet messaging, routing, and request/response RPC on the event loop. `sendRequest` returns `ok, result, packet` and defaults to a 0.5s timeout. - A router (`/programs/router.lua`) must be running somewhere on the network; without it, packets lack `routerId`, `isPacketOk` rejects them, and cross-machine messaging silently fails. - `servers/` listen for requests and start loops; `programs/` are clients that send requests and exit. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 84822ea..6b42d30 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -12,6 +12,10 @@ After cloning the repository, run: just install ``` -This installs the local Git hooks, including a pre-commit hook that runs `just ci`. +This creates `.env` from `.env.sample` when needed and installs the local Git hooks, including a pre-commit hook that runs `just ci`. -`just ci` is the local verification entry point. Today it verifies that `craftos --version` reports v2.8.3 or newer, runs `just check` for `luacheck`, and runs `just test` for CraftOS-PC headless smoke tests. Use `just craftos` to launch CraftOS-PC with repo-local save data under `.craftos` and read-only mounts for `/trapos`, `/apis`, `/programs`, `/servers`, and `/startup`. `just repl` opens the same environment with `--cli` for human interactive use only; LLM agents must not run it. Use the CraftOS-PC glossary when adjusting `--headless`, `--exec`, `--script`, `--rom`, or `--mount-*` usage. +`just ci` is the local verification entry point. Today it verifies that `craftos --version` reports v2.8.3 or newer, runs `just check` for `luacheck`, and runs `just test` for CraftOS-PC headless tests. Use `just craftos` to launch CraftOS-PC with repo-local save data under `.craftos` and read-only mounts for `/trapos`, `/apis`, `/programs`, `/servers`, and `/startup`. `just repl` opens the same environment with `--cli` for human interactive use only; LLM agents must not run it. Use the CraftOS-PC glossary when adjusting `--headless`, `--exec`, `--script`, `--rom`, or `--mount-*` usage. + +Tests live under `tests/` and run inside CraftOS-PC through `just test`. API-level tests should use `require('/apis/libtest')({ ... })`, register cases with `testlib.test(name, fn)`, and call `testlib.run()` at the end. `libtest` prints `__READY__` only when every case passes, which is the contract consumed by the shell harness. Pass `--verbose` to `just test` to list each test script; `libtest` also accepts `--verbose` and prints each case when script stdout is inspected. + +Each CraftOS-PC test process is killed if it does not finish within `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`, defaulting to `3`. Override this in `.env` for slower local probes. diff --git a/apis/libtest.lua b/apis/libtest.lua new file mode 100644 index 0000000..2773486 --- /dev/null +++ b/apis/libtest.lua @@ -0,0 +1,72 @@ +local _VERSION = '1.0.0'; + +local function createLibTest(args) + local api = {}; + local tests = {}; + local verbose = false; + + for _, arg in ipairs(args or {}) do + if arg == '--verbose' then + verbose = true; + end + end + + local function fail(message) + error(message, 2); + end + + function api.test(name, fn) + assert(type(name) == 'string', 'bad argument #1 (string expected)'); + assert(type(fn) == 'function', 'bad argument #2 (function expected)'); + + tests[#tests + 1] = { name = name, fn = fn }; + end + + function api.assertEquals(actual, expected, message) + if actual ~= expected then + fail((message or 'assertEquals failed') .. ': expected ' .. tostring(expected) .. ', got ' .. tostring(actual)); + end + end + + function api.assertTrue(value, message) + if not value then + fail(message or 'assertTrue failed'); + end + end + + function api.assertErrors(fn, expected) + assert(type(fn) == 'function', 'bad argument #1 (function expected)'); + + local ok, err = pcall(fn); + if ok then + fail('expected error'); + end + if expected and not string.find(tostring(err), expected, 1, true) then + fail('expected error containing ' .. expected .. ', got ' .. tostring(err)); + end + end + + function api.run() + for _, t in ipairs(tests) do + if verbose then + print('RUN ' .. t.name); + end + local ok, err = pcall(t.fn); + if not ok then + print('FAIL ' .. t.name .. ': ' .. tostring(err)); + os.shutdown(); + end + end + + print('__READY__'); + os.shutdown(); + end + + function api.version() + return _VERSION; + end + + return api; +end + +return createLibTest; diff --git a/docs/adrs/README.md b/docs/adrs/README.md index 4827beb..a5f8ed3 100644 --- a/docs/adrs/README.md +++ b/docs/adrs/README.md @@ -14,3 +14,4 @@ Future ADRs can reuse the shape of the existing files when it is useful. - [`adr-0004-trapos-branding-and-manifest.md`](adr-0004-trapos-branding-and-manifest.md) - TrapOS branding and manifest-driven installs. - [`adr-0005-craftos-pc-harness.md`](adr-0005-craftos-pc-harness.md) - CraftOS-PC as the local harness. - [`adr-0006-simplify-periphemu-bootstrap.md`](adr-0006-simplify-periphemu-bootstrap.md) - Simplify periphemu bootstrap. +- [`adr-0007-use-libtest-for-craftos-tests.md`](adr-0007-use-libtest-for-craftos-tests.md) - Use libtest for CraftOS tests. diff --git a/docs/adrs/adr-0005-craftos-pc-harness.md b/docs/adrs/adr-0005-craftos-pc-harness.md index e6a953b..12b1d7a 100644 --- a/docs/adrs/adr-0005-craftos-pc-harness.md +++ b/docs/adrs/adr-0005-craftos-pc-harness.md @@ -36,7 +36,8 @@ The existing `CLAUDE.md` constraint ("Do not run Lua locally or add a test harne - Contributors must install CraftOS-PC before `just install` succeeds. The install guide makes this a 4-step copy/paste on macOS. - Contributors should use the local CraftOS-PC glossary first when researching emulator behavior, then follow the linked upstream pages for details. -- Headless smoke tests live under `tests/` and are driven by `just test`. Today there are two: `tests/boot.lua` (prints a marker and shuts down — proves the BIOS started) and `tests/ready.lua` (round-trips a `craftos-ready` event through `os.queueEvent` / `os.pullEventRaw` — proves the CC event queue works). Each is invoked as `craftos --headless --script ` and its stdout is grepped for `__READY__`. Adding a third test means dropping a Lua file in `tests/` and adding it to the loop in the `test:` recipe. +- Headless tests live under `tests/` and are driven by `just test`. Smoke tests still prove CraftOS-PC boots and the event queue works, and API behavior tests use `/apis/libtest.lua` for named cases and assertions. Each script is invoked as `craftos --headless --script ` and its stdout is grepped for `__READY__`. Adding another test means dropping a Lua file in `tests/` and adding it to the loop in the `test:` recipe. +- Each test process is guarded by `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`, defaulting to `3`, so a blocked ComputerCraft event loop fails quickly and prints captured output. - The macOS install symlinks the binary into `/usr/local/bin`, which makes CraftOS-PC unable to auto-discover the ROM that ships inside the `.app` bundle (`Could not mount ROM`). The `test:` recipe works around this by passing `--rom /Applications/CraftOS-PC.app/Contents/Resources` on Darwin. Linux (AppImage) and Windows (installer) auto-discover correctly, so no flag is passed there. - `just craftos` uses repository-local save data under `.craftos/config/` and `.craftos/computer/`. This keeps emulator state out of `~/Library/Application Support/CraftOS-PC` during repository work and keeps repo files visible through read-only mounts instead of copying them into the VM save. - `just repl` is a human-only interactive wrapper around `just craftos --cli`; automation and LLM agents must use headless `just craftos ...` invocations instead. @@ -45,6 +46,6 @@ The existing `CLAUDE.md` constraint ("Do not run Lua locally or add a test harne ## Future Work -- **API-loading smoke test.** Extend the `tests/` set with a script that runs through `just craftos` and `require`s `/apis/eventloop`, `/apis/net`, and the router, asserting the wiring loads without errors. Today's smokes only prove CraftOS-PC itself works, not that our code loads inside it. +- **API-loading smoke test.** Extend the `tests/` set with a script that `require`s `/apis/eventloop`, `/apis/net`, `/apis/libtest`, and the router, asserting the wiring loads without errors. - **CI.** Run `just test` on push using the Linux AppImage. - **Pinned ROM.** CraftOS-PC ships its own copy of the CC:Tweaked ROM per release. If we ever need to test against a specific in-game version, point CraftOS-PC at a vendored ROM via `--rom` (the same flag we already pass on macOS for a different reason). diff --git a/docs/adrs/adr-0007-use-libtest-for-craftos-tests.md b/docs/adrs/adr-0007-use-libtest-for-craftos-tests.md new file mode 100644 index 0000000..53c87e6 --- /dev/null +++ b/docs/adrs/adr-0007-use-libtest-for-craftos-tests.md @@ -0,0 +1,56 @@ +# ADR 0007: Use libtest For CraftOS Tests + +## Status + +Accepted + +## Date + +2026-06-08 + +## Context + +ADR 0005 made CraftOS-PC the local harness. The first real behavior test, `tests/eventloop.lua`, proved the harness can exercise ComputerCraft APIs headlessly, but it also duplicated test-runner concerns directly in the script: + +- Collect named cases. +- Print per-case progress only in verbose mode. +- Fail fast with a useful message. +- Print `__READY__` only after every assertion passes. +- Shut the ComputerCraft process down cleanly. + +Those details are easy to copy incorrectly. A blocked eventloop test also showed that the shell harness needs a timeout and captured output so agentic debugging can proceed without manual interruption. + +## Decision + +Add `/apis/libtest.lua` as the repository's lightweight CraftOS-PC test helper. + +Tests under `tests/` should require it with an absolute ComputerCraft path: + +```lua +local createLibTest = require('/apis/libtest'); +local testlib = createLibTest({ ... }); + +testlib.test('example', function() + testlib.assertEquals(1 + 1, 2); +end); + +testlib.run(); +``` + +`libtest` intentionally stays small. It provides named cases, `assertEquals`, `assertTrue`, `assertErrors`, verbose `RUN ` output, failure reporting, the `__READY__` success marker, and `os.shutdown()` at process end. + +The shell harness keeps ownership of process-level concerns: CraftOS-PC launch flags, read-only mounts, stdout capture, and timeout enforcement through `TRAP_CCLIBS_TEST_TIMEOUT_SECONDS`. + +## Consequences + +- New deterministic behavior should get as many useful CraftOS-PC tests as practical. +- Tests that require human validation, such as complex turtle motion, in-game UX feel, or visual approval, may be skipped, but deterministic pieces should still get unit-style non-regression coverage. +- Test scripts remain normal ComputerCraft programs, not standalone Lua tests. They run through `just test`, not through a separate Lua test framework. +- `libtest` lives under `/apis` and is listed in `manifest.json`, so it can be required consistently in the mounted CraftOS-PC environment. +- The `__READY__` marker remains the single shell-level success contract. + +## Future Work + +- Add more assertions only when tests need them; avoid growing a large framework. +- Consider making `just test` discover `tests/*.lua` automatically once the test set grows enough that the explicit list becomes noisy. +- Explore GitHub Actions with the Linux CraftOS-PC AppImage after local coverage is broader. diff --git a/manifest.json b/manifest.json index 31b4260..ac3a4c6 100644 --- a/manifest.json +++ b/manifest.json @@ -13,6 +13,7 @@ "programs/upgrade.lua", "apis/net.lua", "apis/eventloop.lua", + "apis/libtest.lua", "apis/libtui.lua" ], "autostart": [ diff --git a/tests/eventloop.lua b/tests/eventloop.lua index a3e5912..aaf9893 100644 --- a/tests/eventloop.lua +++ b/tests/eventloop.lua @@ -1,66 +1,29 @@ -- Eventloop behavior tests for the CraftOS-PC harness. -- Invoked via `craftos --headless --script tests/eventloop.lua` from `just test`. local createEventLoop = require('/apis/eventloop'); +local createLibTest = require('/apis/libtest'); -local args = { ... }; -local verbose = false; -local tests = {}; +local testlib = createLibTest({ ... }); -for _, arg in ipairs(args) do - if arg == '--verbose' then - verbose = true; - end -end - -local function test(name, fn) - tests[#tests + 1] = { name = name, fn = fn }; -end - -local function fail(message) - error(message, 2); -end - -local function assertEquals(actual, expected, message) - if actual ~= expected then - fail((message or 'assertEquals failed') .. ': expected ' .. tostring(expected) .. ', got ' .. tostring(actual)); - end -end - -local function assertTrue(value, message) - if not value then - fail(message or 'assertTrue failed'); - end -end - -local function assertErrors(fn, expected) - local ok, err = pcall(fn); - if ok then - fail('expected error'); - end - if expected and not string.find(tostring(err), expected, 1, true) then - fail('expected error containing ' .. expected .. ', got ' .. tostring(err)); - end -end - -test('register dispatches queued event args', function() +testlib.test('register dispatches queued event args', function() local events = createEventLoop(); local called = 0; events.register('eventloop_test_basic', function(a, b) called = called + 1; - assertEquals(a, 'first'); - assertEquals(b, 42); + testlib.assertEquals(a, 'first'); + testlib.assertEquals(b, 42); events.stopLoop(); end); os.queueEvent('eventloop_test_basic', 'first', 42); events.runLoop(); - assertEquals(called, 1); - assertTrue(not events.isRunningLoop()); + testlib.assertEquals(called, 1); + testlib.assertTrue(not events.isRunningLoop()); end); -test('STOP unregisters handler after first call', function() +testlib.test('STOP unregisters handler after first call', function() local events = createEventLoop(); local stoppedHandlerCalls = 0; local observerCalls = 0; @@ -82,11 +45,11 @@ test('STOP unregisters handler after first call', function() os.queueEvent('eventloop_test_stop'); events.runLoop(); - assertEquals(stoppedHandlerCalls, 1); - assertEquals(observerCalls, 2); + testlib.assertEquals(stoppedHandlerCalls, 1); + testlib.assertEquals(observerCalls, 2); end); -test('manual unregister prevents dispatch', function() +testlib.test('manual unregister prevents dispatch', function() local events = createEventLoop(); local called = 0; local dispose = events.register('eventloop_test_unregister', function() @@ -101,10 +64,10 @@ test('manual unregister prevents dispatch', function() os.queueEvent('eventloop_test_unregister'); events.runLoop(); - assertEquals(called, 0); + testlib.assertEquals(called, 0); end); -test('setTimeout before runLoop fires once', function() +testlib.test('setTimeout before runLoop fires once', function() local events = createEventLoop(); local called = 0; @@ -115,10 +78,10 @@ test('setTimeout before runLoop fires once', function() events.runLoop(); - assertEquals(called, 1); + testlib.assertEquals(called, 1); end); -test('setTimeout during runLoop fires after event handler', function() +testlib.test('setTimeout during runLoop fires after event handler', function() local events = createEventLoop(); local order = ''; @@ -134,10 +97,10 @@ test('setTimeout during runLoop fires after event handler', function() os.queueEvent('eventloop_test_runtime_timeout'); events.runLoop(); - assertEquals(order, 'event>timeout'); + testlib.assertEquals(order, 'event>timeout'); end); -test('cleared timeout before runLoop does not fire', function() +testlib.test('cleared timeout before runLoop does not fire', function() local events = createEventLoop(); local called = 0; @@ -151,10 +114,10 @@ test('cleared timeout before runLoop does not fire', function() end, 0); events.runLoop(); - assertEquals(called, 0); + testlib.assertEquals(called, 0); end); -test('onStart and onStop run around loop', function() +testlib.test('onStart and onStop run around loop', function() local events = createEventLoop(); local order = ''; @@ -173,10 +136,10 @@ test('onStart and onStop run around loop', function() events.runLoop(); - assertEquals(order, 'start>timeout>stop'); + testlib.assertEquals(order, 'start>timeout>stop'); end); -test('empty loop returns and runs onStop', function() +testlib.test('empty loop returns and runs onStop', function() local events = createEventLoop(); local stopped = false; @@ -186,44 +149,32 @@ test('empty loop returns and runs onStop', function() events.runLoop(); - assertTrue(stopped); - assertTrue(not events.isRunningLoop()); + testlib.assertTrue(stopped); + testlib.assertTrue(not events.isRunningLoop()); end); -test('error contracts are enforced', function() +testlib.test('error contracts are enforced', function() local events = createEventLoop(); local function handler() end events.register('eventloop_test_errors', handler); - assertErrors(function() + testlib.assertErrors(function() events.register('eventloop_test_errors', handler); end, 'handler already registered'); - assertErrors(function() + testlib.assertErrors(function() events.stopLoop(); end, 'loop is already stopped'); - assertErrors(function() + testlib.assertErrors(function() events.register(1, handler); end, 'string expected'); - assertErrors(function() + testlib.assertErrors(function() events.register('eventloop_test_errors_2', 'not a function'); end, 'function expected'); end); -for _, t in ipairs(tests) do - if verbose then - print('RUN ' .. t.name); - end - local ok, err = pcall(t.fn); - if not ok then - print('FAIL ' .. t.name .. ': ' .. tostring(err)); - os.shutdown(); - end -end - -print('__READY__'); -os.shutdown(); +testlib.run();