fix(sandbox): ignore stale displacement closes
This commit is contained in:
parent
12947f5fd7
commit
88e1f480b7
@ -280,16 +280,24 @@ local function createSandbox()
|
|||||||
scheduleReconnect();
|
scheduleReconnect();
|
||||||
end);
|
end);
|
||||||
|
|
||||||
-- CC:T passes (url, reason, code). A displacement close (code 4000) is terminal:
|
-- CC:T passes (url, reason, code). A displacement close (code 4000) is only
|
||||||
-- another computer registered with our traposId, so we yield rather than reconnect
|
-- terminal when it hits our *live* (ready) socket: another computer registered
|
||||||
-- (reconnecting would just displace it back, an endless ping-pong).
|
-- our traposId, so we yield rather than reconnect (reconnecting would just
|
||||||
|
-- displace it back, an endless ping-pong). A displacement that arrives while we
|
||||||
|
-- are still connecting/awaiting_hello is for a socket our own reconnect has
|
||||||
|
-- already superseded -- ignoring it keeps the healthy new socket alive instead
|
||||||
|
-- of bricking the daemon into a permanent 'displaced' state.
|
||||||
el.register('websocket_closed', function(eventUrl, reason, code)
|
el.register('websocket_closed', function(eventUrl, reason, code)
|
||||||
if eventUrl ~= url then return; end
|
if eventUrl ~= url then return; end
|
||||||
if code == DISPLACED_CLOSE_CODE or reason == 'displaced' then
|
if code == DISPLACED_CLOSE_CODE or reason == 'displaced' then
|
||||||
|
if state == 'ready' then
|
||||||
activeWs = nil;
|
activeWs = nil;
|
||||||
state = 'displaced';
|
state = 'displaced';
|
||||||
lastError = 'displaced';
|
lastError = 'displaced';
|
||||||
log('warn', 'displaced by a newer registration; yielding', { traposId = traposId });
|
log('warn', 'displaced by a newer registration; yielding', { traposId = traposId });
|
||||||
|
else
|
||||||
|
log('info', 'ignoring stale displacement during reconnect', { traposId = traposId });
|
||||||
|
end
|
||||||
return;
|
return;
|
||||||
end
|
end
|
||||||
log('info', 'connection closed', { code = code, reason = reason });
|
log('info', 'connection closed', { code = code, reason = reason });
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "TrapOS",
|
"name": "TrapOS",
|
||||||
"version": "0.9.1",
|
"version": "0.9.2",
|
||||||
"branch": "next",
|
"branch": "next",
|
||||||
"packages": [
|
"packages": [
|
||||||
"trapos"
|
"trapos"
|
||||||
|
|||||||
@ -6,8 +6,8 @@
|
|||||||
"trapos-net": "0.3.0",
|
"trapos-net": "0.3.0",
|
||||||
"trapos-ui": "0.2.2",
|
"trapos-ui": "0.2.2",
|
||||||
"trapos-ai": "0.7.0",
|
"trapos-ai": "0.7.0",
|
||||||
"trapos-sandbox": "0.2.4",
|
"trapos-sandbox": "0.2.5",
|
||||||
"trapos-sandbox-legacy": "0.2.2",
|
"trapos-sandbox-legacy": "0.2.2",
|
||||||
"trapos": "0.9.1"
|
"trapos": "0.9.2"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "trapos-sandbox",
|
"name": "trapos-sandbox",
|
||||||
"version": "0.2.4",
|
"version": "0.2.5",
|
||||||
"description": "TrapOS sandbox gateway client: WS daemon (servers/sandbox) + sandbox command",
|
"description": "TrapOS sandbox gateway client: WS daemon (servers/sandbox) + sandbox command",
|
||||||
"dependencies": ["trapos-core"],
|
"dependencies": ["trapos-core"],
|
||||||
"files": [
|
"files": [
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "trapos",
|
"name": "trapos",
|
||||||
"version": "0.9.1",
|
"version": "0.9.2",
|
||||||
"description": "TrapOS full install meta-package",
|
"description": "TrapOS full install meta-package",
|
||||||
"dependencies": [
|
"dependencies": [
|
||||||
"trapos-boot",
|
"trapos-boot",
|
||||||
|
|||||||
@ -255,6 +255,8 @@ testlib.test('startSession treats a displacement close by reason as terminal', f
|
|||||||
local ctx = startSession();
|
local ctx = startSession();
|
||||||
local ws = fakes.fakeWs();
|
local ws = fakes.fakeWs();
|
||||||
ctx.el.fire('websocket_success', ctx.url, ws);
|
ctx.el.fire('websocket_success', ctx.url, ws);
|
||||||
|
ctx.el.fire('websocket_message', ctx.url, encode({ event = 'gateway_response', type = 'hello', ok = true, payload = {} }));
|
||||||
|
testlib.assertEquals(ctx.session.isReady(), true);
|
||||||
local timersBefore = #ctx.el.timers;
|
local timersBefore = #ctx.el.timers;
|
||||||
|
|
||||||
ctx.el.fire('websocket_closed', ctx.url, 'displaced'); -- reason only, code absent
|
ctx.el.fire('websocket_closed', ctx.url, 'displaced'); -- reason only, code absent
|
||||||
@ -264,6 +266,21 @@ testlib.test('startSession treats a displacement close by reason as terminal', f
|
|||||||
testlib.assertEquals(#ctx.http.connects, 1);
|
testlib.assertEquals(#ctx.http.connects, 1);
|
||||||
end);
|
end);
|
||||||
|
|
||||||
|
testlib.test('startSession ignores a displacement of an already-superseded socket', function()
|
||||||
|
-- A 4000 close that arrives before our reconnect completes hello is for a socket
|
||||||
|
-- our own reconnect superseded; it must not brick the daemon into 'displaced'.
|
||||||
|
local ctx = startSession();
|
||||||
|
local ws = fakes.fakeWs();
|
||||||
|
ctx.el.fire('websocket_success', ctx.url, ws);
|
||||||
|
-- still awaiting_hello (no ack yet): a stale displacement arrives.
|
||||||
|
ctx.el.fire('websocket_closed', ctx.url, 'displaced', 4000);
|
||||||
|
|
||||||
|
testlib.assertEquals(ctx.session.lastError(), nil);
|
||||||
|
-- Not terminal: a hello ack on the live socket still brings us to ready.
|
||||||
|
ctx.el.fire('websocket_message', ctx.url, encode({ event = 'gateway_response', type = 'hello', ok = true, payload = {} }));
|
||||||
|
testlib.assertEquals(ctx.session.isReady(), true);
|
||||||
|
end);
|
||||||
|
|
||||||
testlib.test('hello watchdog closes a socket stuck awaiting ack', function()
|
testlib.test('hello watchdog closes a socket stuck awaiting ack', function()
|
||||||
local ctx = startSession();
|
local ctx = startSession();
|
||||||
local ws = fakes.fakeWs();
|
local ws = fakes.fakeWs();
|
||||||
|
|||||||
@ -7,6 +7,14 @@ import type { AccountId } from "../../auth.js";
|
|||||||
|
|
||||||
let nextConnectionId = 1;
|
let nextConnectionId = 1;
|
||||||
|
|
||||||
|
// WS keepalive. The gateway is idle between requests, so without protocol-level
|
||||||
|
// traffic the CC:T client's socket hits its idle timeout and drops (code 1006
|
||||||
|
// "Timed out"), churning reconnects. We ping every interval; the client auto-pongs
|
||||||
|
// at the protocol level. A socket that misses a full interval (no pong) is presumed
|
||||||
|
// dead and terminated, which also reaps a rebooted machine's stale half-open entry
|
||||||
|
// before its new daemon connects (so newest-wins doesn't fire on a phantom).
|
||||||
|
const HEARTBEAT_INTERVAL_MS = 20_000;
|
||||||
|
|
||||||
export type GatewayModuleOptions = {
|
export type GatewayModuleOptions = {
|
||||||
registry: GatewayRegistry;
|
registry: GatewayRegistry;
|
||||||
probe: OpencodeProbe;
|
probe: OpencodeProbe;
|
||||||
@ -38,13 +46,36 @@ const gatewayModule: FastifyPluginAsync<GatewayModuleOptions> = (fastify, opts)
|
|||||||
log,
|
log,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Heartbeat: a pong (or any inbound frame) marks the socket live for the next
|
||||||
|
// interval; a silent interval terminates it.
|
||||||
|
let isAlive = true;
|
||||||
|
socket.on("pong", () => {
|
||||||
|
isAlive = true;
|
||||||
|
});
|
||||||
|
const heartbeat = setInterval(() => {
|
||||||
|
if (!isAlive) {
|
||||||
|
log.info("gateway connection heartbeat timeout; terminating");
|
||||||
|
socket.terminate();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
isAlive = false;
|
||||||
|
try {
|
||||||
|
socket.ping();
|
||||||
|
} catch {
|
||||||
|
// socket already closing; the close handler clears the interval
|
||||||
|
}
|
||||||
|
}, HEARTBEAT_INTERVAL_MS);
|
||||||
|
|
||||||
socket.on("message", (raw) => {
|
socket.on("message", (raw) => {
|
||||||
|
isAlive = true;
|
||||||
void connection.onMessage(raw);
|
void connection.onMessage(raw);
|
||||||
});
|
});
|
||||||
socket.on("close", (code, reason) => {
|
socket.on("close", (code, reason) => {
|
||||||
|
clearInterval(heartbeat);
|
||||||
connection.onClose(code, reason.toString());
|
connection.onClose(code, reason.toString());
|
||||||
});
|
});
|
||||||
socket.on("error", (err) => {
|
socket.on("error", (err) => {
|
||||||
|
clearInterval(heartbeat);
|
||||||
log.warn({ err }, "gateway socket error");
|
log.warn({ err }, "gateway socket error");
|
||||||
connection.onClose();
|
connection.onClose();
|
||||||
});
|
});
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user