diff --git a/packages/api/src/services/auth-terminal-sessions.ts b/packages/api/src/services/auth-terminal-sessions.ts index 5bc01aa3..f2d6bce0 100644 --- a/packages/api/src/services/auth-terminal-sessions.ts +++ b/packages/api/src/services/auth-terminal-sessions.ts @@ -10,6 +10,7 @@ import { WebSocket, WebSocketServer, type RawData } from "ws" import type { AuthTerminalFlow, AuthTerminalSessionRequest, TerminalSession, TerminalSessionStatus } from "../api/contracts.js" import { ApiConflictError, ApiNotFoundError, describeUnknown } from "../api/errors.js" import { spawnPtyBridge, type PtyBridge } from "./pty-bridge.js" +import { attachWebSocketHeartbeat } from "./websocket-heartbeat.js" type TerminalClientMessage = | { readonly type: "input"; readonly data: string } @@ -26,12 +27,14 @@ type AuthTerminalRecord = { attachTimeout: ReturnType | null args: ReadonlyArray cwd: string + detachTimeout: ReturnType | null pty: PtyBridge | null session: TerminalSession socket: WebSocket | null } const attachTimeoutMs = 30_000 +const reconnectGraceMs = 60_000 const authTerminalProjectId = "__controller__" const authTerminalWsPathPattern = /^(?:\/api)?\/auth\/terminal-sessions\/([^/]+)\/ws$/u const authRunnerPath = fileURLToPath(new URL("../auth-terminal-runner.js", import.meta.url)) @@ -95,6 +98,13 @@ const clearAttachTimeout = (record: AuthTerminalRecord): void => { } } +const clearDetachTimeout = (record: AuthTerminalRecord): void => { + if (record.detachTimeout !== null) { + clearTimeout(record.detachTimeout) + record.detachTimeout = null + } +} + const closeSocket = (socket: WebSocket | null): void => { if (socket === null || socket.readyState === WebSocket.CLOSED) { return @@ -104,6 +114,7 @@ const closeSocket = (socket: WebSocket | null): void => { const cleanupRecord = (record: AuthTerminalRecord): void => { clearAttachTimeout(record) + clearDetachTimeout(record) if (record.pty !== null) { record.pty.kill() record.pty = null @@ -130,6 +141,7 @@ const finalizeRecord = ( record.socket = null record.pty = null clearAttachTimeout(record) + clearDetachTimeout(record) records.delete(record.session.id) } @@ -172,6 +184,10 @@ const resizePty = (pty: PtyBridge | null, cols: number, rows: number): void => { } const startTerminalPty = (record: AuthTerminalRecord, cols: number, rows: number): void => { + if (record.pty !== null) { + resizePty(record.pty, clampTerminalSize(cols, 120), clampTerminalSize(rows, 32)) + return + } const pty = spawnPtyBridge({ args: record.args, cols: clampTerminalSize(cols, 120), @@ -217,6 +233,7 @@ const registerRecord = (request: AuthTerminalSessionRequest): TerminalSession => args: resolveRunnerArgs(request.flow, request.label), attachTimeout: null, cwd: process.cwd(), + detachTimeout: null, pty: null, session, socket: null @@ -226,6 +243,27 @@ const registerRecord = (request: AuthTerminalSessionRequest): TerminalSession => return session } +const createDetachTimeout = (sessionId: string): ReturnType => + setTimeout(() => { + const record = records.get(sessionId) + if (record !== undefined && record.socket === null) { + cleanupRecord(record) + } + }, reconnectGraceMs) + +const detachSocketFromRecord = ( + record: AuthTerminalRecord, + socket: WebSocket +): void => { + const current = records.get(record.session.id) + if (current === undefined || current.socket !== socket) { + return + } + current.socket = null + clearDetachTimeout(current) + current.detachTimeout = createDetachTimeout(current.session.id) +} + const handleSocketMessage = (record: AuthTerminalRecord, raw: RawData): void => { const message = decodeClientMessage(raw) if (message === null) { @@ -249,21 +287,20 @@ const attachSocketToRecord = ( cols: number, rows: number ): void => { - if (record.socket !== null) { + if (record.socket !== null && record.socket.readyState !== WebSocket.CLOSED) { throw new ApiConflictError({ message: `Auth terminal session already attached: ${record.session.id}` }) } clearAttachTimeout(record) + clearDetachTimeout(record) record.socket = socket + attachWebSocketHeartbeat(socket) startTerminalPty(record, cols, rows) sendServerMessage(socket, { type: "ready", session: record.session }) socket.on("message", (raw: RawData) => { handleSocketMessage(record, raw) }) socket.on("close", () => { - const current = records.get(record.session.id) - if (current !== undefined) { - cleanupRecord(current) - } + detachSocketFromRecord(record, socket) }) } diff --git a/packages/api/src/services/terminal-sessions.ts b/packages/api/src/services/terminal-sessions.ts index f6ce7f76..a04273a4 100644 --- a/packages/api/src/services/terminal-sessions.ts +++ b/packages/api/src/services/terminal-sessions.ts @@ -17,6 +17,7 @@ import { ApiConflictError, ApiInternalError, ApiNotFoundError, describeUnknown } import { spawnPtyBridge, type PtyBridge } from "./pty-bridge.js" import { emitProjectEvent } from "./events.js" import { getProjectItemById, upProject } from "./projects.js" +import { attachWebSocketHeartbeat } from "./websocket-heartbeat.js" type TerminalClientMessage = | { readonly type: "input"; readonly data: string } @@ -34,12 +35,14 @@ type TerminalRecord = { pty: PtyBridge | null socket: WebSocket | null attachTimeout: ReturnType | null + detachTimeout: ReturnType | null projectId: string prepared: ReturnType } const records = new Map() const attachTimeoutMs = 30_000 +const reconnectGraceMs = 60_000 const terminalWsPathPattern = /^(?:\/api)?\/projects\/([^/]+)\/terminal-sessions\/([^/]+)\/ws$/u const TerminalClientMessageSchema = Schema.parseJson( @@ -187,6 +190,13 @@ const clearAttachTimeout = (record: TerminalRecord): void => { } } +const clearDetachTimeout = (record: TerminalRecord): void => { + if (record.detachTimeout !== null) { + clearTimeout(record.detachTimeout) + record.detachTimeout = null + } +} + const closeSocket = (socket: WebSocket | null): void => { if (socket === null || socket.readyState === WebSocket.CLOSED) { return @@ -196,6 +206,7 @@ const closeSocket = (socket: WebSocket | null): void => { const cleanupRecord = (record: TerminalRecord): void => { clearAttachTimeout(record) + clearDetachTimeout(record) if (record.pty !== null) { record.pty.kill() record.pty = null @@ -222,6 +233,7 @@ const finalizeRecord = ( record.socket = null record.pty = null clearAttachTimeout(record) + clearDetachTimeout(record) records.delete(record.session.id) } @@ -268,6 +280,10 @@ const startTerminalPty = ( cols: number, rows: number ): void => { + if (record.pty !== null) { + resizePty(record.pty, clampTerminalSize(cols, 120), clampTerminalSize(rows, 32)) + return + } const resolvedCols = clampTerminalSize(cols, 120) const resolvedRows = clampTerminalSize(rows, 32) const pty = spawnPtyBridge({ @@ -316,6 +332,7 @@ const registerRecord = ( } const record: TerminalRecord = { attachTimeout: null, + detachTimeout: null, prepared, projectId, pty: null, @@ -399,6 +416,27 @@ const handleCloseMessage = (record: TerminalRecord): void => { cleanupRecord(record) } +const createDetachTimeout = (sessionId: string): ReturnType => + setTimeout(() => { + const record = records.get(sessionId) + if (record !== undefined && record.socket === null) { + cleanupRecord(record) + } + }, reconnectGraceMs) + +const detachSocketFromRecord = ( + record: TerminalRecord, + socket: WebSocket +): void => { + const current = records.get(record.session.id) + if (current === undefined || current.socket !== socket) { + return + } + current.socket = null + clearDetachTimeout(current) + current.detachTimeout = createDetachTimeout(current.session.id) +} + const handleSocketMessage = (record: TerminalRecord, raw: RawData): void => { const message = decodeClientMessage(raw) if (message === null) { @@ -422,22 +460,21 @@ const attachSocketToRecord = ( cols: number, rows: number ): void => { - if (record.socket !== null) { + if (record.socket !== null && record.socket.readyState !== WebSocket.CLOSED) { throw new ApiConflictError({ message: `Terminal session already attached: ${record.session.id}` }) } clearAttachTimeout(record) + clearDetachTimeout(record) record.socket = socket + attachWebSocketHeartbeat(socket) startTerminalPty(record, cols, rows) sendServerMessage(socket, { type: "ready", session: record.session }) socket.on("message", (raw: RawData) => { handleSocketMessage(record, raw) }) socket.on("close", () => { - const current = records.get(record.session.id) - if (current !== undefined) { - cleanupRecord(current) - } + detachSocketFromRecord(record, socket) }) } diff --git a/packages/api/src/services/websocket-heartbeat.ts b/packages/api/src/services/websocket-heartbeat.ts new file mode 100644 index 00000000..0e59172f --- /dev/null +++ b/packages/api/src/services/websocket-heartbeat.ts @@ -0,0 +1,31 @@ +import { WebSocket } from "ws" + +const defaultHeartbeatIntervalMs = 25_000 + +export const attachWebSocketHeartbeat = ( + socket: WebSocket, + intervalMs = defaultHeartbeatIntervalMs +): void => { + let alive = true + const interval = setInterval(() => { + if (socket.readyState !== WebSocket.OPEN) { + return + } + if (!alive) { + socket.terminate() + return + } + alive = false + socket.ping() + }, intervalMs) + + socket.on("pong", () => { + alive = true + }) + socket.on("close", () => { + clearInterval(interval) + }) + socket.on("error", () => { + clearInterval(interval) + }) +} diff --git a/packages/app/scripts/serve-dist-web.mjs b/packages/app/scripts/serve-dist-web.mjs index 5bff43ea..9a2c3dc0 100644 --- a/packages/app/scripts/serve-dist-web.mjs +++ b/packages/app/scripts/serve-dist-web.mjs @@ -176,6 +176,32 @@ const proxyHttp = ( } const webSocketServer = new WebSocketServer({ noServer: true }) +const webSocketHeartbeatIntervalMs = 25_000 + +const attachWebSocketHeartbeat = (socket) => { + let alive = true + const interval = setInterval(() => { + if (socket.readyState !== WebSocket.OPEN) { + return + } + if (!alive) { + socket.terminate() + return + } + alive = false + socket.ping() + }, webSocketHeartbeatIntervalMs) + + socket.on("pong", () => { + alive = true + }) + socket.on("close", () => { + clearInterval(interval) + }) + socket.on("error", () => { + clearInterval(interval) + }) +} const bridgeWebSockets = (clientSocket, upstream) => { const pending = [] @@ -184,6 +210,8 @@ const bridgeWebSockets = (clientSocket, upstream) => { socket.send(data, { binary: isBinary }) } } + attachWebSocketHeartbeat(clientSocket) + attachWebSocketHeartbeat(upstream) const flushPending = () => { for (const message of pending.splice(0)) { sendWhenOpen(upstream, message.data, message.isBinary) diff --git a/packages/app/src/lib/usecases/projects-ssh.ts b/packages/app/src/lib/usecases/projects-ssh.ts index c89bf9d5..21b3d059 100644 --- a/packages/app/src/lib/usecases/projects-ssh.ts +++ b/packages/app/src/lib/usecases/projects-ssh.ts @@ -55,6 +55,10 @@ const buildSshArgs = (item: ProjectItem): ReadonlyArray => { "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", + "ServerAliveInterval=30", + "-o", + "ServerAliveCountMax=3", "-p", String(port), `${item.sshUser}@${host}` diff --git a/packages/app/src/web/app-dashboard-state.ts b/packages/app/src/web/app-dashboard-state.ts new file mode 100644 index 00000000..72c6e85b --- /dev/null +++ b/packages/app/src/web/app-dashboard-state.ts @@ -0,0 +1,19 @@ +import type { DashboardData } from "./api.js" + +export type DashboardState = + | { readonly _tag: "Loading"; readonly apiBaseUrl: string } + | { readonly _tag: "Error"; readonly apiBaseUrl: string; readonly message: string } + | { readonly _tag: "Ready"; readonly dashboard: DashboardData; readonly refreshedAtMs: number } + +export const mergeDashboardRefreshState = ( + current: DashboardState, + next: DashboardState +): DashboardState => + next._tag === "Error" && current._tag === "Ready" + ? current + : next + +export const createDashboardRefreshReducer = + (next: DashboardState) => + (current: DashboardState): DashboardState => + mergeDashboardRefreshState(current, next) diff --git a/packages/app/src/web/app-ready-terminal-screen.tsx b/packages/app/src/web/app-ready-terminal-screen.tsx index 87cc654c..02d10921 100644 --- a/packages/app/src/web/app-ready-terminal-screen.tsx +++ b/packages/app/src/web/app-ready-terminal-screen.tsx @@ -1,5 +1,7 @@ +import { Effect } from "effect" import type { JSX } from "react" +import { deleteTerminalSessionByPath } from "./api.js" import { canOpenProjectBrowser } from "./app-ready-browser-openable.js" import type { ReadyLayoutProps } from "./app-ready-layout.js" import { Box } from "./elements.js" @@ -16,20 +18,26 @@ type TerminalScreenProps = Pick< | "terminalSession" > +const requestTerminalSessionClose = (closePath: string): void => { + void Effect.runPromise(deleteTerminalSessionByPath(closePath).pipe(Effect.either, Effect.asVoid)) +} + export const TerminalScreen = (props: TerminalScreenProps): JSX.Element | null => { - if (props.terminalSession === null) { + const terminalSession = props.terminalSession + if (terminalSession === null) { return null } - const browserProjectId = props.terminalSession.browserProjectId + const browserProjectId = terminalSession.browserProjectId const canOpenBrowser = canOpenProjectBrowser(props.projectBrowser, browserProjectId) - const returnScreen: BrowserScreen = props.terminalSession.closePath.startsWith("/auth/") + const returnScreen: BrowserScreen = terminalSession.closePath.startsWith("/auth/") ? { tag: "Auth" } : projectPickerScreen() return ( { + requestTerminalSessionClose(terminalSession.closePath) props.onTerminalClose() props.onSetActiveScreen(returnScreen) }} @@ -39,7 +47,7 @@ export const TerminalScreen = (props: TerminalScreenProps): JSX.Element | null = props.onOpenProjectBrowserById(browserProjectId) }} onMessage={props.onTerminalMessage} - session={props.terminalSession} + session={terminalSession} /> ) diff --git a/packages/app/src/web/app.tsx b/packages/app/src/web/app.tsx index 4291562a..f2ec4f08 100644 --- a/packages/app/src/web/app.tsx +++ b/packages/app/src/web/app.tsx @@ -3,16 +3,12 @@ import { type JSX, startTransition, useEffect, useEffectEvent, useState } from " import { webPrimitives } from "../ui/primitives-web.js" import { UiProvider } from "../ui/primitives.js" -import { type DashboardData, loadDashboard, resolveApiBaseUrl } from "./api.js" +import { loadDashboard, resolveApiBaseUrl } from "./api.js" +import { createDashboardRefreshReducer, type DashboardState } from "./app-dashboard-state.js" import { AppReady } from "./app-ready.js" import { ErrorScreen, LoadingScreen } from "./panels.js" import { resolveViewportLayout, type ViewportLayout, type ViewportSize } from "./viewport-layout.js" -type DashboardState = - | { readonly _tag: "Loading"; readonly apiBaseUrl: string } - | { readonly _tag: "Error"; readonly apiBaseUrl: string; readonly message: string } - | { readonly _tag: "Ready"; readonly dashboard: DashboardData; readonly refreshedAtMs: number } - const refreshIntervalMs = 15_000 const resolveViewportSize = (): ViewportSize => ({ @@ -73,7 +69,7 @@ const useDashboardController = () => { const refresh = () => { void Effect.runPromise(loadDashboardState()).then((nextState) => { startTransition(() => { - setState(nextState) + setState(createDashboardRefreshReducer(nextState)) }) }) } diff --git a/packages/app/src/web/panel-terminal.tsx b/packages/app/src/web/panel-terminal.tsx index 72569474..0b77d0db 100644 --- a/packages/app/src/web/panel-terminal.tsx +++ b/packages/app/src/web/panel-terminal.tsx @@ -122,7 +122,7 @@ const TerminalHeader = ( export const TerminalPanel = ( { onClose, onMessage, onOpenBrowser, session }: TerminalPanelProps ): JSX.Element => { - const connectionRef = useRef({ opened: false }) + const connectionRef = useRef({ closing: false, opened: false }) const hostRef = useRef(null) const [status, setStatus] = useState("connecting") const notifyMessage = useEffectEvent(onMessage) @@ -137,7 +137,15 @@ export const TerminalPanel = ( return (
- + { + connectionRef.current.closing = true + onClose() + }} + onOpenBrowser={onOpenBrowser} + session={session} + status={status} + />
) diff --git a/packages/app/src/web/terminal-panel-runtime-core.ts b/packages/app/src/web/terminal-panel-runtime-core.ts new file mode 100644 index 00000000..d84528e1 --- /dev/null +++ b/packages/app/src/web/terminal-panel-runtime-core.ts @@ -0,0 +1,307 @@ +import { Effect, Either } from "effect" +import { Terminal } from "xterm" +import { FitAddon } from "xterm-addon-fit" + +import { deleteTerminalSessionByPath } from "./api.js" +import type { + TerminalCleanupArgs, + TerminalLifecycleState, + TerminalMessageHandlers, + TerminalRuntime, + TerminalSocketConnectArgs, + TerminalSocketListenerArgs, + TerminalSocketRef +} from "./terminal-panel-runtime-types.js" +import { resolveTerminalReconnectDelay, terminalReconnectGraceMs } from "./terminal-reconnect.js" +import { parseTerminalServerMessage, resolveTerminalWebSocketUrl } from "./terminal.js" + +const requestSessionClose = (closePath: string): void => { + void Effect.runPromise(deleteTerminalSessionByPath(closePath).pipe(Effect.either, Effect.asVoid)) +} + +const runOptionalTerminalOperation = (operation: () => void): boolean => + Either.isRight( + Effect.runSync( + Effect.either( + Effect.try({ + try: operation, + catch: (error) => error + }) + ) + ) + ) + +export const createLifecycleState = (): TerminalLifecycleState => ({ + attachedOnce: false, + disposed: false, + readyNotified: false, + reconnectAttempt: 0, + reconnectStartedAtMs: null, + reconnectTimer: null, + terminalEnded: false +}) + +const clearReconnectTimer = (lifecycle: TerminalLifecycleState): void => { + if (lifecycle.reconnectTimer !== null) { + clearTimeout(lifecycle.reconnectTimer) + lifecycle.reconnectTimer = null + } +} + +export const createTerminalRuntime = (host: HTMLDivElement): TerminalRuntime => { + const terminal = new Terminal({ + convertEol: false, + cursorBlink: true, + fontFamily: "'IBM Plex Mono', 'SFMono-Regular', monospace", + fontSize: 14, + theme: { background: "#080a0d", foreground: "#f4f7fb" } + }) + const fitAddon = new FitAddon() + terminal.loadAddon(fitAddon) + terminal.open(host) + fitAddon.fit() + terminal.focus() + return { fitAddon, terminal } +} + +const createTerminalSocket = ( + session: TerminalSocketConnectArgs["session"], + terminal: Terminal +): WebSocket => new WebSocket(resolveTerminalWebSocketUrl(session.websocketPath, terminal.cols, terminal.rows)) + +export const sendTerminalResize = ( + fitAddon: FitAddon, + socketRef: TerminalSocketRef, + terminal: Terminal +): void => { + if ( + !runOptionalTerminalOperation(() => { + fitAddon.fit() + }) + ) { + return + } + const socket = socketRef.current + if (socket === null || socket.readyState !== WebSocket.OPEN) { + return + } + socket.send(JSON.stringify({ + cols: terminal.cols, + rows: terminal.rows, + type: "resize" + })) +} + +export const observeTerminalResize = ( + host: HTMLDivElement, + onResize: () => void +): ResizeObserver | null => { + if (typeof ResizeObserver !== "function") { + return null + } + const resizeObserver = new ResizeObserver(onResize) + resizeObserver.observe(host) + return resizeObserver +} + +export const attachTerminalInput = ( + terminal: Terminal, + socketRef: TerminalSocketRef +) => + terminal.onData((data) => { + const socket = socketRef.current + if (socket === null || socket.readyState !== WebSocket.OPEN) { + return + } + socket.send(JSON.stringify({ data, type: "input" })) + }) + +const notifyTerminalReady = ( + handlers: TerminalMessageHandlers +): void => { + handlers.lifecycle.attachedOnce = true + handlers.connectionRef.current.opened = true + handlers.lifecycle.reconnectAttempt = 0 + handlers.lifecycle.reconnectStartedAtMs = null + clearReconnectTimer(handlers.lifecycle) + handlers.setStatus("attached") + if (handlers.lifecycle.readyNotified) { + handlers.notifyMessage("Terminal reconnected.") + return + } + handlers.lifecycle.readyNotified = true + handlers.notifyMessage(handlers.session.readyMessage) + handlers.session.onReady?.() +} + +const endTerminalSession = ( + handlers: TerminalMessageHandlers, + status: "error" | "exited", + line: string, + message: string +): void => { + handlers.lifecycle.terminalEnded = true + clearReconnectTimer(handlers.lifecycle) + handlers.terminal.writeln(line) + handlers.setStatus(status) + handlers.notifyMessage(message) + if (status === "exited") { + handlers.session.onExit?.() + } +} + +const handleTerminalServerMessage = ( + handlers: TerminalMessageHandlers, + payload: string +): void => { + const message = parseTerminalServerMessage(payload) + if (message === null) { + endTerminalSession(handlers, "error", "\r\n[terminal protocol error]", "Terminal protocol error.") + return + } + if (message.type === "ready") { + notifyTerminalReady(handlers) + return + } + if (message.type === "output") { + handlers.terminal.write(message.data) + return + } + if (message.type === "error") { + endTerminalSession(handlers, "error", `\r\n[error] ${message.message}`, message.message) + return + } + endTerminalSession(handlers, "exited", "\r\n[session ended]", handlers.session.exitMessage) +} + +const attachTerminalSocketListeners = ( + { lifecycle, onClose, onError, onMessage, onOpen, socket }: TerminalSocketListenerArgs +): void => { + socket.addEventListener("open", onOpen) + socket.addEventListener("message", (event) => { + onMessage(typeof event.data === "string" ? event.data : "") + }) + socket.addEventListener("close", () => { + onClose(socket) + }) + socket.addEventListener("error", () => { + if (!lifecycle.disposed) { + onError(socket) + } + }) +} + +const closeSocket = (socket: WebSocket | null): void => { + if (socket === null || socket.readyState === WebSocket.CLOSED) { + return + } + runOptionalTerminalOperation(() => { + socket.close() + }) +} + +export const cleanupTerminalResources = ( + args: TerminalCleanupArgs +): void => { + args.lifecycle.disposed = true + clearReconnectTimer(args.lifecycle) + args.removeInput() + args.resizeObserver?.disconnect() + args.removeResize() + closeSocket(args.socketRef.current) + args.socketRef.current = null + args.terminal.dispose() + if (!args.connectionRef.current.opened && !args.connectionRef.current.closing) { + requestSessionClose(args.session.closePath) + args.notifyMessage(args.session.pendingDeleteMessage) + args.session.onExit?.() + } +} + +export const createMessageHandlers = ( + args: + & Omit + & { readonly terminal: Terminal } +): TerminalMessageHandlers => args + +const failBeforeAttach = ( + args: TerminalSocketConnectArgs, + terminalLine: string, + uiMessage: string +): void => { + args.lifecycle.terminalEnded = true + clearReconnectTimer(args.lifecycle) + args.terminal.writeln(`\r\n${terminalLine}`) + args.setStatus("error") + args.notifyMessage(uiMessage) + requestSessionClose(args.session.closePath) +} + +const scheduleReconnect = (args: TerminalSocketConnectArgs): void => { + if (args.lifecycle.disposed || args.lifecycle.terminalEnded) { + return + } + const startedAt = args.lifecycle.reconnectStartedAtMs ?? Date.now() + args.lifecycle.reconnectStartedAtMs = startedAt + if (Date.now() - startedAt >= terminalReconnectGraceMs) { + failBeforeAttach(args, "[terminal reconnect failed]", "Terminal reconnect failed.") + return + } + if (args.lifecycle.reconnectAttempt === 0) { + args.terminal.writeln("\r\n[terminal connection lost; reconnecting]") + args.notifyMessage("Terminal connection lost. Reconnecting...") + } + args.setStatus("reconnecting") + const delayMs = resolveTerminalReconnectDelay(args.lifecycle.reconnectAttempt) + args.lifecycle.reconnectAttempt += 1 + clearReconnectTimer(args.lifecycle) + args.lifecycle.reconnectTimer = setTimeout(args.reconnect, delayMs) +} + +const handleSocketClose = ( + args: TerminalSocketConnectArgs, + closedSocket: WebSocket +): void => { + if (args.socketRef.current !== closedSocket) { + return + } + args.socketRef.current = null + if (args.lifecycle.disposed || args.lifecycle.terminalEnded) { + return + } + if (!args.lifecycle.attachedOnce) { + failBeforeAttach(args, "[websocket closed before attach]", "Terminal websocket closed before attach.") + return + } + scheduleReconnect(args) +} +const handleSocketError = ( + args: TerminalSocketConnectArgs, + failedSocket: WebSocket +): void => { + if (args.socketRef.current !== failedSocket || args.lifecycle.attachedOnce) { + return + } + failBeforeAttach(args, "[websocket error]", "Terminal websocket error.") +} +export const connectTerminalSocket = (args: TerminalSocketConnectArgs): void => { + if (args.lifecycle.disposed || args.lifecycle.terminalEnded) { + return + } + const socket = createTerminalSocket(args.session, args.terminal) + args.socketRef.current = socket + attachTerminalSocketListeners({ + lifecycle: args.lifecycle, + onClose: (closedSocket) => { + handleSocketClose(args, closedSocket) + }, + onError: (failedSocket) => { + handleSocketError(args, failedSocket) + }, + onMessage: (payload) => { + handleTerminalServerMessage(args.handlers, payload) + }, + onOpen: args.sendResize, + socket + }) +} diff --git a/packages/app/src/web/terminal-panel-runtime-types.ts b/packages/app/src/web/terminal-panel-runtime-types.ts new file mode 100644 index 00000000..036c6503 --- /dev/null +++ b/packages/app/src/web/terminal-panel-runtime-types.ts @@ -0,0 +1,72 @@ +import type { Terminal } from "xterm" +import type { FitAddon } from "xterm-addon-fit" + +import type { ActiveTerminalSession } from "./terminal.js" + +export type TerminalStatus = "attached" | "connecting" | "error" | "exited" | "reconnecting" + +export type TerminalConnectionState = { closing: boolean; opened: boolean } + +export type TerminalRuntime = { readonly fitAddon: FitAddon; readonly terminal: Terminal } + +export type TerminalLifecycleState = { + attachedOnce: boolean + disposed: boolean + readyNotified: boolean + reconnectAttempt: number + reconnectStartedAtMs: number | null + reconnectTimer: ReturnType | null + terminalEnded: boolean +} + +export type TerminalSocketRef = { current: WebSocket | null } + +export type TerminalMessageHandlers = { + readonly connectionRef: { current: TerminalConnectionState } + readonly lifecycle: TerminalLifecycleState + readonly notifyMessage: (message: string) => void + readonly session: ActiveTerminalSession + readonly setStatus: (status: TerminalStatus) => void + readonly terminal: Terminal +} + +export type TerminalCleanupArgs = { + readonly connectionRef: { current: TerminalConnectionState } + readonly lifecycle: TerminalLifecycleState + readonly notifyMessage: (message: string) => void + readonly removeInput: () => void + readonly removeResize: () => void + readonly resizeObserver: ResizeObserver | null + readonly session: ActiveTerminalSession + readonly socketRef: TerminalSocketRef + readonly terminal: Terminal +} + +export type TerminalLifecycleArgs = { + readonly connectionRef: { current: TerminalConnectionState } + readonly hostRef: { readonly current: HTMLDivElement | null } + readonly notifyMessage: (message: string) => void + readonly session: ActiveTerminalSession + readonly setStatus: (status: TerminalStatus) => void +} + +export type TerminalSocketListenerArgs = { + readonly lifecycle: TerminalLifecycleState + readonly onClose: (socket: WebSocket) => void + readonly onError: (socket: WebSocket) => void + readonly onMessage: (payload: string) => void + readonly onOpen: () => void + readonly socket: WebSocket +} + +export type TerminalSocketConnectArgs = { + readonly handlers: TerminalMessageHandlers + readonly lifecycle: TerminalLifecycleState + readonly notifyMessage: (message: string) => void + readonly reconnect: () => void + readonly sendResize: () => void + readonly session: ActiveTerminalSession + readonly setStatus: (status: TerminalStatus) => void + readonly socketRef: TerminalSocketRef + readonly terminal: Terminal +} diff --git a/packages/app/src/web/terminal-panel-runtime.ts b/packages/app/src/web/terminal-panel-runtime.ts index e00852db..877fb396 100644 --- a/packages/app/src/web/terminal-panel-runtime.ts +++ b/packages/app/src/web/terminal-panel-runtime.ts @@ -1,257 +1,49 @@ -import { Effect } from "effect" import { useEffect } from "react" -import { Terminal } from "xterm" -import { FitAddon } from "xterm-addon-fit" -import { deleteTerminalSessionByPath } from "./api.js" -import type { ActiveTerminalSession } from "./terminal.js" -import { parseTerminalServerMessage, resolveTerminalWebSocketUrl } from "./terminal.js" - -export type TerminalStatus = "attached" | "connecting" | "error" | "exited" - -export type TerminalConnectionState = { opened: boolean } - -type TerminalRuntime = { readonly fitAddon: FitAddon; readonly terminal: Terminal } - -type TerminalMessageHandlers = { - readonly notifyMessage: (message: string) => void - readonly session: ActiveTerminalSession - readonly setStatus: (status: TerminalStatus) => void - readonly terminal: Terminal -} - -type TerminalCleanupArgs = { - readonly removeInput: () => void - readonly removeResize: () => void - readonly resizeObserver: ResizeObserver | null - readonly socket: WebSocket - readonly terminal: Terminal -} - -type TerminalLifecycleArgs = { - readonly connectionRef: { current: TerminalConnectionState } - readonly hostRef: { readonly current: HTMLDivElement | null } - readonly notifyMessage: (message: string) => void - readonly session: ActiveTerminalSession - readonly setStatus: (status: TerminalStatus) => void -} - -type TerminalSocketListenerArgs = { - readonly connectionRef: { current: TerminalConnectionState } - readonly onClose: () => void - readonly onError: () => void - readonly onMessage: (payload: string) => void - readonly onOpen: () => void - readonly socket: WebSocket -} - -type TerminalSocketFailureHandlerArgs = { - readonly notifyMessage: (message: string) => void - readonly setStatus: (status: TerminalStatus) => void - readonly terminal: Terminal - readonly terminalLine: string - readonly uiMessage: string -} - -type TerminalSessionSocketArgs = { - readonly connectionRef: { current: TerminalConnectionState } - readonly handlers: TerminalMessageHandlers - readonly notifyMessage: (message: string) => void +import { + attachTerminalInput, + cleanupTerminalResources, + connectTerminalSocket, + createLifecycleState, + createMessageHandlers, + createTerminalRuntime, + observeTerminalResize, + sendTerminalResize +} from "./terminal-panel-runtime-core.js" +import type { + TerminalLifecycleArgs, + TerminalSocketConnectArgs, + TerminalSocketRef +} from "./terminal-panel-runtime-types.js" + +type TerminalCleanupFactoryArgs = { + readonly cleanupArgs: Omit[0], "removeInput" | "removeResize"> + readonly inputDisposable: { readonly dispose: () => void } readonly sendResize: () => void - readonly setStatus: (status: TerminalStatus) => void - readonly socket: WebSocket - readonly terminal: Terminal -} - -const requestSessionClose = (closePath: string): void => { - void Effect.runPromise(deleteTerminalSessionByPath(closePath).pipe(Effect.either, Effect.asVoid)) -} - -const createTerminalRuntime = (host: HTMLDivElement): TerminalRuntime => { - const terminal = new Terminal({ - convertEol: false, - cursorBlink: true, - fontFamily: "'IBM Plex Mono', 'SFMono-Regular', monospace", - fontSize: 14, - theme: { - background: "#080a0d", - foreground: "#f4f7fb" - } - }) - const fitAddon = new FitAddon() - terminal.loadAddon(fitAddon) - terminal.open(host) - fitAddon.fit() - terminal.focus() - return { fitAddon, terminal } -} - -const createTerminalSocket = ( - session: ActiveTerminalSession, - terminal: Terminal -): WebSocket => new WebSocket(resolveTerminalWebSocketUrl(session.websocketPath, terminal.cols, terminal.rows)) - -const sendTerminalResize = ( - fitAddon: FitAddon, - socket: WebSocket, - terminal: Terminal -): void => { - fitAddon.fit() - if (socket.readyState !== WebSocket.OPEN) { - return - } - socket.send(JSON.stringify({ - cols: terminal.cols, - rows: terminal.rows, - type: "resize" - })) -} - -const observeTerminalResize = ( - host: HTMLDivElement, - onResize: () => void -): ResizeObserver | null => { - if (typeof ResizeObserver !== "function") { - return null - } - const resizeObserver = new ResizeObserver(onResize) - resizeObserver.observe(host) - return resizeObserver -} - -const attachTerminalInput = ( - terminal: Terminal, - socket: WebSocket -) => - terminal.onData((data) => { - if (socket.readyState !== WebSocket.OPEN) { - return - } - socket.send(JSON.stringify({ data, type: "input" })) - }) - -const handleTerminalServerMessage = ( - handlers: TerminalMessageHandlers, - payload: string -): void => { - const message = parseTerminalServerMessage(payload) - if (message === null) { - handlers.terminal.writeln("\r\n[terminal protocol error]") - handlers.setStatus("error") - handlers.notifyMessage("Terminal protocol error.") - return - } - if (message.type === "ready") { - handlers.setStatus("attached") - handlers.notifyMessage(handlers.session.readyMessage) - handlers.session.onReady?.() - return - } - if (message.type === "output") { - handlers.terminal.write(message.data) - return - } - if (message.type === "error") { - handlers.terminal.writeln(`\r\n[error] ${message.message}`) - handlers.setStatus("error") - handlers.notifyMessage(message.message) - return - } - handlers.terminal.writeln("\r\n[session ended]") - handlers.setStatus("exited") - handlers.notifyMessage(handlers.session.exitMessage) - handlers.session.onExit?.() } -const attachTerminalSocketListeners = ( - { connectionRef, onClose, onError, onMessage, onOpen, socket }: TerminalSocketListenerArgs -): void => { - socket.addEventListener("open", () => { - connectionRef.current.opened = true - onOpen() - }) - socket.addEventListener("message", (event) => { - onMessage(typeof event.data === "string" ? event.data : "") - }) - socket.addEventListener("close", () => { - if (!connectionRef.current.opened) { - onClose() +const createTerminalCleanup = ( + { cleanupArgs, inputDisposable, sendResize }: TerminalCleanupFactoryArgs +): () => void => +(): void => { + cleanupTerminalResources({ + ...cleanupArgs, + removeInput: () => { + inputDisposable.dispose() + }, + removeResize: () => { + globalThis.removeEventListener("resize", sendResize) } }) - socket.addEventListener("error", onError) } -const cleanupTerminalResources = ( - { removeInput, removeResize, resizeObserver, socket, terminal }: TerminalCleanupArgs -): void => { - removeInput() - resizeObserver?.disconnect() - removeResize() - if (socket.readyState === WebSocket.OPEN) { - socket.send(JSON.stringify({ type: "close" })) +const createConnectSocket = ( + args: Omit +): () => void => { + const connectSocket = () => { + connectTerminalSocket({ ...args, reconnect: connectSocket }) } - socket.close() - terminal.dispose() -} - -const createMessageHandlers = ( - notifyMessage: (message: string) => void, - session: ActiveTerminalSession, - setStatus: (status: TerminalStatus) => void, - terminal: Terminal -): TerminalMessageHandlers => ({ - notifyMessage, - session, - setStatus, - terminal -}) - -const createSocketFailureHandler = ( - { notifyMessage, setStatus, terminal, terminalLine, uiMessage }: TerminalSocketFailureHandlerArgs -) => -(): void => { - terminal.writeln(`\r\n${terminalLine}`) - setStatus("error") - notifyMessage(uiMessage) -} - -const maybeDeletePendingSession = ( - connectionRef: { current: TerminalConnectionState }, - notifyMessage: (message: string) => void, - session: ActiveTerminalSession -): void => { - if (!connectionRef.current.opened) { - requestSessionClose(session.closePath) - notifyMessage(session.pendingDeleteMessage) - session.onExit?.() - } -} - -const attachTerminalSessionSocket = ( - { connectionRef, handlers, notifyMessage, sendResize, setStatus, socket, terminal }: TerminalSessionSocketArgs -): void => { - attachTerminalSocketListeners({ - connectionRef, - onClose: createSocketFailureHandler({ - notifyMessage, - setStatus, - terminal, - terminalLine: "[websocket closed before attach]", - uiMessage: "Terminal websocket closed before attach." - }), - onError: createSocketFailureHandler({ - notifyMessage, - setStatus, - terminal, - terminalLine: "[websocket error]", - uiMessage: "Terminal websocket error." - }), - onMessage: (payload) => { - handleTerminalServerMessage(handlers, payload) - }, - onOpen: sendResize, - socket - }) + return connectSocket } const mountTerminalSession = ( @@ -262,46 +54,50 @@ const mountTerminalSession = ( return undefined } - connectionRef.current = { opened: false } + connectionRef.current = { closing: false, opened: false } + const lifecycle = createLifecycleState() + const socketRef: TerminalSocketRef = { current: null } const { fitAddon, terminal } = createTerminalRuntime(host) - const socket = createTerminalSocket(session, terminal) const sendResize = () => { - sendTerminalResize(fitAddon, socket, terminal) + sendTerminalResize(fitAddon, socketRef, terminal) } const resizeObserver = observeTerminalResize(host, sendResize) - const inputDisposable = attachTerminalInput(terminal, socket) - const handlers = createMessageHandlers( + const inputDisposable = attachTerminalInput(terminal, socketRef) + const handlers = createMessageHandlers({ + connectionRef, + lifecycle, notifyMessage, session, setStatus, terminal - ) - - globalThis.addEventListener("resize", sendResize) - attachTerminalSessionSocket({ - connectionRef, + }) + const connectSocket = createConnectSocket({ handlers, + lifecycle, notifyMessage, sendResize, + session, setStatus, - socket, + socketRef, terminal }) - return () => { - cleanupTerminalResources({ - removeInput: () => { - inputDisposable.dispose() - }, - removeResize: () => { - globalThis.removeEventListener("resize", sendResize) - }, + globalThis.addEventListener("resize", sendResize) + connectSocket() + + return createTerminalCleanup({ + cleanupArgs: { + connectionRef, + lifecycle, + notifyMessage, resizeObserver, - socket, + session, + socketRef, terminal - }) - maybeDeletePendingSession(connectionRef, notifyMessage, session) - } + }, + inputDisposable, + sendResize + }) } export const useTerminalSessionLifecycle = ( @@ -315,5 +111,7 @@ export const useTerminalSessionLifecycle = ( session, setStatus }) - }, [connectionRef, hostRef, session, setStatus]) + }, [connectionRef, hostRef, notifyMessage, session, setStatus]) } + +export { type TerminalConnectionState, type TerminalStatus } from "./terminal-panel-runtime-types.js" diff --git a/packages/app/src/web/terminal-reconnect.ts b/packages/app/src/web/terminal-reconnect.ts new file mode 100644 index 00000000..6e4ca7db --- /dev/null +++ b/packages/app/src/web/terminal-reconnect.ts @@ -0,0 +1,7 @@ +export const terminalReconnectGraceMs = 60_000 + +const reconnectBaseDelayMs = 500 +const reconnectMaxDelayMs = 3000 + +export const resolveTerminalReconnectDelay = (attempt: number): number => + Math.min(reconnectBaseDelayMs * (2 ** Math.max(0, attempt)), reconnectMaxDelayMs) diff --git a/packages/app/tests/docker-git/app-dashboard-state.test.ts b/packages/app/tests/docker-git/app-dashboard-state.test.ts new file mode 100644 index 00000000..2e7fed0a --- /dev/null +++ b/packages/app/tests/docker-git/app-dashboard-state.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "@effect/vitest" + +import type { DashboardData } from "../../src/web/api.js" +import { type DashboardState, mergeDashboardRefreshState } from "../../src/web/app-dashboard-state.js" + +const dashboard: DashboardData = { + apiBaseUrl: "/api", + health: { + cwd: "/home/dev", + ok: true, + projectsRoot: "/home/dev/workspaces", + revision: "rev" + }, + projects: [] +} + +describe("web dashboard state", () => { + it("keeps the ready screen mounted when a background refresh fails", () => { + const ready: DashboardState = { + _tag: "Ready", + dashboard, + refreshedAtMs: 10 + } + const error: DashboardState = { + _tag: "Error", + apiBaseUrl: "/api", + message: "temporary tunnel failure" + } + + expect(mergeDashboardRefreshState(ready, error)).toBe(ready) + }) + + it("still surfaces initial loading failures", () => { + const loading: DashboardState = { + _tag: "Loading", + apiBaseUrl: "/api" + } + const error: DashboardState = { + _tag: "Error", + apiBaseUrl: "/api", + message: "api unavailable" + } + + expect(mergeDashboardRefreshState(loading, error)).toBe(error) + }) +}) diff --git a/packages/app/tests/docker-git/terminal.test.ts b/packages/app/tests/docker-git/terminal.test.ts index a86a356e..39792f00 100644 --- a/packages/app/tests/docker-git/terminal.test.ts +++ b/packages/app/tests/docker-git/terminal.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "@effect/vitest" import { afterEach, beforeEach, vi } from "vitest" +import { resolveTerminalReconnectDelay } from "../../src/web/terminal-reconnect.js" import { parseTerminalServerMessage, resolveTerminalWebSocketUrl } from "../../src/web/terminal.js" import type { TerminalServerMessage } from "../../src/web/terminal.js" @@ -66,4 +67,14 @@ describe("browser terminal helpers", () => { it("rejects malformed terminal messages", () => { expect(parseTerminalServerMessage("{\"type\":\"output\",\"data\":1}")).toBeNull() }) + + it("caps reconnect backoff inside the server reconnect grace window", () => { + expect([ + resolveTerminalReconnectDelay(-1), + resolveTerminalReconnectDelay(0), + resolveTerminalReconnectDelay(1), + resolveTerminalReconnectDelay(2), + resolveTerminalReconnectDelay(3) + ]).toEqual([500, 500, 1000, 2000, 3000]) + }) }) diff --git a/packages/app/vite.web.config.ts b/packages/app/vite.web.config.ts index 1695398f..bf5bbcdb 100644 --- a/packages/app/vite.web.config.ts +++ b/packages/app/vite.web.config.ts @@ -15,6 +15,7 @@ const noStoreHeaders = { "Cache-Control": "no-store, no-cache, must-revalidate, max-age=0", Pragma: "no-cache" } +const webSocketHeartbeatIntervalMs = 25_000 const createProxy = (apiTarget: string) => ({ "/b": { @@ -60,6 +61,31 @@ const proxyForwardHeaders = (request: IncomingMessage): Record = } } +const attachWebSocketHeartbeat = (socket: WebSocket): void => { + let alive = true + const interval = setInterval(() => { + if (socket.readyState !== WebSocket.OPEN) { + return + } + if (!alive) { + socket.terminate() + return + } + alive = false + socket.ping() + }, webSocketHeartbeatIntervalMs) + + socket.on("pong", () => { + alive = true + }) + socket.on("close", () => { + clearInterval(interval) + }) + socket.on("error", () => { + clearInterval(interval) + }) +} + const bridgeWebSockets = (clientSocket: WebSocket, upstream: WebSocket): void => { const pending: Array<{ readonly data: RawData; readonly isBinary: boolean }> = [] const sendWhenOpen = (socket: WebSocket, data: RawData, isBinary: boolean): void => { @@ -67,6 +93,8 @@ const bridgeWebSockets = (clientSocket: WebSocket, upstream: WebSocket): void => socket.send(data, { binary: isBinary }) } } + attachWebSocketHeartbeat(clientSocket) + attachWebSocketHeartbeat(upstream) const flushPending = (): void => { for (const message of pending.splice(0)) { sendWhenOpen(upstream, message.data, message.isBinary) diff --git a/packages/lib/src/usecases/projects-ssh.ts b/packages/lib/src/usecases/projects-ssh.ts index b1a4acda..c326e8ac 100644 --- a/packages/lib/src/usecases/projects-ssh.ts +++ b/packages/lib/src/usecases/projects-ssh.ts @@ -54,6 +54,10 @@ const buildSshArgs = (item: ProjectItem): ReadonlyArray => { "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", + "ServerAliveInterval=30", + "-o", + "ServerAliveCountMax=3", "-p", String(port), `${item.sshUser}@${host}` diff --git a/packages/lib/tests/usecases/projects-ssh.test.ts b/packages/lib/tests/usecases/projects-ssh.test.ts new file mode 100644 index 00000000..97ae040e --- /dev/null +++ b/packages/lib/tests/usecases/projects-ssh.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "@effect/vitest" + +import type { ProjectItem } from "../../src/usecases/projects-core.js" +import { prepareProjectSsh } from "../../src/usecases/projects-ssh.js" + +const projectItem: ProjectItem = { + authorizedKeysExists: true, + authorizedKeysPath: "/tmp/project/.ssh/authorized_keys", + codexAuthPath: "/tmp/auth.json", + codexHome: "/tmp/codex", + containerName: "project-container", + displayName: "org/repo", + envGlobalPath: "/tmp/.env", + envProjectPath: "/tmp/project/.env", + projectDir: "/tmp/project", + repoRef: "main", + repoUrl: "https://example.com/org/repo.git", + serviceName: "app", + sshCommand: "ssh -p 2222 dev@localhost", + sshKeyPath: "/tmp/key", + sshPort: 2222, + sshUser: "dev", + targetDir: "/workspace" +} + +describe("project ssh preparation", () => { + it("adds ssh keepalive options for interactive sessions", () => { + const prepared = prepareProjectSsh(projectItem) + + expect(prepared.args).toContain("ServerAliveInterval=30") + expect(prepared.args).toContain("ServerAliveCountMax=3") + expect(prepared.args).toEqual([ + "-i", + "/tmp/key", + "-tt", + "-Y", + "-o", + "LogLevel=ERROR", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "ServerAliveInterval=30", + "-o", + "ServerAliveCountMax=3", + "-p", + "2222", + "dev@localhost" + ]) + }) +})