From ae73519cad28e0e557353a3907d0a3a201a554a1 Mon Sep 17 00:00:00 2001 From: "{ \"message\": \"Bad credentials\", \"documentation_url\": \"https://docs.github.com/rest\", \"status\": \"401\"}" <{ "message": "Bad credentials", "documentation_url": "https://docs.github.com/rest", "status": "401"}+{ "message": "Bad credentials", "documentation_url": "https://docs.github.com/rest", "status": "401"}@users.noreply.github.com> Date: Thu, 16 Apr 2026 20:12:49 +0000 Subject: [PATCH 1/2] feat: stream project creation logs --- packages/api/src/api/contracts.ts | 9 + packages/api/src/api/schema.ts | 3 +- packages/api/src/http.ts | 8 +- packages/api/src/services/projects.ts | 301 +++++++++++------- packages/api/tests/projects.test.ts | 68 ++++ packages/api/tests/schema.test.ts | 4 +- .../app/src/docker-git/api-client-create.ts | 10 +- .../app/src/docker-git/api-client-events.ts | 98 +++++- packages/app/src/docker-git/api-client.ts | 47 ++- .../app/src/docker-git/api-project-codec.ts | 25 ++ packages/app/src/lib/shell/command-runner.ts | 80 +++++ packages/app/src/lib/shell/docker-compose.ts | 4 +- packages/app/src/web/actions-output.ts | 22 ++ .../app/src/web/actions-project-create.ts | 102 ++++++ packages/app/src/web/actions-projects.ts | 48 +-- packages/app/src/web/actions-shared.ts | 2 +- packages/app/src/web/api-create-project.ts | 13 + packages/app/src/web/api-project-core.ts | 40 +++ packages/app/src/web/api-schema.ts | 7 + packages/app/src/web/api.ts | 42 +-- packages/app/src/web/project-events.ts | 10 +- .../docker-git/api-project-codec.test.ts | 23 ++ packages/lib/src/shell/command-runner.ts | 80 +++++ packages/lib/src/shell/docker.ts | 6 +- 24 files changed, 841 insertions(+), 211 deletions(-) create mode 100644 packages/app/src/web/actions-output.ts create mode 100644 packages/app/src/web/actions-project-create.ts create mode 100644 packages/app/src/web/api-create-project.ts create mode 100644 packages/app/src/web/api-project-core.ts create mode 100644 packages/app/tests/docker-git/api-project-codec.test.ts diff --git a/packages/api/src/api/contracts.ts b/packages/api/src/api/contracts.ts index e1b55070..5b01217a 100644 --- a/packages/api/src/api/contracts.ts +++ b/packages/api/src/api/contracts.ts @@ -34,6 +34,14 @@ export type ProjectDetails = ProjectSummary & { readonly codexHome: string } +export type CreateProjectAccepted = { + readonly accepted: true + readonly projectId: string + readonly cursor: number +} + +export type CreateProjectResult = ProjectDetails | CreateProjectAccepted + export type ProjectPortForwardStatus = "running" | "stopped" | "unknown" export type ProjectPortForward = { @@ -295,6 +303,7 @@ export type CreateProjectRequest = { readonly force?: boolean | undefined readonly forceEnv?: boolean | undefined readonly waitForClone?: boolean | undefined + readonly async?: boolean | undefined } export type AgentEnvVar = { diff --git a/packages/api/src/api/schema.ts b/packages/api/src/api/schema.ts index 1236a233..8e23efb8 100644 --- a/packages/api/src/api/schema.ts +++ b/packages/api/src/api/schema.ts @@ -36,7 +36,8 @@ export const CreateProjectRequestSchema = Schema.Struct({ openSsh: OptionalBoolean, force: OptionalBoolean, forceEnv: OptionalBoolean, - waitForClone: OptionalBoolean + waitForClone: OptionalBoolean, + async: OptionalBoolean }) export const GithubAuthLoginRequestSchema = Schema.Struct({ diff --git a/packages/api/src/http.ts b/packages/api/src/http.ts index 571a9c1c..bd309960 100644 --- a/packages/api/src/http.ts +++ b/packages/api/src/http.ts @@ -687,8 +687,12 @@ export const makeRouter = () => { "/projects", Effect.gen(function*(_) { const request = yield* _(readCreateProjectRequest()) - const project = yield* _(createProjectFromRequest(request)) - return yield* _(jsonResponse({ project }, 201)) + const result = yield* _(createProjectFromRequest(request)) + return yield* _( + "accepted" in result && result.accepted === true + ? jsonResponse(result, 202) + : jsonResponse({ project: result }, 201) + ) }).pipe(Effect.catchAll(errorResponse)) ), HttpRouter.post( diff --git a/packages/api/src/services/projects.ts b/packages/api/src/services/projects.ts index 93052dd9..6dfa718d 100644 --- a/packages/api/src/services/projects.ts +++ b/packages/api/src/services/projects.ts @@ -22,10 +22,16 @@ import type { CreateCommand as LibCreateCommand } from "@effect-template/lib/cor import type { ProjectItem } from "@effect-template/lib/usecases/projects" import { Effect, Either, Logger } from "effect" -import type { CreateProjectRequest, ProjectDetails, ProjectStatus, ProjectSummary } from "../api/contracts.js" +import type { + CreateProjectAccepted, + CreateProjectRequest, + ProjectDetails, + ProjectStatus, + ProjectSummary +} from "../api/contracts.js" import { ApiAuthRequiredError, ApiConflictError, ApiInternalError, ApiNotFoundError, ApiBadRequestError } from "../api/errors.js" import { ensureGithubAuthForCreate } from "./auth.js" -import { emitProjectEvent } from "./events.js" +import { clearProjectEvents, emitProjectEvent, latestProjectCursor } from "./events.js" import { resolveCreateAuthorizedKeysContents, resolveManagedAuthorizedKeysContents } from "./project-authorized-keys.js" import { projectShortKey } from "./project-port-proxy-core.js" import { loadProjectRuntimeByProject, runtimeForProject } from "./project-runtime.js" @@ -66,7 +72,7 @@ const runComposeCapture = ( Effect.tap((output) => Effect.sync(() => { for (const line of output.split(/\r?\n/u)) { - const trimmed = line.trimEnd() + const trimmed = redactProjectLogLine(line.trimEnd()) if (trimmed.length > 0) { emitProjectEvent(projectId, "project.deployment.log", { line: trimmed, @@ -85,7 +91,7 @@ const runWithProjectEventLogs = ( Effect.gen(function*(_) { const logger = Logger.make(({ message }) => { for (const line of String(message).split(/\r?\n/u)) { - const trimmed = line.trimEnd() + const trimmed = redactProjectLogLine(line.trimEnd()) if (trimmed.length > 0) { emitProjectEvent(projectId, "project.deployment.log", { line: trimmed }) } @@ -95,6 +101,11 @@ const runWithProjectEventLogs = ( return yield* _(effect.pipe(Effect.provide(Logger.replace(Logger.defaultLogger, logger)))) }) +const redactProjectLogLine = (line: string): string => + line + .replace(/https:\/\/([^:\s/@]+):([^@\s]+)@/gu, "https://$1:***@") + .replace(/\b(GH_TOKEN|GITHUB_TOKEN|GIT_AUTH_TOKEN)=\S+/gu, "$1=***") + const toProjectStatus = (raw: string): ProjectStatus => { const normalized = raw.toLowerCase() if (normalized.includes("up") || normalized.includes("running")) { @@ -295,6 +306,179 @@ export const seedAuthorizedKeysForCreate = ( } }) +type PreparedCreateProject = { + readonly command: LibCreateCommand + readonly projectId: string +} + +const toCreateRawOptions = (request: CreateProjectRequest): RawOptions => ({ + ...(request.repoUrl === undefined ? {} : { repoUrl: request.repoUrl }), + ...(request.repoRef === undefined ? {} : { repoRef: request.repoRef }), + ...(request.targetDir === undefined ? {} : { targetDir: request.targetDir }), + ...(request.sshPort === undefined ? {} : { sshPort: request.sshPort }), + ...(request.sshUser === undefined ? {} : { sshUser: request.sshUser }), + ...(request.containerName === undefined ? {} : { containerName: request.containerName }), + ...(request.serviceName === undefined ? {} : { serviceName: request.serviceName }), + ...(request.volumeName === undefined ? {} : { volumeName: request.volumeName }), + ...(request.secretsRoot === undefined ? {} : { secretsRoot: request.secretsRoot }), + ...(request.authorizedKeysPath === undefined ? {} : { authorizedKeysPath: request.authorizedKeysPath }), + ...(request.envGlobalPath === undefined ? {} : { envGlobalPath: request.envGlobalPath }), + ...(request.envProjectPath === undefined ? {} : { envProjectPath: request.envProjectPath }), + ...(request.codexAuthPath === undefined ? {} : { codexAuthPath: request.codexAuthPath }), + ...(request.codexHome === undefined ? {} : { codexHome: request.codexHome }), + ...(request.cpuLimit === undefined ? {} : { cpuLimit: request.cpuLimit }), + ...(request.ramLimit === undefined ? {} : { ramLimit: request.ramLimit }), + ...(request.dockerNetworkMode === undefined ? {} : { dockerNetworkMode: request.dockerNetworkMode }), + ...(request.dockerSharedNetworkName === undefined + ? {} + : { dockerSharedNetworkName: request.dockerSharedNetworkName }), + ...(request.enableMcpPlaywright === undefined ? {} : { enableMcpPlaywright: request.enableMcpPlaywright }), + ...(request.outDir === undefined ? {} : { outDir: request.outDir }), + ...(request.gitTokenLabel === undefined ? {} : { gitTokenLabel: request.gitTokenLabel }), + ...(request.skipGithubAuth === undefined ? {} : { skipGithubAuth: request.skipGithubAuth }), + ...(request.codexTokenLabel === undefined ? {} : { codexTokenLabel: request.codexTokenLabel }), + ...(request.claudeTokenLabel === undefined ? {} : { claudeTokenLabel: request.claudeTokenLabel }), + ...(request.agentAutoMode === undefined ? {} : { agentAutoMode: request.agentAutoMode }), + ...(request.up === undefined ? {} : { up: request.up }), + ...(request.openSsh === undefined ? {} : { openSsh: request.openSsh }), + ...(request.force === undefined ? {} : { force: request.force }), + ...(request.forceEnv === undefined ? {} : { forceEnv: request.forceEnv }) +}) + +const parseCreateCommandRequest = ( + request: CreateProjectRequest +): Effect.Effect => { + const parsed = buildCreateCommand(toCreateRawOptions(request)) + if (Either.isLeft(parsed)) { + return Effect.fail( + new ApiBadRequestError({ + message: "Invalid create payload.", + details: formatParseError(parsed.left) + }) + ) + } + + return Effect.succeed({ + ...parsed.right, + openSsh: false, + waitForClone: request.waitForClone ?? parsed.right.waitForClone + }) +} + +const prepareCreateProjectRequest = ( + request: CreateProjectRequest +) => + Effect.gen(function*(_) { + const parsedCommand = yield* _(parseCreateCommandRequest(request)) + const requestAuthorizedKeysContents = request.authorizedKeysContents ?? ( + request.useManagedAuthorizedKeys === true + ? yield* _(resolveCreateAuthorizedKeysContents(parsedCommand.outDir, parsedCommand.config.authorizedKeysPath)) + : undefined + ) + const resolvedAuthorizedKeysContents = yield* _( + resolveRequestedAuthorizedKeysContents(requestAuthorizedKeysContents, request.useManagedAuthorizedKeys === true) + ) + + const command = withManagedAuthorizedKeysForCreate(parsedCommand, resolvedAuthorizedKeysContents) + + yield* _(seedAuthorizedKeysForCreate(command.outDir, resolvedAuthorizedKeysContents)) + yield* _(ensureGithubAuthForCreate(command.config)) + + return { + command, + projectId: command.outDir + } + }) + +const emitCreateStatus = ( + projectId: string, + phase: string, + message: string +) => + Effect.sync(() => { + emitProjectEvent(projectId, "project.deployment.status", { phase, message }) + }) + +const emitProjectCreatedEvents = ( + projectId: string, + project: ProjectItem +) => + Effect.sync(() => { + const payload = { + projectId: project.projectDir, + containerName: project.containerName + } + emitProjectEvent(project.projectDir, "project.created", payload) + if (project.projectDir !== projectId) { + emitProjectEvent(projectId, "project.created", payload) + } + }) + +const toCreateFailureMessage = (error: ProjectApiError): string => + error instanceof ApiAuthRequiredError || + error instanceof ApiBadRequestError || + error instanceof ApiConflictError || + error instanceof ApiInternalError || + error instanceof ApiNotFoundError + ? error.message + : renderError(error) + +const runPreparedCreateProject = ( + prepared: PreparedCreateProject +) => + Effect.gen(function*(_) { + const { command, projectId } = prepared + + yield* _( + runWithProjectEventLogs( + projectId, + createProject(command).pipe( + Effect.catchTag("DockerIdentityConflictError", (error) => + Effect.fail(new ApiConflictError({ message: renderError(error) })) + ) + ) + ) + ) + + const project = yield* _( + resolveCreatedProject( + command.config.containerName, + command.config.repoUrl, + command.config.repoRef + ) + ) + const runtimeByProject = yield* _(loadProjectRuntimeByProject([project])) + const summary = yield* _(withProjectRuntime(project, runtimeForProject(runtimeByProject, project))) + + yield* _(emitProjectCreatedEvents(projectId, project)) + + return toProjectDetails(project, summary) + }).pipe(Effect.mapError(toProjectApiError)) + +const startCreateProjectJob = ( + prepared: PreparedCreateProject +) => + Effect.gen(function*(_) { + clearProjectEvents(prepared.projectId) + const cursor = latestProjectCursor(prepared.projectId) + yield* _(emitCreateStatus(prepared.projectId, "create", "Project creation started")) + yield* _( + runPreparedCreateProject(prepared).pipe( + Effect.matchEffect({ + onFailure: (error) => + emitCreateStatus(prepared.projectId, "failed", toCreateFailureMessage(error)), + onSuccess: () => Effect.void + }), + Effect.forkDaemon + ) + ) + return { + accepted: true, + projectId: prepared.projectId, + cursor + } satisfies CreateProjectAccepted + }) + export const listProjects = () => listProjectItems.pipe( Effect.flatMap((projects) => @@ -343,111 +527,14 @@ export const createProjectFromRequest = ( request: CreateProjectRequest ) => Effect.gen(function*(_) { - const raw: RawOptions = { - ...(request.repoUrl === undefined ? {} : { repoUrl: request.repoUrl }), - ...(request.repoRef === undefined ? {} : { repoRef: request.repoRef }), - ...(request.targetDir === undefined ? {} : { targetDir: request.targetDir }), - ...(request.sshPort === undefined ? {} : { sshPort: request.sshPort }), - ...(request.sshUser === undefined ? {} : { sshUser: request.sshUser }), - ...(request.containerName === undefined ? {} : { containerName: request.containerName }), - ...(request.serviceName === undefined ? {} : { serviceName: request.serviceName }), - ...(request.volumeName === undefined ? {} : { volumeName: request.volumeName }), - ...(request.secretsRoot === undefined ? {} : { secretsRoot: request.secretsRoot }), - ...(request.authorizedKeysPath === undefined ? {} : { authorizedKeysPath: request.authorizedKeysPath }), - ...(request.envGlobalPath === undefined ? {} : { envGlobalPath: request.envGlobalPath }), - ...(request.envProjectPath === undefined ? {} : { envProjectPath: request.envProjectPath }), - ...(request.codexAuthPath === undefined ? {} : { codexAuthPath: request.codexAuthPath }), - ...(request.codexHome === undefined ? {} : { codexHome: request.codexHome }), - ...(request.cpuLimit === undefined ? {} : { cpuLimit: request.cpuLimit }), - ...(request.ramLimit === undefined ? {} : { ramLimit: request.ramLimit }), - ...(request.dockerNetworkMode === undefined ? {} : { dockerNetworkMode: request.dockerNetworkMode }), - ...(request.dockerSharedNetworkName === undefined ? {} : { dockerSharedNetworkName: request.dockerSharedNetworkName }), - ...(request.enableMcpPlaywright === undefined ? {} : { enableMcpPlaywright: request.enableMcpPlaywright }), - ...(request.outDir === undefined ? {} : { outDir: request.outDir }), - ...(request.gitTokenLabel === undefined ? {} : { gitTokenLabel: request.gitTokenLabel }), - ...(request.skipGithubAuth === undefined ? {} : { skipGithubAuth: request.skipGithubAuth }), - ...(request.codexTokenLabel === undefined ? {} : { codexTokenLabel: request.codexTokenLabel }), - ...(request.claudeTokenLabel === undefined ? {} : { claudeTokenLabel: request.claudeTokenLabel }), - ...(request.agentAutoMode === undefined ? {} : { agentAutoMode: request.agentAutoMode }), - ...(request.up === undefined ? {} : { up: request.up }), - ...(request.openSsh === undefined ? {} : { openSsh: request.openSsh }), - ...(request.force === undefined ? {} : { force: request.force }), - ...(request.forceEnv === undefined ? {} : { forceEnv: request.forceEnv }) + const prepared = yield* _(prepareCreateProjectRequest(request).pipe(Effect.mapError(toProjectApiError))) + if (request.async === true) { + return yield* _(startCreateProjectJob(prepared)) } - const parsed = buildCreateCommand(raw) - if (Either.isLeft(parsed)) { - return yield* _( - Effect.fail( - new ApiBadRequestError({ - message: "Invalid create payload.", - details: formatParseError(parsed.left) - }) - ) - ) - } - - const parsedCommand = { - ...parsed.right, - openSsh: false, - waitForClone: request.waitForClone ?? parsed.right.waitForClone - } - - const requestAuthorizedKeysContents = request.authorizedKeysContents ?? ( - request.useManagedAuthorizedKeys === true - ? yield* _(resolveCreateAuthorizedKeysContents(parsedCommand.outDir, parsedCommand.config.authorizedKeysPath)) - : undefined - ) - const resolvedAuthorizedKeysContents = yield* _( - resolveRequestedAuthorizedKeysContents(requestAuthorizedKeysContents, request.useManagedAuthorizedKeys === true) - ) - - const command = withManagedAuthorizedKeysForCreate(parsedCommand, resolvedAuthorizedKeysContents) - - yield* _(seedAuthorizedKeysForCreate(command.outDir, resolvedAuthorizedKeysContents)) - - yield* _(ensureGithubAuthForCreate(command.config)) - - yield* _( - Effect.sync(() => { - emitProjectEvent(command.outDir, "project.deployment.status", { - phase: "create", - message: "Project creation started" - }) - }) - ) - - yield* _( - runWithProjectEventLogs( - command.outDir, - createProject(command).pipe( - Effect.catchTag("DockerIdentityConflictError", (error) => - Effect.fail(new ApiConflictError({ message: renderError(error) })) - ) - ) - ) - ) - - const project = yield* _( - resolveCreatedProject( - command.config.containerName, - command.config.repoUrl, - command.config.repoRef - ) - ) - const runtimeByProject = yield* _(loadProjectRuntimeByProject([project])) - const summary = yield* _(withProjectRuntime(project, runtimeForProject(runtimeByProject, project))) - - yield* _( - Effect.sync(() => { - emitProjectEvent(project.projectDir, "project.created", { - projectId: project.projectDir, - containerName: project.containerName - }) - }) - ) - - return toProjectDetails(project, summary) + clearProjectEvents(prepared.projectId) + yield* _(emitCreateStatus(prepared.projectId, "create", "Project creation started")) + return yield* _(runPreparedCreateProject(prepared).pipe(Effect.mapError(toProjectApiError))) }).pipe(Effect.mapError(toProjectApiError)) export const deleteProjectById = ( diff --git a/packages/api/tests/projects.test.ts b/packages/api/tests/projects.test.ts index 3ba4bc3a..b1c9be6b 100644 --- a/packages/api/tests/projects.test.ts +++ b/packages/api/tests/projects.test.ts @@ -6,8 +6,10 @@ import { describe, expect, it } from "@effect/vitest" import { Effect } from "effect" import * as Scope from "effect/Scope" +import type { ApiEvent } from "../src/api/contracts.js" import { ApiConflictError, ApiInternalError } from "../src/api/errors.js" import { resolveManagedAuthorizedKeysContents } from "../src/services/project-authorized-keys.js" +import { listProjectEventsSince } from "../src/services/events.js" import { createProjectFromRequest, seedAuthorizedKeysForCreate } from "../src/services/projects.js" const withTempDir = ( @@ -92,6 +94,25 @@ const withEnvVar = ( ).pipe(Effect.flatMap(() => effect)) ) +const realSleep = (milliseconds: number): Effect.Effect => + Effect.promise(() => new Promise((resolve) => { + globalThis.setTimeout(resolve, milliseconds) + })) + +const waitForEvents = ( + projectId: string, + predicate: (events: ReadonlyArray) => boolean, + attempts: number +): Effect.Effect> => + Effect.gen(function*(_) { + const events = listProjectEventsSince(projectId, 0) + if (predicate(events) || attempts <= 0) { + return events + } + yield* _(realSleep(50)) + return yield* _(waitForEvents(projectId, predicate, attempts - 1)) + }) + describe("projects service", () => { it.effect("seeds host SSH keys into the controller managed authorized_keys file", () => withTempDir((root) => @@ -184,6 +205,53 @@ describe("projects service", () => { }) ).pipe(Effect.provide(NodeContext.layer))) + it.effect("accepts async create and records realtime lifecycle events on the request project id", () => + withTempDir((root) => + Effect.gen(function*(_) { + const path = yield* _(Path.Path) + const projectsRoot = path.join(root, ".docker-git") + const projectId = path.join(projectsRoot, "async", "realtime") + + yield* _( + withProjectsRoot( + projectsRoot, + withWorkingDirectory( + root, + Effect.gen(function*(_) { + const accepted = yield* _( + createProjectFromRequest({ + repoUrl: "https://git.example.test/test-owner/realtime.git", + repoRef: "main", + outDir: projectId, + skipGithubAuth: true, + up: false, + async: true + }) + ) + + expect(accepted).toMatchObject({ + accepted: true, + projectId, + cursor: 0 + }) + + const events = yield* _( + waitForEvents(projectId, (items) => items.some((event) => event.type === "project.created"), 20) + ) + + expect(events.map((event) => event.type)).toContain("project.deployment.status") + expect(events.map((event) => event.type)).toContain("project.created") + expect(events.find((event) => event.type === "project.deployment.status")?.payload).toMatchObject({ + phase: "create", + message: "Project creation started" + }) + }) + ) + ) + ) + }) + ).pipe(Effect.provide(NodeContext.layer))) + it.effect("maps duplicate docker identities to API conflict for create", () => withTempDir((root) => Effect.gen(function*(_) { diff --git a/packages/api/tests/schema.test.ts b/packages/api/tests/schema.test.ts index dad2e1d9..4185082f 100644 --- a/packages/api/tests/schema.test.ts +++ b/packages/api/tests/schema.test.ts @@ -32,7 +32,8 @@ describe("api schemas", () => { authorizedKeysContents: "ssh-ed25519 AAAA-test test@example\n", skipGithubAuth: true, up: true, - force: false + force: false, + async: true }) Either.match(result, { @@ -44,6 +45,7 @@ describe("api schemas", () => { expect(value.authorizedKeysContents).toContain("ssh-ed25519") expect(value.skipGithubAuth).toBe(true) expect(value.up).toBe(true) + expect(value.async).toBe(true) } }) })) diff --git a/packages/app/src/docker-git/api-client-create.ts b/packages/app/src/docker-git/api-client-create.ts index cdb4d466..2cecb494 100644 --- a/packages/app/src/docker-git/api-client-create.ts +++ b/packages/app/src/docker-git/api-client-create.ts @@ -6,9 +6,14 @@ type ResolvedCreateRequestPaths = { readonly authorizedKeysContents?: string | undefined } +type CreateProjectRequestOptions = { + readonly async?: boolean | undefined +} + export const buildCreateProjectRequest = ( command: CreateCommand, - resolvedPaths: ResolvedCreateRequestPaths + resolvedPaths: ResolvedCreateRequestPaths, + options?: CreateProjectRequestOptions ) => { const config = command.config return { @@ -44,6 +49,7 @@ export const buildCreateProjectRequest = ( openSsh: false, force: command.force, forceEnv: command.forceEnv, - waitForClone: command.waitForClone + waitForClone: command.waitForClone, + ...(options?.async === true ? { async: true } : {}) } satisfies JsonRequest } diff --git a/packages/app/src/docker-git/api-client-events.ts b/packages/app/src/docker-git/api-client-events.ts index 6ae954c1..3f7b6c8a 100644 --- a/packages/app/src/docker-git/api-client-events.ts +++ b/packages/app/src/docker-git/api-client-events.ts @@ -3,12 +3,14 @@ import * as Fiber from "effect/Fiber" import { request } from "./api-http.js" import { asArray, asObject, asString, type JsonValue } from "./api-json.js" +import type { ControllerRuntime } from "./controller.js" +import type { ApiAuthRequiredError, ApiRequestError } from "./host-errors.js" import { formatProjectEventLine } from "./project-event-lines.js" const projectPath = (projectId: string, suffix = ""): string => `/projects/${encodeURIComponent(projectId)}${suffix}` const projectEventPollInterval = Duration.millis(250) -type ProjectEvent = { +export type ProjectEvent = { readonly seq: number readonly type: string readonly payload: JsonValue | undefined @@ -19,6 +21,8 @@ type ProjectEventPollResponse = { readonly events: ReadonlyArray } +type ProjectCreationWaitError = ApiAuthRequiredError | ApiRequestError + export type ProjectEventPolling = { readonly cursorRef: Ref.Ref readonly fiber: Fiber.RuntimeFiber @@ -76,6 +80,38 @@ const writeProjectEventLines = (events: ReadonlyArray) => } }) +const readProjectEventPayloadField = ( + event: ProjectEvent, + key: string +): string | null => { + const object = asObject(event.payload) + return object === null ? null : asString(object[key]) +} + +const readCreatedProjectId = ( + event: ProjectEvent, + fallbackProjectId: string +): string | null => + event.type === "project.created" + ? (readProjectEventPayloadField(event, "projectId") ?? fallbackProjectId) + : null + +const readFailedMessage = (event: ProjectEvent): string | null => + event.type === "project.deployment.status" && readProjectEventPayloadField(event, "phase") === "failed" + ? (readProjectEventPayloadField(event, "message") ?? "Project creation failed.") + : null + +const toProjectCreationError = ( + projectId: string, + message: string +): ApiRequestError => ({ + _tag: "ApiRequestError", + method: "POST", + path: "/projects", + message: `${message}\nProject event stream: ${projectId}`, + displayOnlyMessage: true +}) + export const readProjectEventCursor = (projectId: string) => request("GET", projectPath(projectId, "/events-poll")).pipe( Effect.map((payload) => decodeProjectEventPollResponse(payload)?.cursor ?? 0) @@ -90,11 +126,69 @@ const pollProjectEventsOnce = ( const payload = yield* _(request("GET", projectPath(projectId, `/events-poll?cursor=${cursor}`))) const response = decodeProjectEventPollResponse(payload) if (response === null) { - return + return { + cursor, + events: [] + } satisfies ProjectEventPollResponse } yield* _(Ref.set(cursorRef, response.cursor)) yield* _(writeProjectEventLines(response.events)) + return response + }) + +const findCreatedProjectId = ( + projectId: string, + events: ReadonlyArray +): string | null => { + for (const event of events) { + const createdProjectId = readCreatedProjectId(event, projectId) + if (createdProjectId !== null) { + return createdProjectId + } + } + return null +} + +const findFailureMessage = ( + events: ReadonlyArray +): string | null => { + for (const event of events) { + const message = readFailedMessage(event) + if (message !== null) { + return message + } + } + return null +} + +const waitForProjectCreationLoop = ( + projectId: string, + cursorRef: Ref.Ref +): Effect.Effect => + Effect.gen(function*(_) { + const response = yield* _(pollProjectEventsOnce(projectId, cursorRef)) + const failureMessage = findFailureMessage(response.events) + if (failureMessage !== null) { + return yield* _(Effect.fail(toProjectCreationError(projectId, failureMessage))) + } + + const createdProjectId = findCreatedProjectId(projectId, response.events) + if (createdProjectId !== null) { + return createdProjectId + } + + yield* _(Effect.sleep(projectEventPollInterval)) + return yield* _(waitForProjectCreationLoop(projectId, cursorRef)) + }) + +export const waitForProjectCreation = ( + projectId: string, + initialCursor: number +) => + Effect.gen(function*(_) { + const cursorRef = yield* _(Ref.make(initialCursor)) + return yield* _(waitForProjectCreationLoop(projectId, cursorRef)) }) export const startProjectEventPolling = (projectId: string, initialCursor: number) => diff --git a/packages/app/src/docker-git/api-client.ts b/packages/app/src/docker-git/api-client.ts index 1afcd949..3118b964 100644 --- a/packages/app/src/docker-git/api-client.ts +++ b/packages/app/src/docker-git/api-client.ts @@ -1,11 +1,16 @@ import { Effect } from "effect" import { buildCreateProjectRequest } from "./api-client-create.js" -import { readProjectEventCursor, startProjectEventPolling, stopProjectEventPolling } from "./api-client-events.js" +import { + readProjectEventCursor, + startProjectEventPolling, + stopProjectEventPolling, + waitForProjectCreation +} from "./api-client-events.js" import { readProjectOutput, resolveCreateRequestPaths } from "./api-client-helpers.js" import { request, requestVoid } from "./api-http.js" import { asArray, asObject, asString, type JsonValue } from "./api-json.js" -import { decodeProjectDetails, decodeProjectSummary } from "./api-project-codec.js" +import { decodeCreateProjectAccepted, decodeProjectDetails, decodeProjectSummary } from "./api-project-codec.js" import { decodeTerminalSession } from "./api-terminal-codec.js" import type { CreateCommand, @@ -13,6 +18,7 @@ import type { StateInitCommand, StateSyncCommand } from "./frontend-lib/core/domain.js" +import type { ApiRequestError } from "./host-errors.js" export { codexImport, @@ -42,6 +48,34 @@ const decodeProjectResponse = (payload: JsonValue) => { : decodeProjectDetails(object["project"] ?? payload) } +const invalidCreateAcceptedResponse = (): ApiRequestError => ({ + _tag: "ApiRequestError", + method: "POST", + path: "/projects", + message: "API accepted async create but returned an invalid response." +}) + +type ResolvedCreateRequestPaths = { + readonly authorizedKeysPath: string + readonly authorizedKeysContents?: string | undefined +} + +const createProjectAsync = ( + command: CreateCommand, + resolvedPaths: ResolvedCreateRequestPaths +) => + Effect.gen(function*(_) { + const createRequest = buildCreateProjectRequest(command, resolvedPaths, { async: true }) + const payload = yield* _(request("POST", "/projects", createRequest)) + const accepted = decodeCreateProjectAccepted(payload) + if (accepted === null) { + return yield* _(Effect.fail(invalidCreateAcceptedResponse())) + } + + const createdProjectId = yield* _(waitForProjectCreation(accepted.projectId, accepted.cursor)) + return yield* _(getProject(createdProjectId)) + }) + export const listProjects = () => request("GET", "/projects").pipe( Effect.map((payload) => { @@ -60,12 +94,13 @@ export const getProject = (projectId: string) => const createProjectWithResolvedPaths = ( command: CreateCommand, - resolvedPaths: { - readonly authorizedKeysPath: string - readonly authorizedKeysContents?: string | undefined - } + resolvedPaths: ResolvedCreateRequestPaths ) => Effect.gen(function*(_) { + if (command.runUp) { + return yield* _(createProjectAsync(command, resolvedPaths)) + } + const createRequest = buildCreateProjectRequest(command, resolvedPaths) const projectId = asString(createRequest.outDir) const initialCursor = projectId === null diff --git a/packages/app/src/docker-git/api-project-codec.ts b/packages/app/src/docker-git/api-project-codec.ts index 3d7ac836..07e2d437 100644 --- a/packages/app/src/docker-git/api-project-codec.ts +++ b/packages/app/src/docker-git/api-project-codec.ts @@ -29,6 +29,12 @@ export type ApiProjectDetails = ApiProjectSummary & { readonly codexHome: string } +export type ApiCreateProjectAccepted = { + readonly accepted: true + readonly projectId: string + readonly cursor: number +} + type ProjectDetailFields = Omit type RawProjectDetailFields = { readonly containerName: string | null @@ -54,6 +60,8 @@ const stringOrEmpty = (value: string | null): string => value ?? "" const numberOrZero = (value: number | null): number => value ?? 0 const readNullableNumber = (value: JsonValue | undefined): number | null => typeof value === "number" ? value : null +const readRequiredNumber = (value: JsonValue | undefined): number | null => + typeof value === "number" && Number.isFinite(value) ? value : null const readSummaryBaseFields = ( object: ReturnType @@ -165,5 +173,22 @@ export const decodeProjectDetails = (value: JsonValue): ApiProjectDetails | null return summary === null || details === null ? null : { ...summary, ...details } } +export const decodeCreateProjectAccepted = (value: JsonValue): ApiCreateProjectAccepted | null => { + const object = asObject(value) + if (object === null || object["accepted"] !== true) { + return null + } + + const projectId = asString(object["projectId"]) + const cursor = readRequiredNumber(object["cursor"]) + return projectId === null || cursor === null + ? null + : { + accepted: true, + projectId, + cursor + } +} + export const renderProjectSummaryLine = (project: ApiProjectSummary): string => `${project.displayName} [${project.statusLabel}] ${project.repoRef} ${project.repoUrl}` diff --git a/packages/app/src/lib/shell/command-runner.ts b/packages/app/src/lib/shell/command-runner.ts index 596d70f0..e724478b 100644 --- a/packages/app/src/lib/shell/command-runner.ts +++ b/packages/app/src/lib/shell/command-runner.ts @@ -86,6 +86,59 @@ const collectStreamText = ( ): Effect.Effect => pipe(stream, Stream.runCollect, Effect.map((chunks) => decodeUint8Array(collectUint8Array(chunks)))) +type StreamCommandState = { + readonly output: string + readonly remainder: string +} + +const maxStreamingFailureOutputLength = 20_000 + +const appendLimitedOutput = (output: string, chunk: string): string => { + const next = output + chunk + return next.length <= maxStreamingFailureOutputLength + ? next + : next.slice(next.length - maxStreamingFailureOutputLength) +} + +const emitStreamLines = ( + lines: ReadonlyArray, + onLine: (line: string) => Effect.Effect +): Effect.Effect => + Effect.forEach( + lines, + (line) => { + const trimmed = line.trimEnd() + return trimmed.length === 0 ? Effect.void : onLine(trimmed) + }, + { discard: true } + ) + +const collectStreamTextWithLines = ( + stream: Stream.Stream, + onLine: (line: string) => Effect.Effect +): Effect.Effect => + pipe( + stream, + Stream.decodeText(), + Stream.runFoldEffect({ output: "", remainder: "" } satisfies StreamCommandState, (state, chunk) => { + const normalized = `${state.remainder}${chunk}`.replaceAll("\r", "\n") + const parts = normalized.split("\n") + const remainder = parts.at(-1) ?? "" + const lines = parts.slice(0, -1) + + return emitStreamLines(lines, onLine).pipe( + Effect.as( + { + output: appendLimitedOutput(state.output, chunk), + remainder + } satisfies StreamCommandState + ) + ) + }), + Effect.tap((state) => emitStreamLines([state.remainder], onLine)), + Effect.map((state) => state.output) + ) + const combineCommandOutput = (stdout: string, stderr: string): string => [stdout.trim(), stderr.trim()].filter((chunk) => chunk.length > 0).join("\n") @@ -143,4 +196,31 @@ export const runCommandWithCapturedOutput = ( ) }) ) + +export const runCommandWithStreamingOutput = ( + spec: RunCommandSpec, + okExitCodes: ReadonlyArray, + onFailure: (exitCode: number, output: string) => E, + onLine: (line: string) => Effect.Effect = (line) => Effect.log(line) +): Effect.Effect => + Effect.scoped( + Effect.gen(function*(_) { + const executor = yield* _(CommandExecutor.CommandExecutor) + const process = yield* _(executor.start(buildCommand(spec, "pipe", "pipe", "pipe"))) + const [stdout, stderr] = yield* _( + Effect.all( + [ + collectStreamTextWithLines(process.stdout, onLine), + collectStreamTextWithLines(process.stderr, onLine) + ], + { concurrency: "unbounded" } + ) + ) + const exitCode = yield* _(process.exitCode) + yield* _( + ensureExitCode(Number(exitCode), okExitCodes, (numericExitCode) => + onFailure(numericExitCode, combineCommandOutput(stdout, stderr))) + ) + }) + ) /* jscpd:ignore-end */ diff --git a/packages/app/src/lib/shell/docker-compose.ts b/packages/app/src/lib/shell/docker-compose.ts index d4ad1927..c5d5f8cb 100644 --- a/packages/app/src/lib/shell/docker-compose.ts +++ b/packages/app/src/lib/shell/docker-compose.ts @@ -3,7 +3,7 @@ import type * as CommandExecutor from "@effect/platform/CommandExecutor" import type { PlatformError } from "@effect/platform/Error" import { Duration, Effect, pipe, Schedule } from "effect" -import { runCommandCapture, runCommandWithCapturedOutput } from "./command-runner.js" +import { runCommandCapture, runCommandWithStreamingOutput } from "./command-runner.js" import { composeSpec, resolveDockerComposeEnv } from "./docker-compose-env.js" import { DockerCommandError } from "./errors.js" @@ -24,7 +24,7 @@ const runCompose = ( Effect.gen(function*(_) { const env = yield* _(resolveDockerComposeEnv(cwd)) yield* _( - runCommandWithCapturedOutput( + runCommandWithStreamingOutput( buildComposeCommand(cwd, args, env), okExitCodes, (exitCode, output) => new DockerCommandError({ exitCode, ...(output.length > 0 ? { details: output } : {}) }) diff --git a/packages/app/src/web/actions-output.ts b/packages/app/src/web/actions-output.ts new file mode 100644 index 00000000..fd07eb0e --- /dev/null +++ b/packages/app/src/web/actions-output.ts @@ -0,0 +1,22 @@ +import { appendOutputChunk, type BrowserActionContext } from "./actions-shared.js" + +export const appendOutputLine = ( + context: BrowserActionContext, + line: string +) => { + const trimmed = line.trim() + if (trimmed.length === 0) { + return + } + appendOutputChunk(context, `${trimmed}\n`) +} + +export const appendOutputLineHandler = + (context: BrowserActionContext) => + (line: string) => { + appendOutputLine(context, line) + } + +export const notifyProjectEventRateLimit = (context: BrowserActionContext) => { + context.setMessage("HTTP 429: tunnel or proxy rate limited the live stream. Retry or request a fresh tunnel URL.") +} diff --git a/packages/app/src/web/actions-project-create.ts b/packages/app/src/web/actions-project-create.ts new file mode 100644 index 00000000..0c634857 --- /dev/null +++ b/packages/app/src/web/actions-project-create.ts @@ -0,0 +1,102 @@ +import { createProjectDraftFromInputs } from "../docker-git/menu-create-shared.js" +import type { CreateInputs } from "../docker-git/menu-types.js" +import { appendOutputLine, appendOutputLineHandler, notifyProjectEventRateLimit } from "./actions-output.js" +import { type BrowserActionContext, withBusy } from "./actions-shared.js" +import { type ApiEvent, loadProjectDetails, type ProjectDetails, startCreateProject } from "./api.js" +import { openProjectEventStream } from "./project-events.js" +import { outputScreen, projectPickerScreen } from "./screen.js" + +const readEventPayloadString = ( + event: ApiEvent, + key: string +): string | null => { + const payload = event.payload + if (payload === null || typeof payload !== "object" || Array.isArray(payload)) { + return null + } + const value = Object.entries(payload).find(([name]) => name === key)?.[1] + return typeof value === "string" ? value : null +} + +const readCreatedProjectId = (event: ApiEvent): string | null => + event.type === "project.created" ? readEventPayloadString(event, "projectId") : null + +const readCreateFailureMessage = (event: ApiEvent): string | null => + event.type === "project.deployment.status" && readEventPayloadString(event, "phase") === "failed" + ? (readEventPayloadString(event, "message") ?? "Project creation failed.") + : null + +const applyCreatedProject = ( + context: BrowserActionContext, + project: ProjectDetails +) => { + context.reloadDashboard() + context.setProjectAuthSnapshot(null) + context.setSelectedMenuIndex(1) + context.setActiveScreen(projectPickerScreen()) + context.setSelectedProject(project) + context.setSelectedProjectId(project.id) + context.setMessage(`Created ${project.displayName}.`) +} + +const finishCreateFromEvent = ( + context: BrowserActionContext, + projectId: string +) => { + appendOutputLine(context, "[create] Project created") + withBusy({ + context, + effect: loadProjectDetails(projectId), + label: "Loading created project", + onFailure: (error) => { + appendOutputLine(context, `[error] ${error}`) + }, + onSuccess: (project) => { + applyCreatedProject(context, project) + } + }) +} + +export const submitCreateInputs = ( + inputs: CreateInputs, + context: BrowserActionContext +) => { + context.setOutput("") + context.setActiveScreen(outputScreen()) + appendOutputLine(context, "[create] Project creation requested") + withBusy({ + context, + effect: startCreateProject(createProjectDraftFromInputs(inputs)), + label: "Starting project", + onFailure: (error) => { + appendOutputLine(context, `[error] ${error}`) + }, + onSuccess: (accepted) => { + appendOutputLine(context, `[create] Project accepted: ${accepted.projectId}`) + context.setMessage("Project creation is running. Live logs are open.") + let stream: ReturnType | null = null + stream = openProjectEventStream(accepted.projectId, { + initialCursor: accepted.cursor, + onEvent: (event) => { + const failureMessage = readCreateFailureMessage(event) + if (failureMessage !== null) { + stream?.close() + appendOutputLine(context, `[error] ${failureMessage}`) + context.setMessage(failureMessage) + return + } + + const projectId = readCreatedProjectId(event) + if (projectId !== null) { + stream?.close() + finishCreateFromEvent(context, projectId) + } + }, + onLine: appendOutputLineHandler(context), + onRateLimit: () => { + notifyProjectEventRateLimit(context) + } + }) + } + }) +} diff --git a/packages/app/src/web/actions-projects.ts b/packages/app/src/web/actions-projects.ts index 7219fdfb..ae629977 100644 --- a/packages/app/src/web/actions-projects.ts +++ b/packages/app/src/web/actions-projects.ts @@ -1,7 +1,6 @@ -import { createProjectDraftFromInputs } from "../docker-git/menu-create-shared.js" -import type { CreateInputs } from "../docker-git/menu-types.js" import { openSelectedProjectBrowser } from "./actions-browser.js" import { openSelectedProjectDatabaseEditor } from "./actions-databases.js" +import { appendOutputLine, appendOutputLineHandler, notifyProjectEventRateLimit } from "./actions-output.js" import { openSelectedProjectPort } from "./actions-port-forwards.js" import { type BrowserActionContext, @@ -12,7 +11,6 @@ import { withSelectedProjectBusy } from "./actions-shared.js" import { - createProject, createProjectTerminalSession, deleteProject, downAllProjects, @@ -23,22 +21,9 @@ import { } from "./api.js" import type { BrowserMenuTag } from "./menu.js" import { openProjectEventStream } from "./project-events.js" -import { outputScreen, projectPickerScreen } from "./screen.js" +import { outputScreen } from "./screen.js" -const appendOutputLine = ( - context: BrowserActionContext, - line: string -) => { - context.setOutput((current) => { - const trimmed = line.trim() - if (trimmed.length === 0) { - return current - } - const next = current.trim().length === 0 ? trimmed : `${current}\n${trimmed}` - const lines = next.split("\n") - return lines.length <= 120 ? next : lines.slice(-120).join("\n") - }) -} +export { submitCreateInputs } from "./actions-project-create.js" export const loadSelectedProjectInfo = ( context: BrowserActionContext, @@ -62,27 +47,6 @@ export const loadSelectedProjectInfo = ( }) } -export const submitCreateInputs = ( - inputs: CreateInputs, - context: BrowserActionContext -) => { - withBusy({ - context, - effect: createProject(createProjectDraftFromInputs(inputs)), - label: "Creating project", - onSuccess: (project) => { - context.reloadDashboard() - context.setOutput("") - context.setProjectAuthSnapshot(null) - context.setSelectedMenuIndex(1) - context.setActiveScreen(projectPickerScreen()) - context.setSelectedProject(project) - context.setSelectedProjectId(project.id) - context.setMessage(`Created ${project.displayName}.`) - } - }) -} - export const connectSelectedProject = (context: BrowserActionContext) => { const projectId = requireSelectedProjectId(context) if (projectId === null) { @@ -99,11 +63,9 @@ export const connectProjectById = ( context.setOutput("") appendOutputLine(context, "[ssh.prepare] Preparing SSH session") const stream = openProjectEventStream(projectId, { - onLine: (line) => { - appendOutputLine(context, line) - }, + onLine: appendOutputLineHandler(context), onRateLimit: () => { - context.setMessage("HTTP 429: tunnel or proxy rate limited the live stream. Retry or request a fresh tunnel URL.") + notifyProjectEventRateLimit(context) } }) withBusy({ diff --git a/packages/app/src/web/actions-shared.ts b/packages/app/src/web/actions-shared.ts index 2c81b3e9..b9a682c4 100644 --- a/packages/app/src/web/actions-shared.ts +++ b/packages/app/src/web/actions-shared.ts @@ -45,7 +45,7 @@ type AuthSuccessState = { readonly snapshot: AuthSnapshot } -const outputLineLimit = 120 +const outputLineLimit = 4000 export type BrowserActionContext = { readonly databaseConnectionInput: string diff --git a/packages/app/src/web/api-create-project.ts b/packages/app/src/web/api-create-project.ts new file mode 100644 index 00000000..a2ae250c --- /dev/null +++ b/packages/app/src/web/api-create-project.ts @@ -0,0 +1,13 @@ +import type { Effect } from "effect" + +import { requestJson } from "./api-http.js" +import { CreateProjectAcceptedResponseSchema } from "./api-schema.js" +import type { CreateProjectAcceptedResponse, CreateProjectDraft } from "./api-schema.js" + +export const startCreateProject = (draft: CreateProjectDraft): Effect.Effect => + requestJson( + "POST", + "/projects", + CreateProjectAcceptedResponseSchema, + { ...draft, async: true, openSsh: false, useManagedAuthorizedKeys: true } + ) diff --git a/packages/app/src/web/api-project-core.ts b/packages/app/src/web/api-project-core.ts new file mode 100644 index 00000000..f6f32bb1 --- /dev/null +++ b/packages/app/src/web/api-project-core.ts @@ -0,0 +1,40 @@ +import { Effect } from "effect" + +import { requestJson } from "./api-http.js" +import type { CreateProjectDraft } from "./api-schema.js" +import { OutputResponseSchema, ProjectResponseSchema } from "./api-schema.js" + +export const loadProjectDetails = (projectId: string) => + requestJson("GET", `/projects/${encodeURIComponent(projectId)}`, ProjectResponseSchema).pipe( + Effect.map((response) => response.project) + ) + +export const loadProjectPs = (projectId: string) => + requestJson("GET", `/projects/${encodeURIComponent(projectId)}/ps`, OutputResponseSchema).pipe( + Effect.map((response) => response.output) + ) + +export const loadProjectLogs = (projectId: string) => + requestJson("GET", `/projects/${encodeURIComponent(projectId)}/logs`, OutputResponseSchema).pipe( + Effect.map((response) => response.output) + ) + +export const createProject = (draft: CreateProjectDraft) => + requestJson( + "POST", + "/projects", + ProjectResponseSchema, + { ...draft, openSsh: false, useManagedAuthorizedKeys: true } + ).pipe( + Effect.map((response) => response.project) + ) + +export const upProject = (projectId: string) => + requestJson( + "POST", + `/projects/${encodeURIComponent(projectId)}/up`, + ProjectResponseSchema, + { useManagedAuthorizedKeys: true } + ).pipe( + Effect.map((response) => response.project) + ) diff --git a/packages/app/src/web/api-schema.ts b/packages/app/src/web/api-schema.ts index 1fe35887..7cb84256 100644 --- a/packages/app/src/web/api-schema.ts +++ b/packages/app/src/web/api-schema.ts @@ -60,6 +60,12 @@ export const ProjectResponseSchema = Schema.Struct({ project: ProjectDetailsSchema }) +export const CreateProjectAcceptedResponseSchema = Schema.Struct({ + accepted: Schema.Literal(true), + projectId: Schema.String, + cursor: Schema.Number +}) + const ProjectPublishedStatusSchema = Schema.Union( Schema.Literal("running"), Schema.Literal("stopped"), @@ -283,6 +289,7 @@ export const ProjectEventsPollResponseSchema = Schema.Struct({ export type ProjectSummary = Schema.Schema.Type export type ProjectDetails = Schema.Schema.Type +export type CreateProjectAcceptedResponse = Schema.Schema.Type export type ProjectPortForward = Schema.Schema.Type export type ProjectBrowserSession = Schema.Schema.Type export type ProjectDatabaseForward = Schema.Schema.Type diff --git a/packages/app/src/web/api.ts b/packages/app/src/web/api.ts index a5b93ab0..e7a21cad 100644 --- a/packages/app/src/web/api.ts +++ b/packages/app/src/web/api.ts @@ -7,7 +7,6 @@ import { AuthTerminalSessionResponseSchema, GithubStatusResponseSchema, HealthResponseSchema, - OutputResponseSchema, ProjectAuthSnapshotResponseSchema, ProjectBrowserResponseSchema, ProjectDatabaseForwardResponseSchema, @@ -18,13 +17,11 @@ import { ProjectEventsPollResponseSchema, ProjectPortForwardResponseSchema, ProjectPortForwardsResponseSchema, - ProjectResponseSchema, ProjectsResponseSchema, TerminalSessionResponseSchema } from "./api-schema.js" import type { AuthMenuFlow, - CreateProjectDraft, DashboardData, ProjectAuthFlow, ProjectBrowserSession, @@ -33,10 +30,14 @@ import type { ProjectPortForward } from "./api-schema.js" +export { startCreateProject } from "./api-create-project.js" +export { createProject, loadProjectDetails, loadProjectLogs, loadProjectPs, upProject } from "./api-project-core.js" + export type { ApiEvent, AuthMenuFlow, AuthSnapshot, + CreateProjectAcceptedResponse, CreateProjectDraft, DashboardData, GithubAuthStatus, @@ -76,21 +77,6 @@ export const loadDashboard = (): Effect.Effect => })) ) -export const loadProjectDetails = (projectId: string) => - requestJson("GET", `/projects/${encodeURIComponent(projectId)}`, ProjectResponseSchema).pipe( - Effect.map((response) => response.project) - ) - -export const loadProjectPs = (projectId: string) => - requestJson("GET", `/projects/${encodeURIComponent(projectId)}/ps`, OutputResponseSchema).pipe( - Effect.map((response) => response.output) - ) - -export const loadProjectLogs = (projectId: string) => - requestJson("GET", `/projects/${encodeURIComponent(projectId)}/logs`, OutputResponseSchema).pipe( - Effect.map((response) => response.output) - ) - export const loadProjectPortForwards = (projectId: string) => requestJson("GET", `/projects/${encodeURIComponent(projectId)}/ports`, ProjectPortForwardsResponseSchema).pipe( Effect.map((response) => response.forwards) @@ -209,26 +195,6 @@ export const deleteProjectPortForward = ( targetPort: number ) => requestText("DELETE", `/projects/${encodeURIComponent(projectId)}/ports/${targetPort}`).pipe(Effect.asVoid) -export const createProject = (draft: CreateProjectDraft) => - requestJson( - "POST", - "/projects", - ProjectResponseSchema, - { ...draft, openSsh: false, useManagedAuthorizedKeys: true } - ).pipe( - Effect.map((response) => response.project) - ) - -export const upProject = (projectId: string) => - requestJson( - "POST", - `/projects/${encodeURIComponent(projectId)}/up`, - ProjectResponseSchema, - { useManagedAuthorizedKeys: true } - ).pipe( - Effect.map((response) => response.project) - ) - export const createProjectTerminalSession = (projectId: string) => requestJson( "POST", diff --git a/packages/app/src/web/project-events.ts b/packages/app/src/web/project-events.ts index 7162dd71..857ab24c 100644 --- a/packages/app/src/web/project-events.ts +++ b/packages/app/src/web/project-events.ts @@ -10,7 +10,9 @@ type EventStreamControls = { type EventStreamHandlers = { readonly onLine: (line: string) => void + readonly onEvent?: (event: ApiEvent) => void readonly onRateLimit: () => void + readonly initialCursor?: number | undefined } type PollState = { @@ -53,6 +55,7 @@ const handlePollFailure = ( const handlePollSuccess = ( state: PollState, onLine: (line: string) => void, + onEvent: ((event: ApiEvent) => void) | undefined, response: EventPollSuccess, runPoll: () => void ): void => { @@ -69,6 +72,7 @@ const handlePollSuccess = ( if (line !== null) { onLine(line) } + onEvent?.(event) } let delayMs = 150 if (isInitialPoll) { @@ -81,11 +85,11 @@ const handlePollSuccess = ( export const openProjectEventStream = ( projectId: string, - { onLine, onRateLimit }: EventStreamHandlers + { initialCursor, onEvent, onLine, onRateLimit }: EventStreamHandlers ): EventStreamControls => { const state: PollState = { closed: false, - cursor: undefined, + cursor: initialCursor, timeout: null } @@ -97,7 +101,7 @@ export const openProjectEventStream = ( handlePollFailure(state, onLine, onRateLimit, error, runPoll) }, onSuccess: (response) => { - handlePollSuccess(state, onLine, response, runPoll) + handlePollSuccess(state, onLine, onEvent, response, runPoll) } }) ) diff --git a/packages/app/tests/docker-git/api-project-codec.test.ts b/packages/app/tests/docker-git/api-project-codec.test.ts new file mode 100644 index 00000000..1c4af45f --- /dev/null +++ b/packages/app/tests/docker-git/api-project-codec.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "@effect/vitest" + +import { decodeCreateProjectAccepted } from "../../src/docker-git/api-project-codec.js" + +describe("api project codec", () => { + it("decodes async create accepted responses", () => { + const accepted = decodeCreateProjectAccepted({ + accepted: true, + projectId: ".docker-git/org/repo", + cursor: 0 + }) + + expect(accepted).toEqual({ + accepted: true, + projectId: ".docker-git/org/repo", + cursor: 0 + }) + }) + + it("rejects incomplete async create accepted responses", () => { + expect(decodeCreateProjectAccepted({ accepted: true, projectId: ".docker-git/org/repo" })).toBeNull() + }) +}) diff --git a/packages/lib/src/shell/command-runner.ts b/packages/lib/src/shell/command-runner.ts index 1709f97c..017b1855 100644 --- a/packages/lib/src/shell/command-runner.ts +++ b/packages/lib/src/shell/command-runner.ts @@ -86,6 +86,59 @@ const collectStreamText = ( ): Effect.Effect => pipe(stream, Stream.runCollect, Effect.map((chunks) => decodeUint8Array(collectUint8Array(chunks)))) +type StreamCommandState = { + readonly output: string + readonly remainder: string +} + +const maxStreamingFailureOutputLength = 20_000 + +const appendLimitedOutput = (output: string, chunk: string): string => { + const next = output + chunk + return next.length <= maxStreamingFailureOutputLength + ? next + : next.slice(next.length - maxStreamingFailureOutputLength) +} + +const emitStreamLines = ( + lines: ReadonlyArray, + onLine: (line: string) => Effect.Effect +): Effect.Effect => + Effect.forEach( + lines, + (line) => { + const trimmed = line.trimEnd() + return trimmed.length === 0 ? Effect.void : onLine(trimmed) + }, + { discard: true } + ) + +const collectStreamTextWithLines = ( + stream: Stream.Stream, + onLine: (line: string) => Effect.Effect +): Effect.Effect => + pipe( + stream, + Stream.decodeText(), + Stream.runFoldEffect({ output: "", remainder: "" } satisfies StreamCommandState, (state, chunk) => { + const normalized = `${state.remainder}${chunk}`.replaceAll("\r", "\n") + const parts = normalized.split("\n") + const remainder = parts.at(-1) ?? "" + const lines = parts.slice(0, -1) + + return emitStreamLines(lines, onLine).pipe( + Effect.as( + { + output: appendLimitedOutput(state.output, chunk), + remainder + } satisfies StreamCommandState + ) + ) + }), + Effect.tap((state) => emitStreamLines([state.remainder], onLine)), + Effect.map((state) => state.output) + ) + const combineCommandOutput = (stdout: string, stderr: string): string => [stdout.trim(), stderr.trim()].filter((chunk) => chunk.length > 0).join("\n") @@ -144,3 +197,30 @@ export const runCommandWithCapturedOutput = ( ) }) ) + +export const runCommandWithStreamingOutput = ( + spec: RunCommandSpec, + okExitCodes: ReadonlyArray, + onFailure: (exitCode: number, output: string) => E, + onLine: (line: string) => Effect.Effect = (line) => Effect.log(line) +): Effect.Effect => + Effect.scoped( + Effect.gen(function*(_) { + const executor = yield* _(CommandExecutor.CommandExecutor) + const process = yield* _(executor.start(buildCommand(spec, "pipe", "pipe", "pipe"))) + const [stdout, stderr] = yield* _( + Effect.all( + [ + collectStreamTextWithLines(process.stdout, onLine), + collectStreamTextWithLines(process.stderr, onLine) + ], + { concurrency: "unbounded" } + ) + ) + const exitCode = yield* _(process.exitCode) + yield* _( + ensureExitCode(Number(exitCode), okExitCodes, (numericExitCode) => + onFailure(numericExitCode, combineCommandOutput(stdout, stderr))) + ) + }) + ) diff --git a/packages/lib/src/shell/docker.ts b/packages/lib/src/shell/docker.ts index 543535d9..a8fe2255 100644 --- a/packages/lib/src/shell/docker.ts +++ b/packages/lib/src/shell/docker.ts @@ -7,8 +7,8 @@ import { Duration, Effect, pipe, Schedule } from "effect" import { runCommandCapture, runCommandExitCode, - runCommandWithCapturedOutput, - runCommandWithExitCodes + runCommandWithExitCodes, + runCommandWithStreamingOutput } from "./command-runner.js" import { composeSpec, resolveDockerComposeEnv } from "./docker-compose-env.js" import { parseInspectNetworkEntry } from "./docker-inspect-parse.js" @@ -25,7 +25,7 @@ const runCompose = ( Effect.gen(function*(_) { const env = yield* _(resolveDockerComposeEnv(cwd)) yield* _( - runCommandWithCapturedOutput( + runCommandWithStreamingOutput( { ...composeSpec(cwd, args), ...(Object.keys(env).length > 0 ? { env } : {}) From 3c70e846197a4afa1086171ba4fa1d45d801552c Mon Sep 17 00:00:00 2001 From: "{ \"message\": \"Bad credentials\", \"documentation_url\": \"https://docs.github.com/rest\", \"status\": \"401\"}" <{ "message": "Bad credentials", "documentation_url": "https://docs.github.com/rest", "status": "401"}+{ "message": "Bad credentials", "documentation_url": "https://docs.github.com/rest", "status": "401"}@users.noreply.github.com> Date: Thu, 16 Apr 2026 20:53:18 +0000 Subject: [PATCH 2/2] fix: return created project from realtime events --- packages/api/src/services/projects.ts | 45 ++++++++++++++----- packages/api/tests/projects.test.ts | 6 +++ .../app/src/docker-git/api-client-events.ts | 40 +++++++++++------ packages/app/src/docker-git/api-client.ts | 7 ++- .../app/src/web/actions-project-create.ts | 28 +++++++++++- 5 files changed, 97 insertions(+), 29 deletions(-) diff --git a/packages/api/src/services/projects.ts b/packages/api/src/services/projects.ts index 6dfa718d..684b6744 100644 --- a/packages/api/src/services/projects.ts +++ b/packages/api/src/services/projects.ts @@ -175,15 +175,35 @@ const toProjectDetails = ( codexHome: project.codexHome }) +const projectIdAliases = ( + path: Path.Path, + projectId: string +): ReadonlySet => { + const projectsRoot = path.resolve(defaultProjectsRoot(process.cwd())) + const normalized = projectId + .replaceAll("\\", "/") + .replace(/^\.\//u, "") + const rooted = normalized === ".docker-git" + ? projectsRoot + : normalized.startsWith(".docker-git/") + ? path.join(projectsRoot, normalized.slice(".docker-git/".length)) + : projectId + const absolute = path.isAbsolute(rooted) ? path.resolve(rooted) : path.resolve(process.cwd(), rooted) + return new Set([projectId, rooted, absolute]) +} + const findProjectById = (projectId: string) => - listProjectItems.pipe( - Effect.flatMap((projects) => { - const project = projects.find((item) => item.projectDir === projectId) + Effect.gen(function*(_) { + const path = yield* _(Path.Path) + const aliases = projectIdAliases(path, projectId) + const projects = yield* _(listProjectItems) + const project = projects.find((item) => item.projectDir === projectId) + ?? projects.find((item) => aliases.has(item.projectDir) || aliases.has(path.resolve(item.projectDir))) + if (project) { return project - ? Effect.succeed(project) - : Effect.fail(new ApiNotFoundError({ message: `Project not found: ${projectId}` })) - }) - ) + } + return yield* _(Effect.fail(new ApiNotFoundError({ message: `Project not found: ${projectId}` }))) + }) export const getProjectItemById = (projectId: string) => findProjectById(projectId) @@ -401,12 +421,14 @@ const emitCreateStatus = ( const emitProjectCreatedEvents = ( projectId: string, - project: ProjectItem + project: ProjectItem, + details: ProjectDetails ) => Effect.sync(() => { const payload = { projectId: project.projectDir, - containerName: project.containerName + containerName: project.containerName, + project: details } emitProjectEvent(project.projectDir, "project.created", payload) if (project.projectDir !== projectId) { @@ -449,10 +471,11 @@ const runPreparedCreateProject = ( ) const runtimeByProject = yield* _(loadProjectRuntimeByProject([project])) const summary = yield* _(withProjectRuntime(project, runtimeForProject(runtimeByProject, project))) + const details = toProjectDetails(project, summary) - yield* _(emitProjectCreatedEvents(projectId, project)) + yield* _(emitProjectCreatedEvents(projectId, project, details)) - return toProjectDetails(project, summary) + return details }).pipe(Effect.mapError(toProjectApiError)) const startCreateProjectJob = ( diff --git a/packages/api/tests/projects.test.ts b/packages/api/tests/projects.test.ts index b1c9be6b..08531426 100644 --- a/packages/api/tests/projects.test.ts +++ b/packages/api/tests/projects.test.ts @@ -245,6 +245,12 @@ describe("projects service", () => { phase: "create", message: "Project creation started" }) + expect(events.find((event) => event.type === "project.created")?.payload).toMatchObject({ + projectId, + project: { + projectDir: projectId + } + }) }) ) ) diff --git a/packages/app/src/docker-git/api-client-events.ts b/packages/app/src/docker-git/api-client-events.ts index 3f7b6c8a..ff3e753f 100644 --- a/packages/app/src/docker-git/api-client-events.ts +++ b/packages/app/src/docker-git/api-client-events.ts @@ -3,6 +3,7 @@ import * as Fiber from "effect/Fiber" import { request } from "./api-http.js" import { asArray, asObject, asString, type JsonValue } from "./api-json.js" +import { type ApiProjectDetails, decodeProjectDetails } from "./api-project-codec.js" import type { ControllerRuntime } from "./controller.js" import type { ApiAuthRequiredError, ApiRequestError } from "./host-errors.js" import { formatProjectEventLine } from "./project-event-lines.js" @@ -23,6 +24,11 @@ type ProjectEventPollResponse = { type ProjectCreationWaitError = ApiAuthRequiredError | ApiRequestError +export type ProjectCreationResult = { + readonly projectId: string + readonly project: ApiProjectDetails | null +} + export type ProjectEventPolling = { readonly cursorRef: Ref.Ref readonly fiber: Fiber.RuntimeFiber @@ -88,13 +94,19 @@ const readProjectEventPayloadField = ( return object === null ? null : asString(object[key]) } -const readCreatedProjectId = ( +const readCreatedProject = ( event: ProjectEvent, fallbackProjectId: string -): string | null => - event.type === "project.created" - ? (readProjectEventPayloadField(event, "projectId") ?? fallbackProjectId) - : null +): ProjectCreationResult | null => { + if (event.type !== "project.created") { + return null + } + + const payload = asObject(event.payload) + const projectId = readProjectEventPayloadField(event, "projectId") ?? fallbackProjectId + const project = payload === null ? null : decodeProjectDetails(payload["project"] ?? null) + return { projectId, project } +} const readFailedMessage = (event: ProjectEvent): string | null => event.type === "project.deployment.status" && readProjectEventPayloadField(event, "phase") === "failed" @@ -137,14 +149,14 @@ const pollProjectEventsOnce = ( return response }) -const findCreatedProjectId = ( +const findCreatedProject = ( projectId: string, events: ReadonlyArray -): string | null => { +): ProjectCreationResult | null => { for (const event of events) { - const createdProjectId = readCreatedProjectId(event, projectId) - if (createdProjectId !== null) { - return createdProjectId + const created = readCreatedProject(event, projectId) + if (created !== null) { + return created } } return null @@ -165,7 +177,7 @@ const findFailureMessage = ( const waitForProjectCreationLoop = ( projectId: string, cursorRef: Ref.Ref -): Effect.Effect => +): Effect.Effect => Effect.gen(function*(_) { const response = yield* _(pollProjectEventsOnce(projectId, cursorRef)) const failureMessage = findFailureMessage(response.events) @@ -173,9 +185,9 @@ const waitForProjectCreationLoop = ( return yield* _(Effect.fail(toProjectCreationError(projectId, failureMessage))) } - const createdProjectId = findCreatedProjectId(projectId, response.events) - if (createdProjectId !== null) { - return createdProjectId + const created = findCreatedProject(projectId, response.events) + if (created !== null) { + return created } yield* _(Effect.sleep(projectEventPollInterval)) diff --git a/packages/app/src/docker-git/api-client.ts b/packages/app/src/docker-git/api-client.ts index 3118b964..8f6b01ee 100644 --- a/packages/app/src/docker-git/api-client.ts +++ b/packages/app/src/docker-git/api-client.ts @@ -72,8 +72,11 @@ const createProjectAsync = ( return yield* _(Effect.fail(invalidCreateAcceptedResponse())) } - const createdProjectId = yield* _(waitForProjectCreation(accepted.projectId, accepted.cursor)) - return yield* _(getProject(createdProjectId)) + const created = yield* _(waitForProjectCreation(accepted.projectId, accepted.cursor)) + if (created.project !== null) { + return created.project + } + return yield* _(getProject(created.projectId)) }) export const listProjects = () => diff --git a/packages/app/src/web/actions-project-create.ts b/packages/app/src/web/actions-project-create.ts index 0c634857..9c2f5eb6 100644 --- a/packages/app/src/web/actions-project-create.ts +++ b/packages/app/src/web/actions-project-create.ts @@ -1,7 +1,11 @@ +import * as ParseResult from "@effect/schema/ParseResult" +import { Either } from "effect" + import { createProjectDraftFromInputs } from "../docker-git/menu-create-shared.js" import type { CreateInputs } from "../docker-git/menu-types.js" import { appendOutputLine, appendOutputLineHandler, notifyProjectEventRateLimit } from "./actions-output.js" import { type BrowserActionContext, withBusy } from "./actions-shared.js" +import { ProjectDetailsSchema } from "./api-schema.js" import { type ApiEvent, loadProjectDetails, type ProjectDetails, startCreateProject } from "./api.js" import { openProjectEventStream } from "./project-events.js" import { outputScreen, projectPickerScreen } from "./screen.js" @@ -21,6 +25,21 @@ const readEventPayloadString = ( const readCreatedProjectId = (event: ApiEvent): string | null => event.type === "project.created" ? readEventPayloadString(event, "projectId") : null +const readCreatedProject = (event: ApiEvent): ProjectDetails | null => { + if (event.type !== "project.created") { + return null + } + const payload = event.payload + if (payload === null || typeof payload !== "object" || Array.isArray(payload)) { + return null + } + const project = Object.entries(payload).find(([name]) => name === "project")?.[1] + return Either.match(ParseResult.decodeUnknownEither(ProjectDetailsSchema)(project), { + onLeft: () => null, + onRight: (value) => value + }) +} + const readCreateFailureMessage = (event: ApiEvent): string | null => event.type === "project.deployment.status" && readEventPayloadString(event, "phase") === "failed" ? (readEventPayloadString(event, "message") ?? "Project creation failed.") @@ -41,9 +60,14 @@ const applyCreatedProject = ( const finishCreateFromEvent = ( context: BrowserActionContext, - projectId: string + projectId: string, + project: ProjectDetails | null ) => { appendOutputLine(context, "[create] Project created") + if (project !== null) { + applyCreatedProject(context, project) + return + } withBusy({ context, effect: loadProjectDetails(projectId), @@ -89,7 +113,7 @@ export const submitCreateInputs = ( const projectId = readCreatedProjectId(event) if (projectId !== null) { stream?.close() - finishCreateFromEvent(context, projectId) + finishCreateFromEvent(context, projectId, readCreatedProject(event)) } }, onLine: appendOutputLineHandler(context),