From e6bc57c901fc098724d165a1ae3d33ab64485009 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Mon, 9 Nov 2026 15:55:14 +0000 Subject: [PATCH] Run operator shell commands in a chat's directory on the server chat_shell_run starts $SHELL -lc in the chat session's directory in its own process group, streams merged output to every watcher of the session, keeps a 1 MiB tail and persists each run; chat_shell_stop interrupts the group and kills it after a grace period. chat_shell_share queues an ordinary prompt with the command and a bounded output tail. --- internal/pipeline/chat_shell_runs.go | 123 ++++++++++ internal/pipeline/chat_shell_runs_test.go | 66 ++++++ internal/pipeline/store.go | 4 + internal/protocol/chat_shell_run.go | 107 +++++++++ internal/protocol/message.go | 6 + internal/server/chat.go | 3 + internal/server/chat_hub.go | 3 + internal/server/chat_shell_run_handlers.go | 141 ++++++++++++ internal/server/chat_shell_run_output.go | 96 ++++++++ internal/server/chat_shell_run_test.go | 190 +++++++++++++++ internal/server/chat_shell_runner.go | 256 +++++++++++++++++++++ internal/server/chat_shell_runner_test.go | 143 ++++++++++++ internal/server/chat_shell_share.go | 78 +++++++ internal/server/chat_shell_supervise.go | 132 +++++++++++ internal/server/chat_stream.go | 13 +- internal/server/websocket.go | 7 + 16 files changed, 1367 insertions(+), 1 deletion(-) create mode 100644 internal/pipeline/chat_shell_runs.go create mode 100644 internal/pipeline/chat_shell_runs_test.go create mode 100644 internal/protocol/chat_shell_run.go create mode 100644 internal/server/chat_shell_run_handlers.go create mode 100644 internal/server/chat_shell_run_output.go create mode 100644 internal/server/chat_shell_run_test.go create mode 100644 internal/server/chat_shell_runner.go create mode 100644 internal/server/chat_shell_runner_test.go create mode 100644 internal/server/chat_shell_share.go create mode 100644 internal/server/chat_shell_supervise.go diff --git a/internal/pipeline/chat_shell_runs.go b/internal/pipeline/chat_shell_runs.go new file mode 100644 index 00000000..872b4b71 --- /dev/null +++ b/internal/pipeline/chat_shell_runs.go @@ -0,0 +1,123 @@ +package pipeline + +import ( + "context" + "database/sql" + "errors" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// initChatShellRuns creates the additive table of operator shell commands. A +// run belongs to its chat session and is Pipelines' own record: it is never +// part of the harness transcript. Output holds only the retained tail. +func (s *Store) initChatShellRuns() error { + _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS chat_shell_runs ( +id TEXT PRIMARY KEY, +session_id TEXT NOT NULL, +command TEXT NOT NULL, +directory TEXT NOT NULL, +status TEXT NOT NULL, +exit_code INTEGER, +signal TEXT NOT NULL DEFAULT '', +error TEXT NOT NULL DEFAULT '', +started_at INTEGER NOT NULL, +ended_at INTEGER NOT NULL DEFAULT 0, +duration_ms INTEGER NOT NULL DEFAULT 0, +output TEXT NOT NULL DEFAULT '', +output_start INTEGER NOT NULL DEFAULT 0, +output_bytes INTEGER NOT NULL DEFAULT 0); +CREATE INDEX IF NOT EXISTS chat_shell_runs_session ON chat_shell_runs(session_id, started_at);`) + return err +} + +// SaveChatShellRun inserts or replaces one run. Inserting a new run prunes the +// session's oldest finished runs beyond keep (keep <= 0 keeps everything). +func (s *Store) SaveChatShellRun(ctx context.Context, run protocol.ChatShellRunRecord, keep int) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var exit any + if run.ExitCode != nil { + exit = *run.ExitCode + } + if _, err := tx.ExecContext(ctx, `INSERT INTO chat_shell_runs +(id, session_id, command, directory, status, exit_code, signal, error, started_at, ended_at, duration_ms, output, output_start, output_bytes) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(id) DO UPDATE SET status = excluded.status, exit_code = excluded.exit_code, signal = excluded.signal, +error = excluded.error, ended_at = excluded.ended_at, duration_ms = excluded.duration_ms, output = excluded.output, +output_start = excluded.output_start, output_bytes = excluded.output_bytes`, + run.ID, run.SessionID, run.Command, run.Directory, run.Status, exit, run.Signal, run.Error, + run.StartedAt, run.EndedAt, run.DurationMS, run.Output, run.OutputStart, run.OutputBytes); err != nil { + return err + } + if keep > 0 { + if _, err := tx.ExecContext(ctx, `DELETE FROM chat_shell_runs WHERE session_id = ? AND status != ? AND id NOT IN +(SELECT id FROM chat_shell_runs WHERE session_id = ? ORDER BY started_at DESC, id DESC LIMIT ?)`, + run.SessionID, protocol.ChatShellRunRunning, run.SessionID, keep); err != nil { + return err + } + } + return tx.Commit() +} + +// ChatShellRuns lists one session's runs, oldest first. +func (s *Store) ChatShellRuns(ctx context.Context, sessionID string) ([]protocol.ChatShellRunRecord, error) { + rows, err := s.db.QueryContext(ctx, chatShellRunSelect+` WHERE session_id = ? ORDER BY started_at, id`, sessionID) + if err != nil { + return nil, err + } + defer rows.Close() + var runs []protocol.ChatShellRunRecord + for rows.Next() { + run, err := scanChatShellRun(rows) + if err != nil { + return nil, err + } + runs = append(runs, run) + } + return runs, rows.Err() +} + +// ChatShellRun returns one run of sessionID; found is false when that session +// has no run with this id. +func (s *Store) ChatShellRun(ctx context.Context, sessionID, id string) (protocol.ChatShellRunRecord, bool, error) { + run, err := scanChatShellRun(s.db.QueryRowContext(ctx, chatShellRunSelect+` WHERE session_id = ? AND id = ?`, sessionID, id)) + if errors.Is(err, sql.ErrNoRows) { + return protocol.ChatShellRunRecord{}, false, nil + } + return run, err == nil, err +} + +// InterruptChatShellRuns marks every run still recorded as running as +// interrupted at endedAt. A starting server calls it: no process of a previous +// server is still owned by anyone. +func (s *Store) InterruptChatShellRuns(ctx context.Context, endedAt int64, reason string) (int64, error) { + result, err := s.db.ExecContext(ctx, `UPDATE chat_shell_runs SET status = ?, error = ?, ended_at = ?, +duration_ms = CASE WHEN ? > started_at THEN ? - started_at ELSE 0 END WHERE status = ?`, + protocol.ChatShellRunInterrupted, reason, endedAt, endedAt, endedAt, protocol.ChatShellRunRunning) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const chatShellRunSelect = `SELECT id, session_id, command, directory, status, exit_code, signal, error, started_at, ended_at, duration_ms, output, output_start, output_bytes FROM chat_shell_runs` + +func scanChatShellRun(row chatQueueScanner) (protocol.ChatShellRunRecord, error) { + var run protocol.ChatShellRunRecord + var exit sql.NullInt64 + err := row.Scan(&run.ID, &run.SessionID, &run.Command, &run.Directory, &run.Status, &exit, &run.Signal, &run.Error, + &run.StartedAt, &run.EndedAt, &run.DurationMS, &run.Output, &run.OutputStart, &run.OutputBytes) + if err != nil { + return protocol.ChatShellRunRecord{}, err + } + if exit.Valid { + code := int(exit.Int64) + run.ExitCode = &code + } + run.Truncated = run.OutputStart > 0 + return run, nil +} diff --git a/internal/pipeline/chat_shell_runs_test.go b/internal/pipeline/chat_shell_runs_test.go new file mode 100644 index 00000000..5aae3761 --- /dev/null +++ b/internal/pipeline/chat_shell_runs_test.go @@ -0,0 +1,66 @@ +package pipeline + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestChatShellRunsPersistPruneAndInterrupt(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "shell.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + running := protocol.ChatShellRunRecord{ID: "run_live", SessionID: "ses_1", Command: "sleep 9", Directory: "/w", Status: protocol.ChatShellRunRunning, StartedAt: 1000} + if err := store.SaveChatShellRun(ctx, running, 2); err != nil { + t.Fatal(err) + } + code := 3 + for i := 0; i < 3; i++ { + run := protocol.ChatShellRunRecord{ID: fmt.Sprintf("run_%d", i), SessionID: "ses_1", Command: "false", Directory: "/w", Status: protocol.ChatShellRunExited, ExitCode: &code, StartedAt: int64(2000 + i), EndedAt: int64(2500 + i), Output: "tail", OutputStart: 10, OutputBytes: 14} + if err := store.SaveChatShellRun(ctx, run, 2); err != nil { + t.Fatal(err) + } + } + if err := store.SaveChatShellRun(ctx, protocol.ChatShellRunRecord{ID: "other", SessionID: "ses_2", Command: "ls", Directory: "/w", Status: protocol.ChatShellRunExited, StartedAt: 1}, 2); err != nil { + t.Fatal(err) + } + runs, err := store.ChatShellRuns(ctx, "ses_1") + if err != nil { + t.Fatal(err) + } + // The running run is never pruned; keep counts the newest runs, so only + // the finished ones outside the newest two are dropped. + ids := []string{} + for _, run := range runs { + ids = append(ids, run.ID) + } + if fmt.Sprint(ids) != "[run_live run_1 run_2]" { + t.Fatalf("runs = %v", ids) + } + if got := runs[2]; got.ExitCode == nil || *got.ExitCode != 3 || !got.Truncated || got.Output != "tail" || got.OutputBytes != 14 { + t.Fatalf("finished run = %+v", got) + } + store.Close() + store, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer store.Close() + n, err := store.InterruptChatShellRuns(ctx, 4000, "server restarted") + if err != nil || n != 1 { + t.Fatalf("interrupt = %d, %v", n, err) + } + live, found, err := store.ChatShellRun(ctx, "ses_1", "run_live") + if err != nil || !found || live.Status != protocol.ChatShellRunInterrupted || live.EndedAt != 4000 || live.DurationMS != 3000 || live.ExitCode != nil { + t.Fatalf("interrupted run = %+v found=%v err=%v", live, found, err) + } + if _, found, _ := store.ChatShellRun(ctx, "ses_2", "run_live"); found { + t.Fatal("a run was readable through another session") + } +} diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index 9a78b245..2f215b03 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -101,6 +101,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU db.Close() return nil, err } + if err := s.initChatShellRuns(); err != nil { + db.Close() + return nil, err + } if err := s.initAppNotifications(); err != nil { db.Close() return nil, err diff --git a/internal/protocol/chat_shell_run.go b/internal/protocol/chat_shell_run.go new file mode 100644 index 00000000..8d6d08da --- /dev/null +++ b/internal/protocol/chat_shell_run.go @@ -0,0 +1,107 @@ +package protocol + +// Operator shell commands: commands the operator runs themselves inside one +// chat's working directory (the chat's `!cmd`). They are owned by Pipelines, +// not by the chat harness: the model never sees a command or its output unless +// the operator explicitly shares it (chat_shell_share), which queues an +// ordinary prompt. +const ( + // ChatShellRun starts one command. The body is ChatShellRunRequest + // (session_id, command); the reply carries the new run in ChatShellRun. + ChatShellRun = "chat_shell_run" + // ChatShellStop interrupts one running command (SIGINT to its process + // group, then SIGKILL after a grace period). The body is + // ChatShellRunRequest (session_id, run_id); the reply echoes the run. + ChatShellStop = "chat_shell_stop" + // ChatShellRuns lists the session's recorded runs, oldest first, each with + // at most ChatShellRunListOutput bytes of its retained output tail. The + // body is ChatQuery. + ChatShellRuns = "chat_shell_runs" + // ChatShellRunOutput reads one run with its whole retained output (at most + // ChatShellRunRetained bytes). The body is ChatShellRunRequest. + ChatShellRunOutput = "chat_shell_run_output" + // ChatShellShare queues an ordinary prompt carrying the run's command, exit + // status and bounded output tail. The body is ChatShellRunRequest; the + // reply is the chat_prompt acknowledgement. + ChatShellShare = "chat_shell_share" + // ChatShellRunEvent is pushed to every client watching the session: a + // lifecycle change (Run set) or an output chunk (Output set). + ChatShellRunEvent = "chat_shell_run_event" +) + +// ChatShellRunRecord status values. Running is the only non-final status. +const ( + ChatShellRunRunning = "running" + ChatShellRunExited = "exited" + ChatShellRunStopped = "stopped" + ChatShellRunFailed = "failed" + ChatShellRunInterrupted = "interrupted" +) + +// Bounds of operator shell commands. +const ( + // ChatShellRunCommandLimit bounds one command line. + ChatShellRunCommandLimit = 16 << 10 + // ChatShellRunRetained is the retained (and persisted) output tail of one + // run: older output is dropped and the record says so. + ChatShellRunRetained = 1 << 20 + // ChatShellRunListOutput bounds the output tail each run carries in a + // chat_shell_runs list, so a long history stays one small frame. + ChatShellRunListOutput = 64 << 10 + // ChatShellRunChunkLimit bounds one live output frame. + ChatShellRunChunkLimit = 64 << 10 + // ChatShellRunShareOutput bounds the output tail shared with the agent. + ChatShellRunShareOutput = 16 << 10 + // ChatShellRunConcurrent bounds the commands one session runs at once. + ChatShellRunConcurrent = 4 + // ChatShellRunHistory bounds the finished runs kept per session. + ChatShellRunHistory = 50 +) + +// ChatShellRunRequest names one session and either a new command or a run. +// OperationID is the optional retry identity of chat_shell_share's prompt. +type ChatShellRunRequest struct { + SessionID string `json:"session_id"` + Command string `json:"command,omitempty"` + RunID string `json:"run_id,omitempty"` + OperationID string `json:"operation_id,omitempty"` +} + +// ChatShellRunRecord is one operator command and what it produced. Output is +// the retained text starting at absolute byte OutputStart of everything the +// command wrote (stdout and stderr merged); OutputBytes is that total. A record +// with OutputStart > 0 dropped its beginning. Times are epoch milliseconds. +// ExitCode is set once the process exited with a code; a signal-terminated +// or never-started command has none. +type ChatShellRunRecord struct { + ID string `json:"id"` + SessionID string `json:"session_id"` + Command string `json:"command"` + Directory string `json:"directory"` + Status string `json:"status"` + ExitCode *int `json:"exit_code,omitempty"` + Signal string `json:"signal,omitempty"` + Error string `json:"error,omitempty"` + StartedAt int64 `json:"started_at"` + EndedAt int64 `json:"ended_at,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` + Output string `json:"output,omitempty"` + OutputStart int64 `json:"output_start,omitempty"` + OutputBytes int64 `json:"output_bytes,omitempty"` + // Truncated is true when Output does not hold everything the command + // wrote (OutputStart > 0). + Truncated bool `json:"truncated,omitempty"` +} + +// ChatShellRunEventFrame is one live change to a run. A lifecycle frame sets +// Run (without output); an output frame sets Output covering absolute bytes +// [Offset, End). A client whose own copy ends before Offset missed output and +// shows that gap rather than joining unrelated text. +type ChatShellRunEventFrame struct { + SessionID string `json:"session_id"` + RunID string `json:"run_id"` + Run *ChatShellRunRecord `json:"run,omitempty"` + Output string `json:"output,omitempty"` + Offset int64 `json:"offset,omitempty"` + End int64 `json:"end,omitempty"` +} diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 3b4d0e37..19d9e38a 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -262,6 +262,12 @@ type Message struct { ChatShellOutputQuery *ChatShellOutputQuery `json:"chat_shell_output_query,omitempty"` ChatShells []ChatShell `json:"chat_shells,omitempty"` ChatShellOutput *ChatShellOutputPage `json:"chat_shell_output,omitempty"` + // Operator shell commands (chat_shell_run.go): the request body, one run, + // a session's runs and the pushed live change. + ChatShellRunRequest *ChatShellRunRequest `json:"chat_shell_run_request,omitempty"` + ChatShellRun *ChatShellRunRecord `json:"chat_shell_run,omitempty"` + ChatShellRuns []ChatShellRunRecord `json:"chat_shell_runs,omitempty"` + ChatShellRunEvent *ChatShellRunEventFrame `json:"chat_shell_run_event,omitempty"` // ListDirectory is the list_directory body and DirectoryListing its reply: // the home-restricted folder picker listing (see directories.go). ListDirectory *DirectoryListRequest `json:"list_directory,omitempty"` diff --git a/internal/server/chat.go b/internal/server/chat.go index dc3e0eb9..4ef3031e 100644 --- a/internal/server/chat.go +++ b/internal/server/chat.go @@ -255,6 +255,9 @@ func handleChat(ctx context.Context, request protocol.Message, broker chatBroker case protocol.ChatShellOutput: handleChatShellOutput(ctx, request, broker, registry, write) return true + case protocol.ChatShellRun, protocol.ChatShellStop, protocol.ChatShellRuns, protocol.ChatShellRunOutput, protocol.ChatShellShare: + handleChatShellRunRequest(ctx, request, broker, hub, registry, clientID, write) + return true case protocol.ChatAttachmentImage: handleChatAttachmentImage(request, broker, write) return true diff --git a/internal/server/chat_hub.go b/internal/server/chat_hub.go index 71686418..0e14f29e 100644 --- a/internal/server/chat_hub.go +++ b/internal/server/chat_hub.go @@ -74,6 +74,9 @@ type chatHub struct { // and its sequence source (chat_activity.go). activity map[string]protocol.ChatActivity activitySeq int64 + // shellRuns runs the operator's own shell commands (chat_shell_runner.go); + // nil refuses them. + shellRuns *chatShellRunner } // chatPart is the retained form of one part. Text and reasoning deltas append to diff --git a/internal/server/chat_shell_run_handlers.go b/internal/server/chat_shell_run_handlers.go new file mode 100644 index 00000000..66e38983 --- /dev/null +++ b/internal/server/chat_shell_run_handlers.go @@ -0,0 +1,141 @@ +package server + +import ( + "context" + "errors" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// handleChatShellRunRequest answers the operator shell-command requests. Every +// one names an operator chat session the registry records: the command runs +// only in that chat's own directory on this server's machine, and a run is +// only readable or stoppable through the session that started it. +func handleChatShellRunRequest(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error) { + fail := func(err error) { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) } + runner := hub.shellRunner() + if runner == nil { + fail(errors.New("shell commands are not configured on this server")) + return + } + var sessionID string + body := request.ChatShellRunRequest + switch { + case request.Type == protocol.ChatShellRuns && request.ChatQuery != nil: + sessionID = request.ChatQuery.SessionID + case body != nil: + sessionID = body.SessionID + } + if !validChatSessionID(sessionID) { + fail(errors.New("a valid chat session id is required")) + return + } + if registry == nil { + fail(errors.New("operator session registry is not configured on this server")) + return + } + session, found, err := registry.OperatorSession(ctx, sessionID) + if err != nil { + fail(err) + return + } + if !found { + fail(errors.New("operator session is not available")) + return + } + if request.Type != protocol.ChatShellRuns && body == nil { + fail(errors.New("a chat shell request body is required")) + return + } + switch request.Type { + case protocol.ChatShellRun: + record, err := runner.Start(ctx, sessionID, session.Directory, body.Command) + if err != nil { + fail(err) + return + } + _ = write(protocol.Message{Type: protocol.ChatShellRun, ID: request.ID, ChatShellRun: &record}) + case protocol.ChatShellStop: + record, err := runner.Stop(sessionID, body.RunID) + if err != nil { + fail(err) + return + } + _ = write(protocol.Message{Type: protocol.ChatShellStop, ID: request.ID, ChatShellRun: &record}) + case protocol.ChatShellRuns: + runs, err := runner.List(ctx, sessionID) + if err != nil { + fail(err) + return + } + _ = write(protocol.Message{Type: protocol.ChatShellRuns, ID: request.ID, ChatShellRuns: runs}) + case protocol.ChatShellRunOutput: + record, err := runner.Read(ctx, sessionID, body.RunID) + if err != nil { + fail(err) + return + } + _ = write(protocol.Message{Type: protocol.ChatShellRunOutput, ID: request.ID, ChatShellRun: &record}) + case protocol.ChatShellShare: + record, err := runner.Read(ctx, sessionID, body.RunID) + if err != nil { + fail(err) + return + } + if record.Status == protocol.ChatShellRunRunning { + fail(errors.New("wait for the command to finish before sending it to the agent")) + return + } + prompt := protocol.Message{Type: protocol.ChatPrompt, ID: request.ID, ChatPrompt: &protocol.ChatPromptRequest{ + SessionID: sessionID, Text: chatShellSharePrompt(record), OperationID: body.OperationID, + }} + admitChatPrompt(ctx, prompt, broker, hub, registry, clientID, write, true) + } +} + +// List is the session's recorded runs, oldest first. A running run carries +// its live output rather than the last periodic save; every run carries at +// most protocol.ChatShellRunListOutput bytes of its tail. +func (r *chatShellRunner) List(ctx context.Context, sessionID string) ([]protocol.ChatShellRunRecord, error) { + runs, err := r.store.ChatShellRuns(ctx, sessionID) + if err != nil { + return nil, err + } + for i, run := range runs { + if p := r.activeProcess(sessionID, run.ID); p != nil { + runs[i] = p.snapshot(protocol.ChatShellRunListOutput) + continue + } + runs[i] = chatShellRecordTail(run, protocol.ChatShellRunListOutput) + } + return runs, nil +} + +// Read is one run of sessionID with its whole retained output. +func (r *chatShellRunner) Read(ctx context.Context, sessionID, runID string) (protocol.ChatShellRunRecord, error) { + if p := r.activeProcess(sessionID, runID); p != nil { + return p.snapshot(protocol.ChatShellRunRetained), nil + } + record, found, err := r.store.ChatShellRun(ctx, sessionID, runID) + if err != nil { + return protocol.ChatShellRunRecord{}, err + } + if !found { + return protocol.ChatShellRunRecord{}, errors.New("this chat has no such command") + } + return record, nil +} + +// chatShellRecordTail trims a stored record's output to its last max bytes. +func chatShellRecordTail(record protocol.ChatShellRunRecord, max int) protocol.ChatShellRunRecord { + if len(record.Output) <= max { + return record + } + out := newChatShellOutput(len(record.Output)) + out.write([]byte(record.Output)) + text, from := out.tail(max) + record.Output = text + record.OutputStart += from + record.Truncated = true + return record +} diff --git a/internal/server/chat_shell_run_output.go b/internal/server/chat_shell_run_output.go new file mode 100644 index 00000000..53635a53 --- /dev/null +++ b/internal/server/chat_shell_run_output.go @@ -0,0 +1,96 @@ +package server + +import ( + "strings" + "unicode/utf8" +) + +// chatShellOutput is the bounded tail of one operator command's merged output. +// Offsets are absolute bytes of everything the command wrote, so a client can +// tell a continuation from a gap. The buffer may grow to twice its limit before +// compacting, which keeps a flooding command from copying the tail per write. +type chatShellOutput struct { + limit int + data []byte + start int64 + total int64 + emitted int64 +} + +func newChatShellOutput(limit int) *chatShellOutput { + return &chatShellOutput{limit: limit} +} + +func (o *chatShellOutput) write(p []byte) { + o.data = append(o.data, p...) + o.total += int64(len(p)) + if len(o.data) <= 2*o.limit { + return + } + cut := runeStartAfter(o.data, len(o.data)-o.limit) + o.data = append([]byte(nil), o.data[cut:]...) + o.start += int64(cut) +} + +// tail returns at most max retained bytes ending at the newest byte, starting +// on a character boundary, with the absolute offset of its first byte. +func (o *chatShellOutput) tail(max int) (string, int64) { + from := 0 + if len(o.data) > max { + from = runeStartAfter(o.data, len(o.data)-max) + } + return validChatShellText(o.data[from:]), o.start + int64(from) +} + +// pending returns the output not yet emitted, at most max bytes (the newest +// ones when more is waiting: the skipped bytes are a visible gap to clients). +// An incomplete trailing character is held back unless final is set. +func (o *chatShellOutput) pending(max int, final bool) (text string, offset, end int64, ok bool) { + from := o.emitted + if from < o.start { + from = o.start + } + available := o.data[from-o.start:] + if !final { + available = available[:completeUTF8Prefix(available)] + } + if len(available) == 0 { + return "", 0, 0, false + } + end = from + int64(len(available)) + if len(available) > max { + skip := runeStartAfter(available, len(available)-max) + available = available[skip:] + from += int64(skip) + } + o.emitted = end + return validChatShellText(available), from, end, true +} + +// runeStartAfter moves index forward (at most three bytes) to the first byte +// that starts a character, so a cut never splits one. +func runeStartAfter(data []byte, index int) int { + for step := 0; step < utf8.UTFMax-1 && index < len(data) && !utf8.RuneStart(data[index]); step++ { + index++ + } + return index +} + +// completeUTF8Prefix is the length of data without a trailing partial +// character that a later read may complete. +func completeUTF8Prefix(data []byte) int { + for back := 1; back <= utf8.UTFMax-1 && back <= len(data); back++ { + if utf8.RuneStart(data[len(data)-back]) { + if !utf8.FullRune(data[len(data)-back:]) { + return len(data) - back + } + break + } + } + return len(data) +} + +// validChatShellText keeps binary output representable in a JSON frame. +func validChatShellText(data []byte) string { + return strings.ToValidUTF8(string(data), "�") +} diff --git a/internal/server/chat_shell_run_test.go b/internal/server/chat_shell_run_test.go new file mode 100644 index 00000000..149e7a2e --- /dev/null +++ b/internal/server/chat_shell_run_test.go @@ -0,0 +1,190 @@ +package server + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/coder/websocket" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// shellChat is one server with a recorded operator session whose directory is +// a fresh temporary folder, and two connections watching that session. +func shellChat(t *testing.T) (context.Context, []*websocket.Conn, string) { + t.Helper() + store := newOperatorStore(t) + broker := scriptedChat() + broker.block = true + ctx, _, conns := dialChatConns(t, Options{Chat: broker, OperatorRegistry: store}, 2) + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := store.RecordOperatorSession(ctx, "ses_1", dir, "Shell", protocol.ChatSessionTagOperator); err != nil { + t.Fatal(err) + } + for i, conn := range conns { + writeChat(t, ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}) + if got := readShellFrame(t, ctx, conn, func(m protocol.Message) bool { return m.ID == "sub" }); got.Type != protocol.ChatSubscribe { + t.Fatalf("subscribe %d = %+v", i, got) + } + } + return ctx, conns, dir +} + +func readShellFrame(t *testing.T, ctx context.Context, conn *websocket.Conn, match func(protocol.Message) bool) protocol.Message { + t.Helper() + for { + var message protocol.Message + if err := readTestFrame(ctx, conn, &message); err != nil { + t.Fatal(err) + } + if match(message) { + return message + } + } +} + +func shellRequest(kind, id, runID, command string) protocol.Message { + return protocol.Message{Type: kind, ID: id, ChatShellRunRequest: &protocol.ChatShellRunRequest{SessionID: "ses_1", Command: command, RunID: runID}} +} + +// runShell starts command and returns its running record. +func runShell(t *testing.T, ctx context.Context, conn *websocket.Conn, id, command string) protocol.ChatShellRunRecord { + t.Helper() + writeChat(t, ctx, conn, shellRequest(protocol.ChatShellRun, id, "", command)) + reply := readShellFrame(t, ctx, conn, func(m protocol.Message) bool { return m.ID == id }) + if reply.Type != protocol.ChatShellRun || reply.ChatShellRun == nil || reply.ChatShellRun.Status != protocol.ChatShellRunRunning { + t.Fatalf("chat_shell_run reply = %+v", reply) + } + return *reply.ChatShellRun +} + +// finishedShell reads conn's frames until runID's final lifecycle frame and +// returns it with the live output joined from its output frames. +func finishedShell(t *testing.T, ctx context.Context, conn *websocket.Conn, runID string) (protocol.ChatShellRunRecord, string) { + t.Helper() + var output strings.Builder + var end int64 + for { + frame := readShellFrame(t, ctx, conn, func(m protocol.Message) bool { + return m.Type == protocol.ChatShellRunEvent && m.ChatShellRunEvent != nil && m.ChatShellRunEvent.RunID == runID + }).ChatShellRunEvent + if frame.Run == nil { + if frame.Offset != end { + t.Fatalf("output frame at %d after %d", frame.Offset, end) + } + output.WriteString(frame.Output) + end = frame.End + continue + } + if frame.Run.Status != protocol.ChatShellRunRunning { + return *frame.Run, output.String() + } + } +} + +func TestChatShellRunStreamsToEveryWatcherAndPersists(t *testing.T) { + ctx, conns, dir := shellChat(t) + run := runShell(t, ctx, conns[0], "r1", `pwd; echo out; echo err >&2; exit 3`) + if run.Directory != dir || run.Command == "" || run.ID == "" { + t.Fatalf("running record = %+v", run) + } + for _, conn := range conns { + final, output := finishedShell(t, ctx, conn, run.ID) + if final.Status != protocol.ChatShellRunExited || final.ExitCode == nil || *final.ExitCode != 3 || final.EndedAt < final.StartedAt { + t.Fatalf("final record = %+v", final) + } + if output != dir+"\nout\nerr\n" { + t.Fatalf("streamed output = %q", output) + } + } + writeChat(t, ctx, conns[1], protocol.Message{Type: protocol.ChatShellRuns, ID: "list", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}}) + list := readShellFrame(t, ctx, conns[1], func(m protocol.Message) bool { return m.ID == "list" }) + if len(list.ChatShellRuns) != 1 || list.ChatShellRuns[0].Output != dir+"\nout\nerr\n" || *list.ChatShellRuns[0].ExitCode != 3 { + t.Fatalf("persisted runs = %+v", list) + } +} + +func TestChatShellStopInterruptsTheGroup(t *testing.T) { + ctx, conns, _ := shellChat(t) + // The child sleep is in the shell's group, so the stop reaches it too. + run := runShell(t, ctx, conns[0], "r1", `echo started; sleep 30; echo never`) + writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellStop, "stop", run.ID, "")) + if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "stop" }); reply.Type != protocol.ChatShellStop { + t.Fatalf("stop reply = %+v", reply) + } + final, output := finishedShell(t, ctx, conns[1], run.ID) + if final.Status != protocol.ChatShellRunStopped || strings.Contains(output, "never") || final.DurationMS > 10_000 { + t.Fatalf("stopped record = %+v output=%q", final, output) + } + writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellStop, "again", run.ID, "")) + if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "again" }); reply.Type != protocol.Error { + t.Fatalf("stopping a finished run = %+v", reply) + } +} + +func TestChatShellRunLimitsAndRefusals(t *testing.T) { + ctx, conns, dir := shellChat(t) + var runs []protocol.ChatShellRunRecord + for i := 0; i < protocol.ChatShellRunConcurrent; i++ { + runs = append(runs, runShell(t, ctx, conns[0], "r"+string(rune('a'+i)), "sleep 30")) + } + refusals := []protocol.Message{ + shellRequest(protocol.ChatShellRun, "fifth", "", "sleep 30"), + shellRequest(protocol.ChatShellRun, "blank", "", " "), + shellRequest(protocol.ChatShellRun, "huge", "", strings.Repeat("x", protocol.ChatShellRunCommandLimit+1)), + {Type: protocol.ChatShellRun, ID: "nobody", ChatShellRunRequest: &protocol.ChatShellRunRequest{SessionID: "ses_unknown", Command: "true"}}, + {Type: protocol.ChatShellRun, ID: "badid", ChatShellRunRequest: &protocol.ChatShellRunRequest{SessionID: "../ses", Command: "true"}}, + {Type: protocol.ChatShellRun, ID: "nobody-body"}, + {Type: protocol.ChatShellStop, ID: "foreign", ChatShellRunRequest: &protocol.ChatShellRunRequest{SessionID: "ses_unknown", RunID: runs[0].ID}}, + shellRequest(protocol.ChatShellShare, "share-running", runs[0].ID, ""), + } + for _, request := range refusals { + writeChat(t, ctx, conns[0], request) + if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == request.ID }); reply.Type != protocol.Error { + t.Fatalf("%s was not refused: %+v", request.ID, reply) + } + } + for _, run := range runs { + writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellStop, "stop-"+run.ID, run.ID, "")) + readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "stop-"+run.ID }) + } + for range runs { + readShellFrame(t, ctx, conns[1], func(m protocol.Message) bool { + return m.Type == protocol.ChatShellRunEvent && m.ChatShellRunEvent.Run != nil && m.ChatShellRunEvent.Run.Status == protocol.ChatShellRunStopped + }) + } + // A deleted working directory is refused rather than run somewhere else. + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } + writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellRun, "gone", "", "true")) + if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "gone" }); reply.Type != protocol.Error || !strings.Contains(reply.Error, "not available") { + t.Fatalf("missing directory = %+v", reply) + } +} + +func TestChatShellShareQueuesAnOrdinaryPrompt(t *testing.T) { + ctx, conns, dir := shellChat(t) + run := runShell(t, ctx, conns[0], "r1", "echo shared-output") + finishedShell(t, ctx, conns[0], run.ID) + writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellShare, "share", run.ID, "")) + // The composed prompt is echoed to the sender too: it has no local copy. + for _, conn := range conns { + user := readShellFrame(t, ctx, conn, func(m protocol.Message) bool { + return m.Type == protocol.ChatEvent && m.ChatEvent != nil && m.ChatEvent.Kind == protocol.ChatEventUser + }).ChatEvent + text := user.Part.Text + if !strings.Contains(text, "$ echo shared-output") || !strings.Contains(text, "shared-output\n```") || !strings.Contains(text, dir) || !strings.Contains(text, "exited with status 0") { + t.Fatalf("shared prompt = %q", text) + } + } + if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "share" }); reply.Type != protocol.ChatPrompt { + t.Fatalf("share reply = %+v", reply) + } +} diff --git a/internal/server/chat_shell_runner.go b/internal/server/chat_shell_runner.go new file mode 100644 index 00000000..19675991 --- /dev/null +++ b/internal/server/chat_shell_runner.go @@ -0,0 +1,256 @@ +package server + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// chatShellRunStore persists operator shell runs; *pipeline.Store implements it. +type chatShellRunStore interface { + SaveChatShellRun(ctx context.Context, run protocol.ChatShellRunRecord, keep int) error + ChatShellRuns(ctx context.Context, sessionID string) ([]protocol.ChatShellRunRecord, error) + ChatShellRun(ctx context.Context, sessionID, id string) (protocol.ChatShellRunRecord, bool, error) + InterruptChatShellRuns(ctx context.Context, endedAt int64, reason string) (int64, error) +} + +const ( + chatShellInterruptedReason = "The server stopped while this command was running." + // chatShellDrain is how long output is still read after the command itself + // exited, for descendants that keep the pipe open; the group is then killed. + chatShellDrain = time.Second +) + +// chatShellRunner owns the operator's own commands: each runs `$SHELL -lc` in +// its chat's directory, in its own process group, with stdout and stderr +// merged into one bounded tail that is streamed to the session's watchers and +// persisted as Pipelines' record of the run. +type chatShellRunner struct { + base context.Context + store chatShellRunStore + publish func(sessionID string, frame protocol.ChatShellRunEventFrame) + now func() time.Time + shell func() string + // grace separates Stop's SIGINT from its SIGKILL; flush batches live output + // frames; persist is the interval a running run's output is saved at. + grace time.Duration + flush time.Duration + persist time.Duration + + mu sync.Mutex + active map[string]*chatShellProcess + wg sync.WaitGroup +} + +type chatShellProcess struct { + mu sync.Mutex + record protocol.ChatShellRunRecord + out *chatShellOutput + cmd *exec.Cmd + stopped bool + done chan struct{} +} + +func newChatShellRunner(base context.Context, store chatShellRunStore, publish func(string, protocol.ChatShellRunEventFrame)) *chatShellRunner { + return &chatShellRunner{ + base: base, store: store, publish: publish, now: time.Now, shell: chatShellProgram, + grace: 3 * time.Second, flush: 100 * time.Millisecond, persist: 2 * time.Second, + active: map[string]*chatShellProcess{}, + } +} + +// chatShellProgram is the operator's login shell, or /bin/sh without one. +func chatShellProgram() string { + if shell := os.Getenv("SHELL"); filepath.IsAbs(shell) { + if info, err := os.Stat(shell); err == nil && !info.IsDir() { + return shell + } + } + return "/bin/sh" +} + +// recover marks runs a previous server process left running as interrupted: +// their processes died with it (or are no longer owned by anyone). +func (r *chatShellRunner) recover(ctx context.Context) { + if r.store == nil { + return + } + _, _ = r.store.InterruptChatShellRuns(ctx, r.now().UnixMilli(), chatShellInterruptedReason) +} + +// resolveChatShellDirectory requires the chat's recorded directory to still be +// an existing directory and returns its symlink-free path. +func resolveChatShellDirectory(directory string) (string, error) { + directory = strings.TrimSpace(directory) + if directory == "" || !filepath.IsAbs(directory) { + return "", errors.New("this chat has no absolute working directory to run commands in") + } + real, err := filepath.EvalSymlinks(directory) + if err != nil { + return "", fmt.Errorf("the chat's directory %s is not available: %v", directory, err) + } + info, err := os.Stat(real) + if err != nil || !info.IsDir() { + return "", fmt.Errorf("the chat's directory %s is not a directory", directory) + } + return real, nil +} + +func newChatShellRunID() (string, error) { + var b [12]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return "run_" + hex.EncodeToString(b[:]), nil +} + +// Start launches command for sessionID in directory and returns its running +// record. Validation and start failures are returned without a record. +func (r *chatShellRunner) Start(ctx context.Context, sessionID, directory, command string) (protocol.ChatShellRunRecord, error) { + if r.store == nil { + return protocol.ChatShellRunRecord{}, errors.New("shell commands need the server's store, which is not configured") + } + if strings.TrimSpace(command) == "" { + return protocol.ChatShellRunRecord{}, errors.New("a nonempty command is required") + } + if len(command) > protocol.ChatShellRunCommandLimit { + return protocol.ChatShellRunRecord{}, fmt.Errorf("the command exceeds %d KiB", protocol.ChatShellRunCommandLimit>>10) + } + dir, err := resolveChatShellDirectory(directory) + if err != nil { + return protocol.ChatShellRunRecord{}, err + } + id, err := newChatShellRunID() + if err != nil { + return protocol.ChatShellRunRecord{}, err + } + r.mu.Lock() + defer r.mu.Unlock() + if r.base.Err() != nil { + return protocol.ChatShellRunRecord{}, errors.New("the server is stopping") + } + running := 0 + for _, p := range r.active { + if p.record.SessionID == sessionID { + running++ + } + } + if running >= protocol.ChatShellRunConcurrent { + return protocol.ChatShellRunRecord{}, fmt.Errorf("this chat already runs %d commands; stop one or wait for it to finish", running) + } + reader, writer, err := os.Pipe() + if err != nil { + return protocol.ChatShellRunRecord{}, err + } + cmd := exec.Command(r.shell(), "-lc", command) + cmd.Dir = dir + cmd.Env = os.Environ() + cmd.Stdout, cmd.Stderr = writer, writer + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + reader.Close() + writer.Close() + return protocol.ChatShellRunRecord{}, fmt.Errorf("could not start the command: %v", err) + } + writer.Close() + record := protocol.ChatShellRunRecord{ID: id, SessionID: sessionID, Command: command, Directory: dir, Status: protocol.ChatShellRunRunning, StartedAt: r.now().UnixMilli()} + p := &chatShellProcess{record: record, out: newChatShellOutput(protocol.ChatShellRunRetained), cmd: cmd, done: make(chan struct{})} + if err := r.store.SaveChatShellRun(ctx, record, protocol.ChatShellRunHistory); err != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + reader.Close() + _ = cmd.Wait() + return protocol.ChatShellRunRecord{}, fmt.Errorf("could not record the command: %v", err) + } + r.active[id] = p + r.wg.Add(1) + go r.supervise(p, reader) + r.publishRun(record) + return record, nil +} + +// Stop interrupts a running command's whole process group, then kills it +// after the grace period. It returns the run's current record. +func (r *chatShellRunner) Stop(sessionID, runID string) (protocol.ChatShellRunRecord, error) { + r.mu.Lock() + p := r.active[runID] + r.mu.Unlock() + if p == nil || p.record.SessionID != sessionID { + return protocol.ChatShellRunRecord{}, errors.New("this command is not running") + } + p.mu.Lock() + p.stopped = true + record := p.record + p.mu.Unlock() + pid := p.cmd.Process.Pid + _ = syscall.Kill(-pid, syscall.SIGINT) + go func() { + select { + case <-p.done: + case <-time.After(r.grace): + _ = syscall.Kill(-pid, syscall.SIGKILL) + } + }() + return record, nil +} + +// Wait blocks until every supervised command has finished (tests, shutdown). +func (r *chatShellRunner) Wait() { r.wg.Wait() } + +func (r *chatShellRunner) activeProcess(sessionID, runID string) *chatShellProcess { + r.mu.Lock() + defer r.mu.Unlock() + if p := r.active[runID]; p != nil && p.record.SessionID == sessionID { + return p + } + return nil +} + +// snapshot is the process's current record carrying at most max output bytes. +func (p *chatShellProcess) snapshot(max int) protocol.ChatShellRunRecord { + p.mu.Lock() + defer p.mu.Unlock() + record := p.record + record.Output, record.OutputStart = p.out.tail(max) + record.OutputBytes = p.out.total + record.Truncated = record.OutputStart > 0 + return record +} + +func (r *chatShellRunner) publishRun(record protocol.ChatShellRunRecord) { + if r.publish == nil { + return + } + record.Output = "" + r.publish(record.SessionID, protocol.ChatShellRunEventFrame{SessionID: record.SessionID, RunID: record.ID, Run: &record}) +} + +func (h *chatHub) shellRunner() *chatShellRunner { + if h == nil { + return nil + } + return h.shellRuns +} + +// startShellRunner gives the hub its operator command runner over store, +// streaming to the session's room, after interrupting runs a previous server +// process left recorded as running. A nil store leaves commands refused. +func (h *chatHub) startShellRunner(ctx context.Context, store chatShellRunStore) { + if store == nil { + return + } + h.shellRuns = newChatShellRunner(ctx, store, func(sessionID string, frame protocol.ChatShellRunEventFrame) { + h.fanout(sessionID, protocol.Message{Type: protocol.ChatShellRunEvent, ChatShellRunEvent: &frame}, "") + }) + h.shellRuns.recover(ctx) +} diff --git a/internal/server/chat_shell_runner_test.go b/internal/server/chat_shell_runner_test.go new file mode 100644 index 00000000..2eeb2ef0 --- /dev/null +++ b/internal/server/chat_shell_runner_test.go @@ -0,0 +1,143 @@ +package server + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// shellFrames records every frame a test runner publishes. +type shellFrames struct { + mu sync.Mutex + frames []protocol.ChatShellRunEventFrame +} + +func (f *shellFrames) add(_ string, frame protocol.ChatShellRunEventFrame) { + f.mu.Lock() + f.frames = append(f.frames, frame) + f.mu.Unlock() +} + +func (f *shellFrames) final(runID string) *protocol.ChatShellRunRecord { + f.mu.Lock() + defer f.mu.Unlock() + for _, frame := range f.frames { + if frame.RunID == runID && frame.Run != nil && frame.Run.Status != protocol.ChatShellRunRunning { + return frame.Run + } + } + return nil +} + +func testShellRunner(t *testing.T, ctx context.Context) (*chatShellRunner, *shellFrames) { + t.Helper() + frames := &shellFrames{} + runner := newChatShellRunner(ctx, newOperatorStore(t), frames.add) + runner.grace = 200 * time.Millisecond + runner.flush = 10 * time.Millisecond + return runner, frames +} + +func TestChatShellStopKillsAfterGrace(t *testing.T) { + runner, frames := testShellRunner(t, context.Background()) + run, err := runner.Start(context.Background(), "ses_1", t.TempDir(), `trap "" INT; echo ready; while :; do sleep 0.05; done`) + if err != nil { + t.Fatal(err) + } + time.Sleep(150 * time.Millisecond) + if _, err := runner.Stop("ses_2", run.ID); err == nil { + t.Fatal("another session stopped this run") + } + if _, err := runner.Stop("ses_1", run.ID); err != nil { + t.Fatal(err) + } + runner.Wait() + final := frames.final(run.ID) + if final == nil || final.Status != protocol.ChatShellRunStopped || final.Signal != "killed" || final.DurationMS < 200 { + t.Fatalf("final = %+v", final) + } +} + +func TestChatShellRetainsBoundedTail(t *testing.T) { + runner, frames := testShellRunner(t, context.Background()) + total := protocol.ChatShellRunRetained + 200_000 + run, err := runner.Start(context.Background(), "ses_1", t.TempDir(), `head -c 1248576 /dev/zero | tr '\0' a; printf END`) + if err != nil { + t.Fatal(err) + } + runner.Wait() + stored, err := runner.Read(context.Background(), "ses_1", run.ID) + if err != nil { + t.Fatal(err) + } + if stored.OutputBytes != int64(total+3) || len(stored.Output) != protocol.ChatShellRunRetained || !stored.Truncated || stored.OutputStart != stored.OutputBytes-int64(protocol.ChatShellRunRetained) || !strings.HasSuffix(stored.Output, "aEND") { + t.Fatalf("stored = bytes %d len %d start %d truncated %v", stored.OutputBytes, len(stored.Output), stored.OutputStart, stored.Truncated) + } + list, err := runner.List(context.Background(), "ses_1") + if err != nil || len(list) != 1 || len(list[0].Output) != protocol.ChatShellRunListOutput || list[0].OutputStart != stored.OutputBytes-protocol.ChatShellRunListOutput { + t.Fatalf("list = %d runs, err %v", len(list), err) + } + frames.mu.Lock() + defer frames.mu.Unlock() + for _, frame := range frames.frames { + if len(frame.Output) > protocol.ChatShellRunChunkLimit { + t.Fatalf("an output frame carried %d bytes", len(frame.Output)) + } + } +} + +func TestChatShellShutdownAndRestartInterrupt(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + runner, frames := testShellRunner(t, ctx) + run, err := runner.Start(context.Background(), "ses_1", t.TempDir(), "sleep 30") + if err != nil { + t.Fatal(err) + } + cancel() + runner.Wait() + if final := frames.final(run.ID); final == nil || final.Status != protocol.ChatShellRunInterrupted { + t.Fatalf("shutdown final = %+v", final) + } + if _, err := runner.Start(context.Background(), "ses_1", t.TempDir(), "true"); err == nil { + t.Fatal("a stopping server started a command") + } + // A record a crashed process left running is interrupted on start. + store := runner.store + if err := store.SaveChatShellRun(context.Background(), protocol.ChatShellRunRecord{ID: "run_left", SessionID: "ses_1", Command: "sleep 1", Directory: "/", Status: protocol.ChatShellRunRunning, StartedAt: 1}, 0); err != nil { + t.Fatal(err) + } + next := newChatShellRunner(context.Background(), store, nil) + next.recover(context.Background()) + left, err := next.Read(context.Background(), "ses_1", "run_left") + if err != nil || left.Status != protocol.ChatShellRunInterrupted || left.Error != chatShellInterruptedReason { + t.Fatalf("recovered = %+v, %v", left, err) + } +} + +func TestChatShellOutputKeepsCharactersWhole(t *testing.T) { + out := newChatShellOutput(4) + out.write([]byte("ab\xc3")) + if text, offset, end, ok := out.pending(64, false); !ok || text != "ab" || offset != 0 || end != 2 { + t.Fatalf("partial character emitted: %q %d %d", text, offset, end) + } + out.write([]byte("\xa9cdefghij")) + text, offset, end, ok := out.pending(3, false) + if !ok || text != "hij" || offset != 9 || end != 12 { + t.Fatalf("bounded chunk = %q %d %d", text, offset, end) + } + if tail, start := out.tail(6); tail != "ghij" || start != 8 { + t.Fatalf("tail = %q at %d", tail, start) + } +} + +func TestChatShellSharePromptFencesOutput(t *testing.T) { + code := 1 + prompt := chatShellSharePrompt(protocol.ChatShellRunRecord{Command: "cat notes.md", Directory: "/w", Status: protocol.ChatShellRunExited, ExitCode: &code, DurationMS: 1500, Output: "```go\nx\n```\n", OutputBytes: 13}) + if !strings.Contains(prompt, "````text\n```go\nx\n```\n````") || !strings.Contains(prompt, "exited with status 1 after 1.5s") { + t.Fatalf("prompt = %q", prompt) + } +} diff --git a/internal/server/chat_shell_share.go b/internal/server/chat_shell_share.go new file mode 100644 index 00000000..ee77e8c9 --- /dev/null +++ b/internal/server/chat_shell_share.go @@ -0,0 +1,78 @@ +package server + +import ( + "fmt" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// chatShellSharePrompt is the ordinary prompt chat_shell_share queues: the +// command, where it ran, how it ended and a bounded tail of its output. It is +// the only path by which an operator command reaches the agent. +func chatShellSharePrompt(record protocol.ChatShellRunRecord) string { + output := chatShellRecordTail(record, protocol.ChatShellRunShareOutput) + var b strings.Builder + fmt.Fprintf(&b, "I ran this shell command in %s:\n\n", record.Directory) + fence := chatShellFence(record.Command) + fmt.Fprintf(&b, "%ssh\n$ %s\n%s\n\n", fence, record.Command, fence) + fmt.Fprintf(&b, "%s.\n\n", chatShellOutcome(record)) + if output.Output == "" { + b.WriteString("It printed no output.") + return b.String() + } + if output.OutputStart > 0 { + fmt.Fprintf(&b, "Output (the last %s of %s):\n\n", chatShellBytes(int64(len(output.Output))), chatShellBytes(output.OutputBytes)) + } else { + b.WriteString("Output:\n\n") + } + fence = chatShellFence(output.Output) + text := strings.TrimRight(output.Output, "\n") + fmt.Fprintf(&b, "%stext\n%s\n%s", fence, text, fence) + return b.String() +} + +// chatShellOutcome describes how a run ended, in the card's own words. +func chatShellOutcome(record protocol.ChatShellRunRecord) string { + took := (time.Duration(record.DurationMS) * time.Millisecond).Round(time.Millisecond) + switch record.Status { + case protocol.ChatShellRunStopped: + return fmt.Sprintf("I stopped it after %s", took) + case protocol.ChatShellRunInterrupted: + return "It was interrupted when the server stopped" + case protocol.ChatShellRunFailed: + return "It failed: " + record.Error + } + if record.Signal != "" { + return fmt.Sprintf("It was killed by %s after %s", record.Signal, took) + } + if record.ExitCode != nil { + return fmt.Sprintf("It exited with status %d after %s", *record.ExitCode, took) + } + return "It ended" +} + +// chatShellFence is a Markdown code fence longer than any backtick run in text. +func chatShellFence(text string) string { + longest, run := 0, 0 + for _, r := range text { + if r == '`' { + run++ + longest = max(longest, run) + } else { + run = 0 + } + } + return strings.Repeat("`", max(3, longest+1)) +} + +func chatShellBytes(n int64) string { + switch { + case n >= 1<<20: + return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20)) + case n >= 1<<10: + return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10)) + } + return fmt.Sprintf("%d bytes", n) +} diff --git a/internal/server/chat_shell_supervise.go b/internal/server/chat_shell_supervise.go new file mode 100644 index 00000000..efab60e4 --- /dev/null +++ b/internal/server/chat_shell_supervise.go @@ -0,0 +1,132 @@ +package server + +import ( + "context" + "errors" + "os" + "os/exec" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// supervise reads one command's merged output, streams it in batches, saves +// it periodically and records how the command ended. It owns the reader. +func (r *chatShellRunner) supervise(p *chatShellProcess, reader *os.File) { + defer r.wg.Done() + defer close(p.done) + readDone := make(chan struct{}) + go func() { + defer close(readDone) + buffer := make([]byte, 32<<10) + for { + n, err := reader.Read(buffer) + if n > 0 { + p.mu.Lock() + p.out.write(buffer[:n]) + p.mu.Unlock() + } + if err != nil { + return + } + } + }() + waitDone := make(chan error, 1) + go func() { waitDone <- p.cmd.Wait() }() + pid := p.cmd.Process.Pid + flush := time.NewTicker(r.flush) + defer flush.Stop() + lastSaved := r.now() + var waitErr error + exited, drained, shutdown := false, false, false + base := r.base.Done() + for !exited || !drained { + select { + case <-flush.C: + r.emitOutput(p, false) + if r.now().Sub(lastSaved) >= r.persist { + lastSaved = r.now() + r.save(p.snapshot(protocol.ChatShellRunRetained)) + } + case waitErr = <-waitDone: + exited = true + // Descendants may hold the pipe open; read what they write + // briefly, then stop reading. + _ = reader.SetReadDeadline(time.Now().Add(chatShellDrain)) + case <-readDone: + drained = true + readDone = nil + case <-base: + base = nil + shutdown = true + _ = syscall.Kill(-pid, syscall.SIGKILL) + } + } + // Nothing in the command's group outlives it. + _ = syscall.Kill(-pid, syscall.SIGKILL) + reader.Close() + r.emitOutput(p, true) + p.mu.Lock() + finishChatShellRecord(&p.record, waitErr, p.stopped, shutdown || r.base.Err() != nil, r.now()) + p.mu.Unlock() + record := p.snapshot(protocol.ChatShellRunRetained) + r.save(record) + r.mu.Lock() + delete(r.active, p.record.ID) + r.mu.Unlock() + r.publishRun(record) +} + +// finishChatShellRecord records how a command ended. A requested stop wins +// over the exit status it caused; a server shutdown marks it interrupted. +func finishChatShellRecord(record *protocol.ChatShellRunRecord, waitErr error, stopped, shutdown bool, now time.Time) { + record.EndedAt = now.UnixMilli() + record.DurationMS = max(record.EndedAt-record.StartedAt, 0) + var exitErr *exec.ExitError + switch { + case waitErr == nil: + code := 0 + record.ExitCode = &code + case errors.As(waitErr, &exitErr): + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() { + record.Signal = status.Signal().String() + } else { + code := exitErr.ExitCode() + record.ExitCode = &code + } + default: + record.Error = waitErr.Error() + } + switch { + case shutdown: + record.Status = protocol.ChatShellRunInterrupted + record.Error = chatShellInterruptedReason + case stopped: + record.Status = protocol.ChatShellRunStopped + case record.Error != "": + record.Status = protocol.ChatShellRunFailed + default: + record.Status = protocol.ChatShellRunExited + } +} + +// emitOutput pushes the output written since the last frame. +func (r *chatShellRunner) emitOutput(p *chatShellProcess, final bool) { + p.mu.Lock() + text, offset, end, ok := p.out.pending(protocol.ChatShellRunChunkLimit, final) + sessionID, runID := p.record.SessionID, p.record.ID + p.mu.Unlock() + if !ok || r.publish == nil { + return + } + r.publish(sessionID, protocol.ChatShellRunEventFrame{SessionID: sessionID, RunID: runID, Output: text, Offset: offset, End: end}) +} + +// save persists a record; a shutdown's base context is already canceled, so +// it uses its own bounded context. +func (r *chatShellRunner) save(record protocol.ChatShellRunRecord) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = r.store.SaveChatShellRun(ctx, record, protocol.ChatShellRunHistory) +} diff --git a/internal/server/chat_stream.go b/internal/server/chat_stream.go index 223f752a..ef0b8967 100644 --- a/internal/server/chat_stream.go +++ b/internal/server/chat_stream.go @@ -22,6 +22,13 @@ const chatPermissionPollInterval = 2 * time.Second // every other client, attributed to the sending device's label, and the turn's // output is fanned out to all of them. func handleChatPrompt(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error) { + admitChatPrompt(ctx, request, broker, hub, registry, clientID, write, false) +} + +// admitChatPrompt is handleChatPrompt; echoSender also publishes the user turn +// to the sending client, for a prompt the server composed (chat_shell_share) +// that the sender has no local copy of. +func admitChatPrompt(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error, echoSender bool) { if broker == nil { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat is not configured on this server"}) return @@ -128,7 +135,11 @@ func handleChatPrompt(ctx context.Context, request protocol.Message, broker chat accepted = true queued := admission == chatAdmitQueued // Publish the user turn to every other client; the sender already has it. - hub.broadcastUser(userFrame, clientID) + exceptClientID := clientID + if echoSender { + exceptClientID = "" + } + hub.broadcastUser(userFrame, exceptClientID) if admission == chatAdmitSteered { hub.markOpSteered(sessionID, operationID) // The harness took the message into its running turn: it is not queued, diff --git a/internal/server/websocket.go b/internal/server/websocket.go index ce791257..5cace7d0 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -135,6 +135,13 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin hub.queueStore = stores[0] hub.liveStore = stores[0] } + // Operator shell commands persist beside the operator sessions they belong + // to: the execution store, else an injected store-backed registry. + if len(stores) > 0 { + hub.startShellRunner(ctx, stores[0]) + } else if store, ok := registry.(chatShellRunStore); ok { + hub.startShellRunner(ctx, store) + } acks := newAttentionAckHub() var ackStore attentionAckStore var notes *appNotifier