Run operator shell commands in a chat's directory on the server
chat_shell_run starts $SHELL -lc in the chat session's directory in its own process group, streams merged output to every watcher of the session, keeps a 1 MiB tail and persists each run; chat_shell_stop interrupts the group and kills it after a grace period. chat_shell_share queues an ordinary prompt with the command and a bounded output tail.
This commit is contained in:
@@ -0,0 +1,123 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// initChatShellRuns creates the additive table of operator shell commands. A
|
||||
// run belongs to its chat session and is Pipelines' own record: it is never
|
||||
// part of the harness transcript. Output holds only the retained tail.
|
||||
func (s *Store) initChatShellRuns() error {
|
||||
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS chat_shell_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL,
|
||||
command TEXT NOT NULL,
|
||||
directory TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
exit_code INTEGER,
|
||||
signal TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
started_at INTEGER NOT NULL,
|
||||
ended_at INTEGER NOT NULL DEFAULT 0,
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0,
|
||||
output TEXT NOT NULL DEFAULT '',
|
||||
output_start INTEGER NOT NULL DEFAULT 0,
|
||||
output_bytes INTEGER NOT NULL DEFAULT 0);
|
||||
CREATE INDEX IF NOT EXISTS chat_shell_runs_session ON chat_shell_runs(session_id, started_at);`)
|
||||
return err
|
||||
}
|
||||
|
||||
// SaveChatShellRun inserts or replaces one run. Inserting a new run prunes the
|
||||
// session's oldest finished runs beyond keep (keep <= 0 keeps everything).
|
||||
func (s *Store) SaveChatShellRun(ctx context.Context, run protocol.ChatShellRunRecord, keep int) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var exit any
|
||||
if run.ExitCode != nil {
|
||||
exit = *run.ExitCode
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO chat_shell_runs
|
||||
(id, session_id, command, directory, status, exit_code, signal, error, started_at, ended_at, duration_ms, output, output_start, output_bytes)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET status = excluded.status, exit_code = excluded.exit_code, signal = excluded.signal,
|
||||
error = excluded.error, ended_at = excluded.ended_at, duration_ms = excluded.duration_ms, output = excluded.output,
|
||||
output_start = excluded.output_start, output_bytes = excluded.output_bytes`,
|
||||
run.ID, run.SessionID, run.Command, run.Directory, run.Status, exit, run.Signal, run.Error,
|
||||
run.StartedAt, run.EndedAt, run.DurationMS, run.Output, run.OutputStart, run.OutputBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
if keep > 0 {
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM chat_shell_runs WHERE session_id = ? AND status != ? AND id NOT IN
|
||||
(SELECT id FROM chat_shell_runs WHERE session_id = ? ORDER BY started_at DESC, id DESC LIMIT ?)`,
|
||||
run.SessionID, protocol.ChatShellRunRunning, run.SessionID, keep); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ChatShellRuns lists one session's runs, oldest first.
|
||||
func (s *Store) ChatShellRuns(ctx context.Context, sessionID string) ([]protocol.ChatShellRunRecord, error) {
|
||||
rows, err := s.db.QueryContext(ctx, chatShellRunSelect+` WHERE session_id = ? ORDER BY started_at, id`, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var runs []protocol.ChatShellRunRecord
|
||||
for rows.Next() {
|
||||
run, err := scanChatShellRun(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
runs = append(runs, run)
|
||||
}
|
||||
return runs, rows.Err()
|
||||
}
|
||||
|
||||
// ChatShellRun returns one run of sessionID; found is false when that session
|
||||
// has no run with this id.
|
||||
func (s *Store) ChatShellRun(ctx context.Context, sessionID, id string) (protocol.ChatShellRunRecord, bool, error) {
|
||||
run, err := scanChatShellRun(s.db.QueryRowContext(ctx, chatShellRunSelect+` WHERE session_id = ? AND id = ?`, sessionID, id))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return protocol.ChatShellRunRecord{}, false, nil
|
||||
}
|
||||
return run, err == nil, err
|
||||
}
|
||||
|
||||
// InterruptChatShellRuns marks every run still recorded as running as
|
||||
// interrupted at endedAt. A starting server calls it: no process of a previous
|
||||
// server is still owned by anyone.
|
||||
func (s *Store) InterruptChatShellRuns(ctx context.Context, endedAt int64, reason string) (int64, error) {
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE chat_shell_runs SET status = ?, error = ?, ended_at = ?,
|
||||
duration_ms = CASE WHEN ? > started_at THEN ? - started_at ELSE 0 END WHERE status = ?`,
|
||||
protocol.ChatShellRunInterrupted, reason, endedAt, endedAt, endedAt, protocol.ChatShellRunRunning)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
const chatShellRunSelect = `SELECT id, session_id, command, directory, status, exit_code, signal, error, started_at, ended_at, duration_ms, output, output_start, output_bytes FROM chat_shell_runs`
|
||||
|
||||
func scanChatShellRun(row chatQueueScanner) (protocol.ChatShellRunRecord, error) {
|
||||
var run protocol.ChatShellRunRecord
|
||||
var exit sql.NullInt64
|
||||
err := row.Scan(&run.ID, &run.SessionID, &run.Command, &run.Directory, &run.Status, &exit, &run.Signal, &run.Error,
|
||||
&run.StartedAt, &run.EndedAt, &run.DurationMS, &run.Output, &run.OutputStart, &run.OutputBytes)
|
||||
if err != nil {
|
||||
return protocol.ChatShellRunRecord{}, err
|
||||
}
|
||||
if exit.Valid {
|
||||
code := int(exit.Int64)
|
||||
run.ExitCode = &code
|
||||
}
|
||||
run.Truncated = run.OutputStart > 0
|
||||
return run, nil
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestChatShellRunsPersistPruneAndInterrupt(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "shell.db")
|
||||
store, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
running := protocol.ChatShellRunRecord{ID: "run_live", SessionID: "ses_1", Command: "sleep 9", Directory: "/w", Status: protocol.ChatShellRunRunning, StartedAt: 1000}
|
||||
if err := store.SaveChatShellRun(ctx, running, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code := 3
|
||||
for i := 0; i < 3; i++ {
|
||||
run := protocol.ChatShellRunRecord{ID: fmt.Sprintf("run_%d", i), SessionID: "ses_1", Command: "false", Directory: "/w", Status: protocol.ChatShellRunExited, ExitCode: &code, StartedAt: int64(2000 + i), EndedAt: int64(2500 + i), Output: "tail", OutputStart: 10, OutputBytes: 14}
|
||||
if err := store.SaveChatShellRun(ctx, run, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := store.SaveChatShellRun(ctx, protocol.ChatShellRunRecord{ID: "other", SessionID: "ses_2", Command: "ls", Directory: "/w", Status: protocol.ChatShellRunExited, StartedAt: 1}, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runs, err := store.ChatShellRuns(ctx, "ses_1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The running run is never pruned; keep counts the newest runs, so only
|
||||
// the finished ones outside the newest two are dropped.
|
||||
ids := []string{}
|
||||
for _, run := range runs {
|
||||
ids = append(ids, run.ID)
|
||||
}
|
||||
if fmt.Sprint(ids) != "[run_live run_1 run_2]" {
|
||||
t.Fatalf("runs = %v", ids)
|
||||
}
|
||||
if got := runs[2]; got.ExitCode == nil || *got.ExitCode != 3 || !got.Truncated || got.Output != "tail" || got.OutputBytes != 14 {
|
||||
t.Fatalf("finished run = %+v", got)
|
||||
}
|
||||
store.Close()
|
||||
store, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
n, err := store.InterruptChatShellRuns(ctx, 4000, "server restarted")
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("interrupt = %d, %v", n, err)
|
||||
}
|
||||
live, found, err := store.ChatShellRun(ctx, "ses_1", "run_live")
|
||||
if err != nil || !found || live.Status != protocol.ChatShellRunInterrupted || live.EndedAt != 4000 || live.DurationMS != 3000 || live.ExitCode != nil {
|
||||
t.Fatalf("interrupted run = %+v found=%v err=%v", live, found, err)
|
||||
}
|
||||
if _, found, _ := store.ChatShellRun(ctx, "ses_2", "run_live"); found {
|
||||
t.Fatal("a run was readable through another session")
|
||||
}
|
||||
}
|
||||
@@ -101,6 +101,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initChatShellRuns(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initAppNotifications(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package protocol
|
||||
|
||||
// Operator shell commands: commands the operator runs themselves inside one
|
||||
// chat's working directory (the chat's `!cmd`). They are owned by Pipelines,
|
||||
// not by the chat harness: the model never sees a command or its output unless
|
||||
// the operator explicitly shares it (chat_shell_share), which queues an
|
||||
// ordinary prompt.
|
||||
const (
|
||||
// ChatShellRun starts one command. The body is ChatShellRunRequest
|
||||
// (session_id, command); the reply carries the new run in ChatShellRun.
|
||||
ChatShellRun = "chat_shell_run"
|
||||
// ChatShellStop interrupts one running command (SIGINT to its process
|
||||
// group, then SIGKILL after a grace period). The body is
|
||||
// ChatShellRunRequest (session_id, run_id); the reply echoes the run.
|
||||
ChatShellStop = "chat_shell_stop"
|
||||
// ChatShellRuns lists the session's recorded runs, oldest first, each with
|
||||
// at most ChatShellRunListOutput bytes of its retained output tail. The
|
||||
// body is ChatQuery.
|
||||
ChatShellRuns = "chat_shell_runs"
|
||||
// ChatShellRunOutput reads one run with its whole retained output (at most
|
||||
// ChatShellRunRetained bytes). The body is ChatShellRunRequest.
|
||||
ChatShellRunOutput = "chat_shell_run_output"
|
||||
// ChatShellShare queues an ordinary prompt carrying the run's command, exit
|
||||
// status and bounded output tail. The body is ChatShellRunRequest; the
|
||||
// reply is the chat_prompt acknowledgement.
|
||||
ChatShellShare = "chat_shell_share"
|
||||
// ChatShellRunEvent is pushed to every client watching the session: a
|
||||
// lifecycle change (Run set) or an output chunk (Output set).
|
||||
ChatShellRunEvent = "chat_shell_run_event"
|
||||
)
|
||||
|
||||
// ChatShellRunRecord status values. Running is the only non-final status.
|
||||
const (
|
||||
ChatShellRunRunning = "running"
|
||||
ChatShellRunExited = "exited"
|
||||
ChatShellRunStopped = "stopped"
|
||||
ChatShellRunFailed = "failed"
|
||||
ChatShellRunInterrupted = "interrupted"
|
||||
)
|
||||
|
||||
// Bounds of operator shell commands.
|
||||
const (
|
||||
// ChatShellRunCommandLimit bounds one command line.
|
||||
ChatShellRunCommandLimit = 16 << 10
|
||||
// ChatShellRunRetained is the retained (and persisted) output tail of one
|
||||
// run: older output is dropped and the record says so.
|
||||
ChatShellRunRetained = 1 << 20
|
||||
// ChatShellRunListOutput bounds the output tail each run carries in a
|
||||
// chat_shell_runs list, so a long history stays one small frame.
|
||||
ChatShellRunListOutput = 64 << 10
|
||||
// ChatShellRunChunkLimit bounds one live output frame.
|
||||
ChatShellRunChunkLimit = 64 << 10
|
||||
// ChatShellRunShareOutput bounds the output tail shared with the agent.
|
||||
ChatShellRunShareOutput = 16 << 10
|
||||
// ChatShellRunConcurrent bounds the commands one session runs at once.
|
||||
ChatShellRunConcurrent = 4
|
||||
// ChatShellRunHistory bounds the finished runs kept per session.
|
||||
ChatShellRunHistory = 50
|
||||
)
|
||||
|
||||
// ChatShellRunRequest names one session and either a new command or a run.
|
||||
// OperationID is the optional retry identity of chat_shell_share's prompt.
|
||||
type ChatShellRunRequest struct {
|
||||
SessionID string `json:"session_id"`
|
||||
Command string `json:"command,omitempty"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
OperationID string `json:"operation_id,omitempty"`
|
||||
}
|
||||
|
||||
// ChatShellRunRecord is one operator command and what it produced. Output is
|
||||
// the retained text starting at absolute byte OutputStart of everything the
|
||||
// command wrote (stdout and stderr merged); OutputBytes is that total. A record
|
||||
// with OutputStart > 0 dropped its beginning. Times are epoch milliseconds.
|
||||
// ExitCode is set once the process exited with a code; a signal-terminated
|
||||
// or never-started command has none.
|
||||
type ChatShellRunRecord struct {
|
||||
ID string `json:"id"`
|
||||
SessionID string `json:"session_id"`
|
||||
Command string `json:"command"`
|
||||
Directory string `json:"directory"`
|
||||
Status string `json:"status"`
|
||||
ExitCode *int `json:"exit_code,omitempty"`
|
||||
Signal string `json:"signal,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
StartedAt int64 `json:"started_at"`
|
||||
EndedAt int64 `json:"ended_at,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
OutputStart int64 `json:"output_start,omitempty"`
|
||||
OutputBytes int64 `json:"output_bytes,omitempty"`
|
||||
// Truncated is true when Output does not hold everything the command
|
||||
// wrote (OutputStart > 0).
|
||||
Truncated bool `json:"truncated,omitempty"`
|
||||
}
|
||||
|
||||
// ChatShellRunEventFrame is one live change to a run. A lifecycle frame sets
|
||||
// Run (without output); an output frame sets Output covering absolute bytes
|
||||
// [Offset, End). A client whose own copy ends before Offset missed output and
|
||||
// shows that gap rather than joining unrelated text.
|
||||
type ChatShellRunEventFrame struct {
|
||||
SessionID string `json:"session_id"`
|
||||
RunID string `json:"run_id"`
|
||||
Run *ChatShellRunRecord `json:"run,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
Offset int64 `json:"offset,omitempty"`
|
||||
End int64 `json:"end,omitempty"`
|
||||
}
|
||||
@@ -262,6 +262,12 @@ type Message struct {
|
||||
ChatShellOutputQuery *ChatShellOutputQuery `json:"chat_shell_output_query,omitempty"`
|
||||
ChatShells []ChatShell `json:"chat_shells,omitempty"`
|
||||
ChatShellOutput *ChatShellOutputPage `json:"chat_shell_output,omitempty"`
|
||||
// Operator shell commands (chat_shell_run.go): the request body, one run,
|
||||
// a session's runs and the pushed live change.
|
||||
ChatShellRunRequest *ChatShellRunRequest `json:"chat_shell_run_request,omitempty"`
|
||||
ChatShellRun *ChatShellRunRecord `json:"chat_shell_run,omitempty"`
|
||||
ChatShellRuns []ChatShellRunRecord `json:"chat_shell_runs,omitempty"`
|
||||
ChatShellRunEvent *ChatShellRunEventFrame `json:"chat_shell_run_event,omitempty"`
|
||||
// ListDirectory is the list_directory body and DirectoryListing its reply:
|
||||
// the home-restricted folder picker listing (see directories.go).
|
||||
ListDirectory *DirectoryListRequest `json:"list_directory,omitempty"`
|
||||
|
||||
@@ -255,6 +255,9 @@ func handleChat(ctx context.Context, request protocol.Message, broker chatBroker
|
||||
case protocol.ChatShellOutput:
|
||||
handleChatShellOutput(ctx, request, broker, registry, write)
|
||||
return true
|
||||
case protocol.ChatShellRun, protocol.ChatShellStop, protocol.ChatShellRuns, protocol.ChatShellRunOutput, protocol.ChatShellShare:
|
||||
handleChatShellRunRequest(ctx, request, broker, hub, registry, clientID, write)
|
||||
return true
|
||||
case protocol.ChatAttachmentImage:
|
||||
handleChatAttachmentImage(request, broker, write)
|
||||
return true
|
||||
|
||||
@@ -74,6 +74,9 @@ type chatHub struct {
|
||||
// and its sequence source (chat_activity.go).
|
||||
activity map[string]protocol.ChatActivity
|
||||
activitySeq int64
|
||||
// shellRuns runs the operator's own shell commands (chat_shell_runner.go);
|
||||
// nil refuses them.
|
||||
shellRuns *chatShellRunner
|
||||
}
|
||||
|
||||
// chatPart is the retained form of one part. Text and reasoning deltas append to
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// handleChatShellRunRequest answers the operator shell-command requests. Every
|
||||
// one names an operator chat session the registry records: the command runs
|
||||
// only in that chat's own directory on this server's machine, and a run is
|
||||
// only readable or stoppable through the session that started it.
|
||||
func handleChatShellRunRequest(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error) {
|
||||
fail := func(err error) { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) }
|
||||
runner := hub.shellRunner()
|
||||
if runner == nil {
|
||||
fail(errors.New("shell commands are not configured on this server"))
|
||||
return
|
||||
}
|
||||
var sessionID string
|
||||
body := request.ChatShellRunRequest
|
||||
switch {
|
||||
case request.Type == protocol.ChatShellRuns && request.ChatQuery != nil:
|
||||
sessionID = request.ChatQuery.SessionID
|
||||
case body != nil:
|
||||
sessionID = body.SessionID
|
||||
}
|
||||
if !validChatSessionID(sessionID) {
|
||||
fail(errors.New("a valid chat session id is required"))
|
||||
return
|
||||
}
|
||||
if registry == nil {
|
||||
fail(errors.New("operator session registry is not configured on this server"))
|
||||
return
|
||||
}
|
||||
session, found, err := registry.OperatorSession(ctx, sessionID)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
fail(errors.New("operator session is not available"))
|
||||
return
|
||||
}
|
||||
if request.Type != protocol.ChatShellRuns && body == nil {
|
||||
fail(errors.New("a chat shell request body is required"))
|
||||
return
|
||||
}
|
||||
switch request.Type {
|
||||
case protocol.ChatShellRun:
|
||||
record, err := runner.Start(ctx, sessionID, session.Directory, body.Command)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
_ = write(protocol.Message{Type: protocol.ChatShellRun, ID: request.ID, ChatShellRun: &record})
|
||||
case protocol.ChatShellStop:
|
||||
record, err := runner.Stop(sessionID, body.RunID)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
_ = write(protocol.Message{Type: protocol.ChatShellStop, ID: request.ID, ChatShellRun: &record})
|
||||
case protocol.ChatShellRuns:
|
||||
runs, err := runner.List(ctx, sessionID)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
_ = write(protocol.Message{Type: protocol.ChatShellRuns, ID: request.ID, ChatShellRuns: runs})
|
||||
case protocol.ChatShellRunOutput:
|
||||
record, err := runner.Read(ctx, sessionID, body.RunID)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
_ = write(protocol.Message{Type: protocol.ChatShellRunOutput, ID: request.ID, ChatShellRun: &record})
|
||||
case protocol.ChatShellShare:
|
||||
record, err := runner.Read(ctx, sessionID, body.RunID)
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
if record.Status == protocol.ChatShellRunRunning {
|
||||
fail(errors.New("wait for the command to finish before sending it to the agent"))
|
||||
return
|
||||
}
|
||||
prompt := protocol.Message{Type: protocol.ChatPrompt, ID: request.ID, ChatPrompt: &protocol.ChatPromptRequest{
|
||||
SessionID: sessionID, Text: chatShellSharePrompt(record), OperationID: body.OperationID,
|
||||
}}
|
||||
admitChatPrompt(ctx, prompt, broker, hub, registry, clientID, write, true)
|
||||
}
|
||||
}
|
||||
|
||||
// List is the session's recorded runs, oldest first. A running run carries
|
||||
// its live output rather than the last periodic save; every run carries at
|
||||
// most protocol.ChatShellRunListOutput bytes of its tail.
|
||||
func (r *chatShellRunner) List(ctx context.Context, sessionID string) ([]protocol.ChatShellRunRecord, error) {
|
||||
runs, err := r.store.ChatShellRuns(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, run := range runs {
|
||||
if p := r.activeProcess(sessionID, run.ID); p != nil {
|
||||
runs[i] = p.snapshot(protocol.ChatShellRunListOutput)
|
||||
continue
|
||||
}
|
||||
runs[i] = chatShellRecordTail(run, protocol.ChatShellRunListOutput)
|
||||
}
|
||||
return runs, nil
|
||||
}
|
||||
|
||||
// Read is one run of sessionID with its whole retained output.
|
||||
func (r *chatShellRunner) Read(ctx context.Context, sessionID, runID string) (protocol.ChatShellRunRecord, error) {
|
||||
if p := r.activeProcess(sessionID, runID); p != nil {
|
||||
return p.snapshot(protocol.ChatShellRunRetained), nil
|
||||
}
|
||||
record, found, err := r.store.ChatShellRun(ctx, sessionID, runID)
|
||||
if err != nil {
|
||||
return protocol.ChatShellRunRecord{}, err
|
||||
}
|
||||
if !found {
|
||||
return protocol.ChatShellRunRecord{}, errors.New("this chat has no such command")
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// chatShellRecordTail trims a stored record's output to its last max bytes.
|
||||
func chatShellRecordTail(record protocol.ChatShellRunRecord, max int) protocol.ChatShellRunRecord {
|
||||
if len(record.Output) <= max {
|
||||
return record
|
||||
}
|
||||
out := newChatShellOutput(len(record.Output))
|
||||
out.write([]byte(record.Output))
|
||||
text, from := out.tail(max)
|
||||
record.Output = text
|
||||
record.OutputStart += from
|
||||
record.Truncated = true
|
||||
return record
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// chatShellOutput is the bounded tail of one operator command's merged output.
|
||||
// Offsets are absolute bytes of everything the command wrote, so a client can
|
||||
// tell a continuation from a gap. The buffer may grow to twice its limit before
|
||||
// compacting, which keeps a flooding command from copying the tail per write.
|
||||
type chatShellOutput struct {
|
||||
limit int
|
||||
data []byte
|
||||
start int64
|
||||
total int64
|
||||
emitted int64
|
||||
}
|
||||
|
||||
func newChatShellOutput(limit int) *chatShellOutput {
|
||||
return &chatShellOutput{limit: limit}
|
||||
}
|
||||
|
||||
func (o *chatShellOutput) write(p []byte) {
|
||||
o.data = append(o.data, p...)
|
||||
o.total += int64(len(p))
|
||||
if len(o.data) <= 2*o.limit {
|
||||
return
|
||||
}
|
||||
cut := runeStartAfter(o.data, len(o.data)-o.limit)
|
||||
o.data = append([]byte(nil), o.data[cut:]...)
|
||||
o.start += int64(cut)
|
||||
}
|
||||
|
||||
// tail returns at most max retained bytes ending at the newest byte, starting
|
||||
// on a character boundary, with the absolute offset of its first byte.
|
||||
func (o *chatShellOutput) tail(max int) (string, int64) {
|
||||
from := 0
|
||||
if len(o.data) > max {
|
||||
from = runeStartAfter(o.data, len(o.data)-max)
|
||||
}
|
||||
return validChatShellText(o.data[from:]), o.start + int64(from)
|
||||
}
|
||||
|
||||
// pending returns the output not yet emitted, at most max bytes (the newest
|
||||
// ones when more is waiting: the skipped bytes are a visible gap to clients).
|
||||
// An incomplete trailing character is held back unless final is set.
|
||||
func (o *chatShellOutput) pending(max int, final bool) (text string, offset, end int64, ok bool) {
|
||||
from := o.emitted
|
||||
if from < o.start {
|
||||
from = o.start
|
||||
}
|
||||
available := o.data[from-o.start:]
|
||||
if !final {
|
||||
available = available[:completeUTF8Prefix(available)]
|
||||
}
|
||||
if len(available) == 0 {
|
||||
return "", 0, 0, false
|
||||
}
|
||||
end = from + int64(len(available))
|
||||
if len(available) > max {
|
||||
skip := runeStartAfter(available, len(available)-max)
|
||||
available = available[skip:]
|
||||
from += int64(skip)
|
||||
}
|
||||
o.emitted = end
|
||||
return validChatShellText(available), from, end, true
|
||||
}
|
||||
|
||||
// runeStartAfter moves index forward (at most three bytes) to the first byte
|
||||
// that starts a character, so a cut never splits one.
|
||||
func runeStartAfter(data []byte, index int) int {
|
||||
for step := 0; step < utf8.UTFMax-1 && index < len(data) && !utf8.RuneStart(data[index]); step++ {
|
||||
index++
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
// completeUTF8Prefix is the length of data without a trailing partial
|
||||
// character that a later read may complete.
|
||||
func completeUTF8Prefix(data []byte) int {
|
||||
for back := 1; back <= utf8.UTFMax-1 && back <= len(data); back++ {
|
||||
if utf8.RuneStart(data[len(data)-back]) {
|
||||
if !utf8.FullRune(data[len(data)-back:]) {
|
||||
return len(data) - back
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return len(data)
|
||||
}
|
||||
|
||||
// validChatShellText keeps binary output representable in a JSON frame.
|
||||
func validChatShellText(data []byte) string {
|
||||
return strings.ToValidUTF8(string(data), "�")
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// shellChat is one server with a recorded operator session whose directory is
|
||||
// a fresh temporary folder, and two connections watching that session.
|
||||
func shellChat(t *testing.T) (context.Context, []*websocket.Conn, string) {
|
||||
t.Helper()
|
||||
store := newOperatorStore(t)
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
ctx, _, conns := dialChatConns(t, Options{Chat: broker, OperatorRegistry: store}, 2)
|
||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.RecordOperatorSession(ctx, "ses_1", dir, "Shell", protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, conn := range conns {
|
||||
writeChat(t, ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}})
|
||||
if got := readShellFrame(t, ctx, conn, func(m protocol.Message) bool { return m.ID == "sub" }); got.Type != protocol.ChatSubscribe {
|
||||
t.Fatalf("subscribe %d = %+v", i, got)
|
||||
}
|
||||
}
|
||||
return ctx, conns, dir
|
||||
}
|
||||
|
||||
func readShellFrame(t *testing.T, ctx context.Context, conn *websocket.Conn, match func(protocol.Message) bool) protocol.Message {
|
||||
t.Helper()
|
||||
for {
|
||||
var message protocol.Message
|
||||
if err := readTestFrame(ctx, conn, &message); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if match(message) {
|
||||
return message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func shellRequest(kind, id, runID, command string) protocol.Message {
|
||||
return protocol.Message{Type: kind, ID: id, ChatShellRunRequest: &protocol.ChatShellRunRequest{SessionID: "ses_1", Command: command, RunID: runID}}
|
||||
}
|
||||
|
||||
// runShell starts command and returns its running record.
|
||||
func runShell(t *testing.T, ctx context.Context, conn *websocket.Conn, id, command string) protocol.ChatShellRunRecord {
|
||||
t.Helper()
|
||||
writeChat(t, ctx, conn, shellRequest(protocol.ChatShellRun, id, "", command))
|
||||
reply := readShellFrame(t, ctx, conn, func(m protocol.Message) bool { return m.ID == id })
|
||||
if reply.Type != protocol.ChatShellRun || reply.ChatShellRun == nil || reply.ChatShellRun.Status != protocol.ChatShellRunRunning {
|
||||
t.Fatalf("chat_shell_run reply = %+v", reply)
|
||||
}
|
||||
return *reply.ChatShellRun
|
||||
}
|
||||
|
||||
// finishedShell reads conn's frames until runID's final lifecycle frame and
|
||||
// returns it with the live output joined from its output frames.
|
||||
func finishedShell(t *testing.T, ctx context.Context, conn *websocket.Conn, runID string) (protocol.ChatShellRunRecord, string) {
|
||||
t.Helper()
|
||||
var output strings.Builder
|
||||
var end int64
|
||||
for {
|
||||
frame := readShellFrame(t, ctx, conn, func(m protocol.Message) bool {
|
||||
return m.Type == protocol.ChatShellRunEvent && m.ChatShellRunEvent != nil && m.ChatShellRunEvent.RunID == runID
|
||||
}).ChatShellRunEvent
|
||||
if frame.Run == nil {
|
||||
if frame.Offset != end {
|
||||
t.Fatalf("output frame at %d after %d", frame.Offset, end)
|
||||
}
|
||||
output.WriteString(frame.Output)
|
||||
end = frame.End
|
||||
continue
|
||||
}
|
||||
if frame.Run.Status != protocol.ChatShellRunRunning {
|
||||
return *frame.Run, output.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellRunStreamsToEveryWatcherAndPersists(t *testing.T) {
|
||||
ctx, conns, dir := shellChat(t)
|
||||
run := runShell(t, ctx, conns[0], "r1", `pwd; echo out; echo err >&2; exit 3`)
|
||||
if run.Directory != dir || run.Command == "" || run.ID == "" {
|
||||
t.Fatalf("running record = %+v", run)
|
||||
}
|
||||
for _, conn := range conns {
|
||||
final, output := finishedShell(t, ctx, conn, run.ID)
|
||||
if final.Status != protocol.ChatShellRunExited || final.ExitCode == nil || *final.ExitCode != 3 || final.EndedAt < final.StartedAt {
|
||||
t.Fatalf("final record = %+v", final)
|
||||
}
|
||||
if output != dir+"\nout\nerr\n" {
|
||||
t.Fatalf("streamed output = %q", output)
|
||||
}
|
||||
}
|
||||
writeChat(t, ctx, conns[1], protocol.Message{Type: protocol.ChatShellRuns, ID: "list", ChatQuery: &protocol.ChatQuery{SessionID: "ses_1"}})
|
||||
list := readShellFrame(t, ctx, conns[1], func(m protocol.Message) bool { return m.ID == "list" })
|
||||
if len(list.ChatShellRuns) != 1 || list.ChatShellRuns[0].Output != dir+"\nout\nerr\n" || *list.ChatShellRuns[0].ExitCode != 3 {
|
||||
t.Fatalf("persisted runs = %+v", list)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellStopInterruptsTheGroup(t *testing.T) {
|
||||
ctx, conns, _ := shellChat(t)
|
||||
// The child sleep is in the shell's group, so the stop reaches it too.
|
||||
run := runShell(t, ctx, conns[0], "r1", `echo started; sleep 30; echo never`)
|
||||
writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellStop, "stop", run.ID, ""))
|
||||
if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "stop" }); reply.Type != protocol.ChatShellStop {
|
||||
t.Fatalf("stop reply = %+v", reply)
|
||||
}
|
||||
final, output := finishedShell(t, ctx, conns[1], run.ID)
|
||||
if final.Status != protocol.ChatShellRunStopped || strings.Contains(output, "never") || final.DurationMS > 10_000 {
|
||||
t.Fatalf("stopped record = %+v output=%q", final, output)
|
||||
}
|
||||
writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellStop, "again", run.ID, ""))
|
||||
if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "again" }); reply.Type != protocol.Error {
|
||||
t.Fatalf("stopping a finished run = %+v", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellRunLimitsAndRefusals(t *testing.T) {
|
||||
ctx, conns, dir := shellChat(t)
|
||||
var runs []protocol.ChatShellRunRecord
|
||||
for i := 0; i < protocol.ChatShellRunConcurrent; i++ {
|
||||
runs = append(runs, runShell(t, ctx, conns[0], "r"+string(rune('a'+i)), "sleep 30"))
|
||||
}
|
||||
refusals := []protocol.Message{
|
||||
shellRequest(protocol.ChatShellRun, "fifth", "", "sleep 30"),
|
||||
shellRequest(protocol.ChatShellRun, "blank", "", " "),
|
||||
shellRequest(protocol.ChatShellRun, "huge", "", strings.Repeat("x", protocol.ChatShellRunCommandLimit+1)),
|
||||
{Type: protocol.ChatShellRun, ID: "nobody", ChatShellRunRequest: &protocol.ChatShellRunRequest{SessionID: "ses_unknown", Command: "true"}},
|
||||
{Type: protocol.ChatShellRun, ID: "badid", ChatShellRunRequest: &protocol.ChatShellRunRequest{SessionID: "../ses", Command: "true"}},
|
||||
{Type: protocol.ChatShellRun, ID: "nobody-body"},
|
||||
{Type: protocol.ChatShellStop, ID: "foreign", ChatShellRunRequest: &protocol.ChatShellRunRequest{SessionID: "ses_unknown", RunID: runs[0].ID}},
|
||||
shellRequest(protocol.ChatShellShare, "share-running", runs[0].ID, ""),
|
||||
}
|
||||
for _, request := range refusals {
|
||||
writeChat(t, ctx, conns[0], request)
|
||||
if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == request.ID }); reply.Type != protocol.Error {
|
||||
t.Fatalf("%s was not refused: %+v", request.ID, reply)
|
||||
}
|
||||
}
|
||||
for _, run := range runs {
|
||||
writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellStop, "stop-"+run.ID, run.ID, ""))
|
||||
readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "stop-"+run.ID })
|
||||
}
|
||||
for range runs {
|
||||
readShellFrame(t, ctx, conns[1], func(m protocol.Message) bool {
|
||||
return m.Type == protocol.ChatShellRunEvent && m.ChatShellRunEvent.Run != nil && m.ChatShellRunEvent.Run.Status == protocol.ChatShellRunStopped
|
||||
})
|
||||
}
|
||||
// A deleted working directory is refused rather than run somewhere else.
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellRun, "gone", "", "true"))
|
||||
if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "gone" }); reply.Type != protocol.Error || !strings.Contains(reply.Error, "not available") {
|
||||
t.Fatalf("missing directory = %+v", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellShareQueuesAnOrdinaryPrompt(t *testing.T) {
|
||||
ctx, conns, dir := shellChat(t)
|
||||
run := runShell(t, ctx, conns[0], "r1", "echo shared-output")
|
||||
finishedShell(t, ctx, conns[0], run.ID)
|
||||
writeChat(t, ctx, conns[0], shellRequest(protocol.ChatShellShare, "share", run.ID, ""))
|
||||
// The composed prompt is echoed to the sender too: it has no local copy.
|
||||
for _, conn := range conns {
|
||||
user := readShellFrame(t, ctx, conn, func(m protocol.Message) bool {
|
||||
return m.Type == protocol.ChatEvent && m.ChatEvent != nil && m.ChatEvent.Kind == protocol.ChatEventUser
|
||||
}).ChatEvent
|
||||
text := user.Part.Text
|
||||
if !strings.Contains(text, "$ echo shared-output") || !strings.Contains(text, "shared-output\n```") || !strings.Contains(text, dir) || !strings.Contains(text, "exited with status 0") {
|
||||
t.Fatalf("shared prompt = %q", text)
|
||||
}
|
||||
}
|
||||
if reply := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { return m.ID == "share" }); reply.Type != protocol.ChatPrompt {
|
||||
t.Fatalf("share reply = %+v", reply)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// chatShellRunStore persists operator shell runs; *pipeline.Store implements it.
|
||||
type chatShellRunStore interface {
|
||||
SaveChatShellRun(ctx context.Context, run protocol.ChatShellRunRecord, keep int) error
|
||||
ChatShellRuns(ctx context.Context, sessionID string) ([]protocol.ChatShellRunRecord, error)
|
||||
ChatShellRun(ctx context.Context, sessionID, id string) (protocol.ChatShellRunRecord, bool, error)
|
||||
InterruptChatShellRuns(ctx context.Context, endedAt int64, reason string) (int64, error)
|
||||
}
|
||||
|
||||
const (
|
||||
chatShellInterruptedReason = "The server stopped while this command was running."
|
||||
// chatShellDrain is how long output is still read after the command itself
|
||||
// exited, for descendants that keep the pipe open; the group is then killed.
|
||||
chatShellDrain = time.Second
|
||||
)
|
||||
|
||||
// chatShellRunner owns the operator's own commands: each runs `$SHELL -lc` in
|
||||
// its chat's directory, in its own process group, with stdout and stderr
|
||||
// merged into one bounded tail that is streamed to the session's watchers and
|
||||
// persisted as Pipelines' record of the run.
|
||||
type chatShellRunner struct {
|
||||
base context.Context
|
||||
store chatShellRunStore
|
||||
publish func(sessionID string, frame protocol.ChatShellRunEventFrame)
|
||||
now func() time.Time
|
||||
shell func() string
|
||||
// grace separates Stop's SIGINT from its SIGKILL; flush batches live output
|
||||
// frames; persist is the interval a running run's output is saved at.
|
||||
grace time.Duration
|
||||
flush time.Duration
|
||||
persist time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
active map[string]*chatShellProcess
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
type chatShellProcess struct {
|
||||
mu sync.Mutex
|
||||
record protocol.ChatShellRunRecord
|
||||
out *chatShellOutput
|
||||
cmd *exec.Cmd
|
||||
stopped bool
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newChatShellRunner(base context.Context, store chatShellRunStore, publish func(string, protocol.ChatShellRunEventFrame)) *chatShellRunner {
|
||||
return &chatShellRunner{
|
||||
base: base, store: store, publish: publish, now: time.Now, shell: chatShellProgram,
|
||||
grace: 3 * time.Second, flush: 100 * time.Millisecond, persist: 2 * time.Second,
|
||||
active: map[string]*chatShellProcess{},
|
||||
}
|
||||
}
|
||||
|
||||
// chatShellProgram is the operator's login shell, or /bin/sh without one.
|
||||
func chatShellProgram() string {
|
||||
if shell := os.Getenv("SHELL"); filepath.IsAbs(shell) {
|
||||
if info, err := os.Stat(shell); err == nil && !info.IsDir() {
|
||||
return shell
|
||||
}
|
||||
}
|
||||
return "/bin/sh"
|
||||
}
|
||||
|
||||
// recover marks runs a previous server process left running as interrupted:
|
||||
// their processes died with it (or are no longer owned by anyone).
|
||||
func (r *chatShellRunner) recover(ctx context.Context) {
|
||||
if r.store == nil {
|
||||
return
|
||||
}
|
||||
_, _ = r.store.InterruptChatShellRuns(ctx, r.now().UnixMilli(), chatShellInterruptedReason)
|
||||
}
|
||||
|
||||
// resolveChatShellDirectory requires the chat's recorded directory to still be
|
||||
// an existing directory and returns its symlink-free path.
|
||||
func resolveChatShellDirectory(directory string) (string, error) {
|
||||
directory = strings.TrimSpace(directory)
|
||||
if directory == "" || !filepath.IsAbs(directory) {
|
||||
return "", errors.New("this chat has no absolute working directory to run commands in")
|
||||
}
|
||||
real, err := filepath.EvalSymlinks(directory)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the chat's directory %s is not available: %v", directory, err)
|
||||
}
|
||||
info, err := os.Stat(real)
|
||||
if err != nil || !info.IsDir() {
|
||||
return "", fmt.Errorf("the chat's directory %s is not a directory", directory)
|
||||
}
|
||||
return real, nil
|
||||
}
|
||||
|
||||
func newChatShellRunID() (string, error) {
|
||||
var b [12]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "run_" + hex.EncodeToString(b[:]), nil
|
||||
}
|
||||
|
||||
// Start launches command for sessionID in directory and returns its running
|
||||
// record. Validation and start failures are returned without a record.
|
||||
func (r *chatShellRunner) Start(ctx context.Context, sessionID, directory, command string) (protocol.ChatShellRunRecord, error) {
|
||||
if r.store == nil {
|
||||
return protocol.ChatShellRunRecord{}, errors.New("shell commands need the server's store, which is not configured")
|
||||
}
|
||||
if strings.TrimSpace(command) == "" {
|
||||
return protocol.ChatShellRunRecord{}, errors.New("a nonempty command is required")
|
||||
}
|
||||
if len(command) > protocol.ChatShellRunCommandLimit {
|
||||
return protocol.ChatShellRunRecord{}, fmt.Errorf("the command exceeds %d KiB", protocol.ChatShellRunCommandLimit>>10)
|
||||
}
|
||||
dir, err := resolveChatShellDirectory(directory)
|
||||
if err != nil {
|
||||
return protocol.ChatShellRunRecord{}, err
|
||||
}
|
||||
id, err := newChatShellRunID()
|
||||
if err != nil {
|
||||
return protocol.ChatShellRunRecord{}, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.base.Err() != nil {
|
||||
return protocol.ChatShellRunRecord{}, errors.New("the server is stopping")
|
||||
}
|
||||
running := 0
|
||||
for _, p := range r.active {
|
||||
if p.record.SessionID == sessionID {
|
||||
running++
|
||||
}
|
||||
}
|
||||
if running >= protocol.ChatShellRunConcurrent {
|
||||
return protocol.ChatShellRunRecord{}, fmt.Errorf("this chat already runs %d commands; stop one or wait for it to finish", running)
|
||||
}
|
||||
reader, writer, err := os.Pipe()
|
||||
if err != nil {
|
||||
return protocol.ChatShellRunRecord{}, err
|
||||
}
|
||||
cmd := exec.Command(r.shell(), "-lc", command)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = os.Environ()
|
||||
cmd.Stdout, cmd.Stderr = writer, writer
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
if err := cmd.Start(); err != nil {
|
||||
reader.Close()
|
||||
writer.Close()
|
||||
return protocol.ChatShellRunRecord{}, fmt.Errorf("could not start the command: %v", err)
|
||||
}
|
||||
writer.Close()
|
||||
record := protocol.ChatShellRunRecord{ID: id, SessionID: sessionID, Command: command, Directory: dir, Status: protocol.ChatShellRunRunning, StartedAt: r.now().UnixMilli()}
|
||||
p := &chatShellProcess{record: record, out: newChatShellOutput(protocol.ChatShellRunRetained), cmd: cmd, done: make(chan struct{})}
|
||||
if err := r.store.SaveChatShellRun(ctx, record, protocol.ChatShellRunHistory); err != nil {
|
||||
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||
reader.Close()
|
||||
_ = cmd.Wait()
|
||||
return protocol.ChatShellRunRecord{}, fmt.Errorf("could not record the command: %v", err)
|
||||
}
|
||||
r.active[id] = p
|
||||
r.wg.Add(1)
|
||||
go r.supervise(p, reader)
|
||||
r.publishRun(record)
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// Stop interrupts a running command's whole process group, then kills it
|
||||
// after the grace period. It returns the run's current record.
|
||||
func (r *chatShellRunner) Stop(sessionID, runID string) (protocol.ChatShellRunRecord, error) {
|
||||
r.mu.Lock()
|
||||
p := r.active[runID]
|
||||
r.mu.Unlock()
|
||||
if p == nil || p.record.SessionID != sessionID {
|
||||
return protocol.ChatShellRunRecord{}, errors.New("this command is not running")
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.stopped = true
|
||||
record := p.record
|
||||
p.mu.Unlock()
|
||||
pid := p.cmd.Process.Pid
|
||||
_ = syscall.Kill(-pid, syscall.SIGINT)
|
||||
go func() {
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(r.grace):
|
||||
_ = syscall.Kill(-pid, syscall.SIGKILL)
|
||||
}
|
||||
}()
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// Wait blocks until every supervised command has finished (tests, shutdown).
|
||||
func (r *chatShellRunner) Wait() { r.wg.Wait() }
|
||||
|
||||
func (r *chatShellRunner) activeProcess(sessionID, runID string) *chatShellProcess {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if p := r.active[runID]; p != nil && p.record.SessionID == sessionID {
|
||||
return p
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// snapshot is the process's current record carrying at most max output bytes.
|
||||
func (p *chatShellProcess) snapshot(max int) protocol.ChatShellRunRecord {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
record := p.record
|
||||
record.Output, record.OutputStart = p.out.tail(max)
|
||||
record.OutputBytes = p.out.total
|
||||
record.Truncated = record.OutputStart > 0
|
||||
return record
|
||||
}
|
||||
|
||||
func (r *chatShellRunner) publishRun(record protocol.ChatShellRunRecord) {
|
||||
if r.publish == nil {
|
||||
return
|
||||
}
|
||||
record.Output = ""
|
||||
r.publish(record.SessionID, protocol.ChatShellRunEventFrame{SessionID: record.SessionID, RunID: record.ID, Run: &record})
|
||||
}
|
||||
|
||||
func (h *chatHub) shellRunner() *chatShellRunner {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
return h.shellRuns
|
||||
}
|
||||
|
||||
// startShellRunner gives the hub its operator command runner over store,
|
||||
// streaming to the session's room, after interrupting runs a previous server
|
||||
// process left recorded as running. A nil store leaves commands refused.
|
||||
func (h *chatHub) startShellRunner(ctx context.Context, store chatShellRunStore) {
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
h.shellRuns = newChatShellRunner(ctx, store, func(sessionID string, frame protocol.ChatShellRunEventFrame) {
|
||||
h.fanout(sessionID, protocol.Message{Type: protocol.ChatShellRunEvent, ChatShellRunEvent: &frame}, "")
|
||||
})
|
||||
h.shellRuns.recover(ctx)
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// shellFrames records every frame a test runner publishes.
|
||||
type shellFrames struct {
|
||||
mu sync.Mutex
|
||||
frames []protocol.ChatShellRunEventFrame
|
||||
}
|
||||
|
||||
func (f *shellFrames) add(_ string, frame protocol.ChatShellRunEventFrame) {
|
||||
f.mu.Lock()
|
||||
f.frames = append(f.frames, frame)
|
||||
f.mu.Unlock()
|
||||
}
|
||||
|
||||
func (f *shellFrames) final(runID string) *protocol.ChatShellRunRecord {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, frame := range f.frames {
|
||||
if frame.RunID == runID && frame.Run != nil && frame.Run.Status != protocol.ChatShellRunRunning {
|
||||
return frame.Run
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func testShellRunner(t *testing.T, ctx context.Context) (*chatShellRunner, *shellFrames) {
|
||||
t.Helper()
|
||||
frames := &shellFrames{}
|
||||
runner := newChatShellRunner(ctx, newOperatorStore(t), frames.add)
|
||||
runner.grace = 200 * time.Millisecond
|
||||
runner.flush = 10 * time.Millisecond
|
||||
return runner, frames
|
||||
}
|
||||
|
||||
func TestChatShellStopKillsAfterGrace(t *testing.T) {
|
||||
runner, frames := testShellRunner(t, context.Background())
|
||||
run, err := runner.Start(context.Background(), "ses_1", t.TempDir(), `trap "" INT; echo ready; while :; do sleep 0.05; done`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if _, err := runner.Stop("ses_2", run.ID); err == nil {
|
||||
t.Fatal("another session stopped this run")
|
||||
}
|
||||
if _, err := runner.Stop("ses_1", run.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner.Wait()
|
||||
final := frames.final(run.ID)
|
||||
if final == nil || final.Status != protocol.ChatShellRunStopped || final.Signal != "killed" || final.DurationMS < 200 {
|
||||
t.Fatalf("final = %+v", final)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellRetainsBoundedTail(t *testing.T) {
|
||||
runner, frames := testShellRunner(t, context.Background())
|
||||
total := protocol.ChatShellRunRetained + 200_000
|
||||
run, err := runner.Start(context.Background(), "ses_1", t.TempDir(), `head -c 1248576 /dev/zero | tr '\0' a; printf END`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner.Wait()
|
||||
stored, err := runner.Read(context.Background(), "ses_1", run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stored.OutputBytes != int64(total+3) || len(stored.Output) != protocol.ChatShellRunRetained || !stored.Truncated || stored.OutputStart != stored.OutputBytes-int64(protocol.ChatShellRunRetained) || !strings.HasSuffix(stored.Output, "aEND") {
|
||||
t.Fatalf("stored = bytes %d len %d start %d truncated %v", stored.OutputBytes, len(stored.Output), stored.OutputStart, stored.Truncated)
|
||||
}
|
||||
list, err := runner.List(context.Background(), "ses_1")
|
||||
if err != nil || len(list) != 1 || len(list[0].Output) != protocol.ChatShellRunListOutput || list[0].OutputStart != stored.OutputBytes-protocol.ChatShellRunListOutput {
|
||||
t.Fatalf("list = %d runs, err %v", len(list), err)
|
||||
}
|
||||
frames.mu.Lock()
|
||||
defer frames.mu.Unlock()
|
||||
for _, frame := range frames.frames {
|
||||
if len(frame.Output) > protocol.ChatShellRunChunkLimit {
|
||||
t.Fatalf("an output frame carried %d bytes", len(frame.Output))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellShutdownAndRestartInterrupt(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
runner, frames := testShellRunner(t, ctx)
|
||||
run, err := runner.Start(context.Background(), "ses_1", t.TempDir(), "sleep 30")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel()
|
||||
runner.Wait()
|
||||
if final := frames.final(run.ID); final == nil || final.Status != protocol.ChatShellRunInterrupted {
|
||||
t.Fatalf("shutdown final = %+v", final)
|
||||
}
|
||||
if _, err := runner.Start(context.Background(), "ses_1", t.TempDir(), "true"); err == nil {
|
||||
t.Fatal("a stopping server started a command")
|
||||
}
|
||||
// A record a crashed process left running is interrupted on start.
|
||||
store := runner.store
|
||||
if err := store.SaveChatShellRun(context.Background(), protocol.ChatShellRunRecord{ID: "run_left", SessionID: "ses_1", Command: "sleep 1", Directory: "/", Status: protocol.ChatShellRunRunning, StartedAt: 1}, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := newChatShellRunner(context.Background(), store, nil)
|
||||
next.recover(context.Background())
|
||||
left, err := next.Read(context.Background(), "ses_1", "run_left")
|
||||
if err != nil || left.Status != protocol.ChatShellRunInterrupted || left.Error != chatShellInterruptedReason {
|
||||
t.Fatalf("recovered = %+v, %v", left, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellOutputKeepsCharactersWhole(t *testing.T) {
|
||||
out := newChatShellOutput(4)
|
||||
out.write([]byte("ab\xc3"))
|
||||
if text, offset, end, ok := out.pending(64, false); !ok || text != "ab" || offset != 0 || end != 2 {
|
||||
t.Fatalf("partial character emitted: %q %d %d", text, offset, end)
|
||||
}
|
||||
out.write([]byte("\xa9cdefghij"))
|
||||
text, offset, end, ok := out.pending(3, false)
|
||||
if !ok || text != "hij" || offset != 9 || end != 12 {
|
||||
t.Fatalf("bounded chunk = %q %d %d", text, offset, end)
|
||||
}
|
||||
if tail, start := out.tail(6); tail != "ghij" || start != 8 {
|
||||
t.Fatalf("tail = %q at %d", tail, start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellSharePromptFencesOutput(t *testing.T) {
|
||||
code := 1
|
||||
prompt := chatShellSharePrompt(protocol.ChatShellRunRecord{Command: "cat notes.md", Directory: "/w", Status: protocol.ChatShellRunExited, ExitCode: &code, DurationMS: 1500, Output: "```go\nx\n```\n", OutputBytes: 13})
|
||||
if !strings.Contains(prompt, "````text\n```go\nx\n```\n````") || !strings.Contains(prompt, "exited with status 1 after 1.5s") {
|
||||
t.Fatalf("prompt = %q", prompt)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// chatShellSharePrompt is the ordinary prompt chat_shell_share queues: the
|
||||
// command, where it ran, how it ended and a bounded tail of its output. It is
|
||||
// the only path by which an operator command reaches the agent.
|
||||
func chatShellSharePrompt(record protocol.ChatShellRunRecord) string {
|
||||
output := chatShellRecordTail(record, protocol.ChatShellRunShareOutput)
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "I ran this shell command in %s:\n\n", record.Directory)
|
||||
fence := chatShellFence(record.Command)
|
||||
fmt.Fprintf(&b, "%ssh\n$ %s\n%s\n\n", fence, record.Command, fence)
|
||||
fmt.Fprintf(&b, "%s.\n\n", chatShellOutcome(record))
|
||||
if output.Output == "" {
|
||||
b.WriteString("It printed no output.")
|
||||
return b.String()
|
||||
}
|
||||
if output.OutputStart > 0 {
|
||||
fmt.Fprintf(&b, "Output (the last %s of %s):\n\n", chatShellBytes(int64(len(output.Output))), chatShellBytes(output.OutputBytes))
|
||||
} else {
|
||||
b.WriteString("Output:\n\n")
|
||||
}
|
||||
fence = chatShellFence(output.Output)
|
||||
text := strings.TrimRight(output.Output, "\n")
|
||||
fmt.Fprintf(&b, "%stext\n%s\n%s", fence, text, fence)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// chatShellOutcome describes how a run ended, in the card's own words.
|
||||
func chatShellOutcome(record protocol.ChatShellRunRecord) string {
|
||||
took := (time.Duration(record.DurationMS) * time.Millisecond).Round(time.Millisecond)
|
||||
switch record.Status {
|
||||
case protocol.ChatShellRunStopped:
|
||||
return fmt.Sprintf("I stopped it after %s", took)
|
||||
case protocol.ChatShellRunInterrupted:
|
||||
return "It was interrupted when the server stopped"
|
||||
case protocol.ChatShellRunFailed:
|
||||
return "It failed: " + record.Error
|
||||
}
|
||||
if record.Signal != "" {
|
||||
return fmt.Sprintf("It was killed by %s after %s", record.Signal, took)
|
||||
}
|
||||
if record.ExitCode != nil {
|
||||
return fmt.Sprintf("It exited with status %d after %s", *record.ExitCode, took)
|
||||
}
|
||||
return "It ended"
|
||||
}
|
||||
|
||||
// chatShellFence is a Markdown code fence longer than any backtick run in text.
|
||||
func chatShellFence(text string) string {
|
||||
longest, run := 0, 0
|
||||
for _, r := range text {
|
||||
if r == '`' {
|
||||
run++
|
||||
longest = max(longest, run)
|
||||
} else {
|
||||
run = 0
|
||||
}
|
||||
}
|
||||
return strings.Repeat("`", max(3, longest+1))
|
||||
}
|
||||
|
||||
func chatShellBytes(n int64) string {
|
||||
switch {
|
||||
case n >= 1<<20:
|
||||
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
|
||||
case n >= 1<<10:
|
||||
return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
|
||||
}
|
||||
return fmt.Sprintf("%d bytes", n)
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// supervise reads one command's merged output, streams it in batches, saves
|
||||
// it periodically and records how the command ended. It owns the reader.
|
||||
func (r *chatShellRunner) supervise(p *chatShellProcess, reader *os.File) {
|
||||
defer r.wg.Done()
|
||||
defer close(p.done)
|
||||
readDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(readDone)
|
||||
buffer := make([]byte, 32<<10)
|
||||
for {
|
||||
n, err := reader.Read(buffer)
|
||||
if n > 0 {
|
||||
p.mu.Lock()
|
||||
p.out.write(buffer[:n])
|
||||
p.mu.Unlock()
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
waitDone := make(chan error, 1)
|
||||
go func() { waitDone <- p.cmd.Wait() }()
|
||||
pid := p.cmd.Process.Pid
|
||||
flush := time.NewTicker(r.flush)
|
||||
defer flush.Stop()
|
||||
lastSaved := r.now()
|
||||
var waitErr error
|
||||
exited, drained, shutdown := false, false, false
|
||||
base := r.base.Done()
|
||||
for !exited || !drained {
|
||||
select {
|
||||
case <-flush.C:
|
||||
r.emitOutput(p, false)
|
||||
if r.now().Sub(lastSaved) >= r.persist {
|
||||
lastSaved = r.now()
|
||||
r.save(p.snapshot(protocol.ChatShellRunRetained))
|
||||
}
|
||||
case waitErr = <-waitDone:
|
||||
exited = true
|
||||
// Descendants may hold the pipe open; read what they write
|
||||
// briefly, then stop reading.
|
||||
_ = reader.SetReadDeadline(time.Now().Add(chatShellDrain))
|
||||
case <-readDone:
|
||||
drained = true
|
||||
readDone = nil
|
||||
case <-base:
|
||||
base = nil
|
||||
shutdown = true
|
||||
_ = syscall.Kill(-pid, syscall.SIGKILL)
|
||||
}
|
||||
}
|
||||
// Nothing in the command's group outlives it.
|
||||
_ = syscall.Kill(-pid, syscall.SIGKILL)
|
||||
reader.Close()
|
||||
r.emitOutput(p, true)
|
||||
p.mu.Lock()
|
||||
finishChatShellRecord(&p.record, waitErr, p.stopped, shutdown || r.base.Err() != nil, r.now())
|
||||
p.mu.Unlock()
|
||||
record := p.snapshot(protocol.ChatShellRunRetained)
|
||||
r.save(record)
|
||||
r.mu.Lock()
|
||||
delete(r.active, p.record.ID)
|
||||
r.mu.Unlock()
|
||||
r.publishRun(record)
|
||||
}
|
||||
|
||||
// finishChatShellRecord records how a command ended. A requested stop wins
|
||||
// over the exit status it caused; a server shutdown marks it interrupted.
|
||||
func finishChatShellRecord(record *protocol.ChatShellRunRecord, waitErr error, stopped, shutdown bool, now time.Time) {
|
||||
record.EndedAt = now.UnixMilli()
|
||||
record.DurationMS = max(record.EndedAt-record.StartedAt, 0)
|
||||
var exitErr *exec.ExitError
|
||||
switch {
|
||||
case waitErr == nil:
|
||||
code := 0
|
||||
record.ExitCode = &code
|
||||
case errors.As(waitErr, &exitErr):
|
||||
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() {
|
||||
record.Signal = status.Signal().String()
|
||||
} else {
|
||||
code := exitErr.ExitCode()
|
||||
record.ExitCode = &code
|
||||
}
|
||||
default:
|
||||
record.Error = waitErr.Error()
|
||||
}
|
||||
switch {
|
||||
case shutdown:
|
||||
record.Status = protocol.ChatShellRunInterrupted
|
||||
record.Error = chatShellInterruptedReason
|
||||
case stopped:
|
||||
record.Status = protocol.ChatShellRunStopped
|
||||
case record.Error != "":
|
||||
record.Status = protocol.ChatShellRunFailed
|
||||
default:
|
||||
record.Status = protocol.ChatShellRunExited
|
||||
}
|
||||
}
|
||||
|
||||
// emitOutput pushes the output written since the last frame.
|
||||
func (r *chatShellRunner) emitOutput(p *chatShellProcess, final bool) {
|
||||
p.mu.Lock()
|
||||
text, offset, end, ok := p.out.pending(protocol.ChatShellRunChunkLimit, final)
|
||||
sessionID, runID := p.record.SessionID, p.record.ID
|
||||
p.mu.Unlock()
|
||||
if !ok || r.publish == nil {
|
||||
return
|
||||
}
|
||||
r.publish(sessionID, protocol.ChatShellRunEventFrame{SessionID: sessionID, RunID: runID, Output: text, Offset: offset, End: end})
|
||||
}
|
||||
|
||||
// save persists a record; a shutdown's base context is already canceled, so
|
||||
// it uses its own bounded context.
|
||||
func (r *chatShellRunner) save(record protocol.ChatShellRunRecord) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = r.store.SaveChatShellRun(ctx, record, protocol.ChatShellRunHistory)
|
||||
}
|
||||
@@ -22,6 +22,13 @@ const chatPermissionPollInterval = 2 * time.Second
|
||||
// every other client, attributed to the sending device's label, and the turn's
|
||||
// output is fanned out to all of them.
|
||||
func handleChatPrompt(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error) {
|
||||
admitChatPrompt(ctx, request, broker, hub, registry, clientID, write, false)
|
||||
}
|
||||
|
||||
// admitChatPrompt is handleChatPrompt; echoSender also publishes the user turn
|
||||
// to the sending client, for a prompt the server composed (chat_shell_share)
|
||||
// that the sender has no local copy of.
|
||||
func admitChatPrompt(ctx context.Context, request protocol.Message, broker chatBroker, hub *chatHub, registry operatorRegistry, clientID string, write func(protocol.Message) error, echoSender bool) {
|
||||
if broker == nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat is not configured on this server"})
|
||||
return
|
||||
@@ -128,7 +135,11 @@ func handleChatPrompt(ctx context.Context, request protocol.Message, broker chat
|
||||
accepted = true
|
||||
queued := admission == chatAdmitQueued
|
||||
// Publish the user turn to every other client; the sender already has it.
|
||||
hub.broadcastUser(userFrame, clientID)
|
||||
exceptClientID := clientID
|
||||
if echoSender {
|
||||
exceptClientID = ""
|
||||
}
|
||||
hub.broadcastUser(userFrame, exceptClientID)
|
||||
if admission == chatAdmitSteered {
|
||||
hub.markOpSteered(sessionID, operationID)
|
||||
// The harness took the message into its running turn: it is not queued,
|
||||
|
||||
@@ -135,6 +135,13 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
hub.queueStore = stores[0]
|
||||
hub.liveStore = stores[0]
|
||||
}
|
||||
// Operator shell commands persist beside the operator sessions they belong
|
||||
// to: the execution store, else an injected store-backed registry.
|
||||
if len(stores) > 0 {
|
||||
hub.startShellRunner(ctx, stores[0])
|
||||
} else if store, ok := registry.(chatShellRunStore); ok {
|
||||
hub.startShellRunner(ctx, store)
|
||||
}
|
||||
acks := newAttentionAckHub()
|
||||
var ackStore attentionAckStore
|
||||
var notes *appNotifier
|
||||
|
||||
Reference in New Issue
Block a user