From 277384c864f1915c3d3049aa5c00552fe4e9c68b Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Mon, 9 Nov 2026 21:30:15 +0000 Subject: [PATCH] Add persistent machine terminals with tmux-backed reattach --- internal/pipeline/store.go | 4 + internal/pipeline/terminals.go | 88 +++++++ internal/pipeline/terminals_test.go | 54 ++++ internal/protocol/message.go | 6 + internal/protocol/terminal.go | 110 ++++++++ internal/server/terminal_clients.go | 91 +++++++ internal/server/terminal_cwd.go | 41 +++ internal/server/terminal_handlers.go | 65 +++++ internal/server/terminal_manager.go | 314 +++++++++++++++++++++++ internal/server/terminal_process.go | 173 +++++++++++++ internal/server/terminal_restart_test.go | 223 ++++++++++++++++ internal/server/terminal_supervise.go | 169 ++++++++++++ internal/server/terminal_title.go | 90 +++++++ internal/server/terminal_tmux.go | 120 +++++++++ internal/server/terminal_ws_test.go | 173 +++++++++++++ internal/server/websocket.go | 37 ++- 16 files changed, 1757 insertions(+), 1 deletion(-) create mode 100644 internal/pipeline/terminals.go create mode 100644 internal/pipeline/terminals_test.go create mode 100644 internal/protocol/terminal.go create mode 100644 internal/server/terminal_clients.go create mode 100644 internal/server/terminal_cwd.go create mode 100644 internal/server/terminal_handlers.go create mode 100644 internal/server/terminal_manager.go create mode 100644 internal/server/terminal_process.go create mode 100644 internal/server/terminal_restart_test.go create mode 100644 internal/server/terminal_supervise.go create mode 100644 internal/server/terminal_title.go create mode 100644 internal/server/terminal_tmux.go create mode 100644 internal/server/terminal_ws_test.go diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index a0eca5b7..75493470 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -109,6 +109,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU db.Close() return nil, err } + if err := s.initTerminals(); err != nil { + db.Close() + return nil, err + } if err := s.initAppNotifications(); err != nil { db.Close() return nil, err diff --git a/internal/pipeline/terminals.go b/internal/pipeline/terminals.go new file mode 100644 index 00000000..2883fcf5 --- /dev/null +++ b/internal/pipeline/terminals.go @@ -0,0 +1,88 @@ +package pipeline + +import ( + "context" + "database/sql" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// initTerminals creates the additive table of machine terminals. It records +// only what a restarted server needs to list and reattach them (id, title, +// cwd, shell, tmux session, status). Output and typed input are never stored. +func (s *Store) initTerminals() error { + _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS terminals ( +id TEXT PRIMARY KEY, +title TEXT NOT NULL DEFAULT '', +cwd TEXT NOT NULL, +shell TEXT NOT NULL DEFAULT '', +created_at INTEGER NOT NULL, +tmux TEXT NOT NULL DEFAULT '', +status TEXT NOT NULL, +ended_at INTEGER NOT NULL DEFAULT 0, +exit_code INTEGER, +reason TEXT NOT NULL DEFAULT '', +cols INTEGER NOT NULL DEFAULT 0, +rows INTEGER NOT NULL DEFAULT 0);`) + return err +} + +// SaveTerminal inserts or replaces one terminal. Saving an ended terminal +// prunes the oldest ended ones beyond keep (keep <= 0 keeps everything). +func (s *Store) SaveTerminal(ctx context.Context, t protocol.TerminalRecord, keep int) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var exit any + if t.ExitCode != nil { + exit = *t.ExitCode + } + if _, err := tx.ExecContext(ctx, `INSERT INTO terminals +(id, title, cwd, shell, created_at, tmux, status, ended_at, exit_code, reason, cols, rows) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(id) DO UPDATE SET title = excluded.title, status = excluded.status, ended_at = excluded.ended_at, +exit_code = excluded.exit_code, reason = excluded.reason, cols = excluded.cols, rows = excluded.rows`, + t.ID, t.Title, t.Cwd, t.Shell, t.CreatedAt, t.Tmux, t.Status, t.EndedAt, exit, t.Reason, t.Cols, t.Rows); err != nil { + return err + } + if keep > 0 && t.Status != protocol.TerminalRunning { + if _, err := tx.ExecContext(ctx, `DELETE FROM terminals WHERE status != ? AND id NOT IN +(SELECT id FROM terminals WHERE status != ? ORDER BY ended_at DESC, id DESC LIMIT ?)`, + protocol.TerminalRunning, protocol.TerminalRunning, keep); err != nil { + return err + } + } + return tx.Commit() +} + +// DeleteTerminal removes one terminal's record. +func (s *Store) DeleteTerminal(ctx context.Context, id string) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM terminals WHERE id = ?`, id) + return err +} + +// Terminals lists every recorded terminal, oldest first. +func (s *Store) Terminals(ctx context.Context) ([]protocol.TerminalRecord, error) { + rows, err := s.db.QueryContext(ctx, `SELECT id, title, cwd, shell, created_at, tmux, status, ended_at, exit_code, reason, cols, rows +FROM terminals ORDER BY created_at, id`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []protocol.TerminalRecord + for rows.Next() { + var t protocol.TerminalRecord + var exit sql.NullInt64 + if err := rows.Scan(&t.ID, &t.Title, &t.Cwd, &t.Shell, &t.CreatedAt, &t.Tmux, &t.Status, &t.EndedAt, &exit, &t.Reason, &t.Cols, &t.Rows); err != nil { + return nil, err + } + if exit.Valid { + code := int(exit.Int64) + t.ExitCode = &code + } + out = append(out, t) + } + return out, rows.Err() +} diff --git a/internal/pipeline/terminals_test.go b/internal/pipeline/terminals_test.go new file mode 100644 index 00000000..8d99ffbf --- /dev/null +++ b/internal/pipeline/terminals_test.go @@ -0,0 +1,54 @@ +package pipeline + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestTerminalsPersistPruneAndDelete(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "terminals.db") + store, err := Open(path) + if err != nil { + t.Fatal(err) + } + live := protocol.TerminalRecord{ID: "term_live", Cwd: "/home/a", Shell: "/bin/zsh", CreatedAt: 1, Tmux: "pipelines-term_live", Status: protocol.TerminalRunning, Cols: 80, Rows: 24} + if err := store.SaveTerminal(ctx, live, 2); err != nil { + t.Fatal(err) + } + code := 0 + for i := 0; i < 3; i++ { + ended := protocol.TerminalRecord{ID: fmt.Sprintf("term_%d", i), Cwd: "/home/a", CreatedAt: int64(10 + i), Status: protocol.TerminalExited, EndedAt: int64(20 + i), ExitCode: &code} + if err := store.SaveTerminal(ctx, ended, 2); err != nil { + t.Fatal(err) + } + } + live.Title, live.Cols = "vim", 120 + if err := store.SaveTerminal(ctx, live, 2); err != nil { + t.Fatal(err) + } + store.Close() + store, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer store.Close() + list, err := store.Terminals(ctx) + if err != nil { + t.Fatal(err) + } + if len(list) != 3 || list[0].ID != "term_live" || list[0].Title != "vim" || list[0].Cols != 120 || list[0].Tmux != "pipelines-term_live" || + list[1].ID != "term_1" || list[2].ID != "term_2" || list[2].ExitCode == nil || *list[2].ExitCode != 0 { + t.Fatalf("terminals = %+v", list) + } + if err := store.DeleteTerminal(ctx, "term_live"); err != nil { + t.Fatal(err) + } + if list, _ := store.Terminals(ctx); len(list) != 2 { + t.Fatalf("after delete = %+v", list) + } +} diff --git a/internal/protocol/message.go b/internal/protocol/message.go index e2c2ba21..570c7062 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -280,6 +280,12 @@ type Message struct { ChatShellRun *ChatShellRunRecord `json:"chat_shell_run,omitempty"` ChatShellRuns []ChatShellRunRecord `json:"chat_shell_runs,omitempty"` ChatShellRunEvent *ChatShellRunEventFrame `json:"chat_shell_run_event,omitempty"` + // Machine terminals (terminal.go): the request body, one terminal, the + // machine's terminals and the pushed change. + TerminalRequest *TerminalRequest `json:"terminal_request,omitempty"` + Terminal *TerminalRecord `json:"terminal,omitempty"` + Terminals []TerminalRecord `json:"terminals,omitempty"` + TerminalEvent *TerminalEventFrame `json:"terminal_event,omitempty"` // ListDirectory is the list_directory body and DirectoryListing its reply: // the home-restricted folder picker listing (see directories.go). ListDirectory *DirectoryListRequest `json:"list_directory,omitempty"` diff --git a/internal/protocol/terminal.go b/internal/protocol/terminal.go new file mode 100644 index 00000000..8107afc4 --- /dev/null +++ b/internal/protocol/terminal.go @@ -0,0 +1,110 @@ +package protocol + +// Machine terminals: interactive login shells the operator opens on a +// server's machine, independent of any chat. Each runs `$SHELL -l` (else +// /bin/sh) in a pseudo-terminal; where tmux is installed the shell lives in +// a Pipelines-owned tmux server (socket name "pipelines", never the +// operator's own), so it survives a Pipelines server restart and the next +// process reattaches it. Without tmux a shell ends with the server and is +// listed as ended. Typed input is never logged or stored. +const ( + // TerminalOpen starts a shell. The body is TerminalRequest (cwd, cols, + // rows); cwd defaults to the operator's home and must be an existing + // directory inside it. The reply carries the new terminal; the + // connection is then subscribed to its terminal_event output frames. + TerminalOpen = "terminal_open" + // TerminalList lists this machine's terminals (running and ended), + // oldest first, without output. The body is optional. + TerminalList = "terminal_list" + // TerminalInput writes Data (UTF-8, control characters included, at most + // ChatShellInputLimit bytes) to a running terminal. No reply body. + TerminalInput = "terminal_input" + // TerminalResize sets a running terminal's size (cols, rows); the reply + // echoes the terminal. + TerminalResize = "terminal_resize" + // TerminalOutput reads one terminal with its retained output from the + // absolute byte After on (all of it when After precedes the retained + // tail; OutputStart says where it starts). It also subscribes the + // connection to the terminal's terminal_event output frames. + TerminalOutput = "terminal_output" + // TerminalClose ends a terminal: its shell (and tmux session) is killed + // and its record removed. The reply echoes the last record. + TerminalClose = "terminal_close" + // TerminalEvent is pushed: an output chunk to the connections subscribed + // to that terminal (Output, Offset, End), or a lifecycle change (Terminal + // set: opened, title, exited, closed) to every connection. + TerminalEvent = "terminal_event" +) + +// Terminal status values. Running is the only live status. +const ( + TerminalRunning = "running" + TerminalExited = "exited" + TerminalClosed = "closed" +) + +// Terminal bounds. +const ( + // TerminalOpenLimit bounds the running terminals of one machine. + TerminalOpenLimit = 8 + // TerminalRetained is the output tail kept for reconnecting clients. + TerminalRetained = 1 << 20 + // TerminalChunkLimit bounds one live output frame. + TerminalChunkLimit = 64 << 10 + // TerminalEndedKept bounds the ended terminals still listed. + TerminalEndedKept = 16 + // TerminalTmuxHistory is the tmux scrollback of one tmux-backed shell. + TerminalTmuxHistory = 10000 + // TerminalTitleLimit bounds a recorded title. + TerminalTitleLimit = 200 +) + +// TerminalRequest is the body of every terminal request. +type TerminalRequest struct { + TerminalID string `json:"terminal_id,omitempty"` + Cwd string `json:"cwd,omitempty"` + Cols int `json:"cols,omitempty"` + Rows int `json:"rows,omitempty"` + Data string `json:"data,omitempty"` + After int64 `json:"after,omitempty"` +} + +// TerminalRecord is one terminal. Output and OutputStart are set only on +// terminal_output replies. +type TerminalRecord struct { + ID string `json:"id"` + // Title is the last terminal title (OSC 0/2) the shell set, or empty; + // clients show the cwd then. + Title string `json:"title,omitempty"` + Cwd string `json:"cwd"` + Shell string `json:"shell"` + CreatedAt int64 `json:"created_at"` + // Tmux is the tmux session name keeping the shell alive across server + // restarts; empty when the shell is a direct child of the server and + // ends with it. + Tmux string `json:"tmux,omitempty"` + Status string `json:"status"` + EndedAt int64 `json:"ended_at,omitempty"` + ExitCode *int `json:"exit_code,omitempty"` + // Reason says why an ended terminal ended when it was not simply its + // shell exiting (the server restarted without tmux, its tmux session + // was gone). + Reason string `json:"reason,omitempty"` + Cols int `json:"cols"` + Rows int `json:"rows"` + // OutputEnd is the absolute byte count this server process has read; + // it restarts at zero when a restarted server reattaches. + OutputEnd int64 `json:"output_end"` + Output string `json:"output,omitempty"` + OutputStart int64 `json:"output_start,omitempty"` +} + +// TerminalEventFrame is one pushed terminal change: either Terminal (a +// lifecycle change) or one output chunk covering bytes [Offset, End). +type TerminalEventFrame struct { + TerminalID string `json:"terminal_id"` + Terminal *TerminalRecord `json:"terminal,omitempty"` + Output string `json:"output,omitempty"` + Offset int64 `json:"offset,omitempty"` + End int64 `json:"end,omitempty"` +} diff --git a/internal/server/terminal_clients.go b/internal/server/terminal_clients.go new file mode 100644 index 00000000..4578dd63 --- /dev/null +++ b/internal/server/terminal_clients.go @@ -0,0 +1,91 @@ +package server + +import ( + "sync" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// terminalClientQueue bounds the frames waiting for one slow connection. +// A full queue drops output frames; the client sees the gap in the next +// frame's offset and reads the retained tail again (terminal_output after). +const terminalClientQueue = 256 + +// terminalClients delivers terminal_event frames to open connections, in +// order per connection: lifecycle changes to every connection, output only +// to connections subscribed to that terminal (by terminal_open or +// terminal_output). It holds each connection's serialized write function, +// never the socket. The zero value is ready to use. +type terminalClients struct { + mu sync.Mutex + clients map[string]*terminalClient +} + +type terminalClient struct { + ch chan protocol.Message + done chan struct{} + subs map[string]bool +} + +func (c *terminalClients) attach(clientID string, write func(protocol.Message) error) { + client := &terminalClient{ch: make(chan protocol.Message, terminalClientQueue), done: make(chan struct{}), subs: map[string]bool{}} + c.mu.Lock() + if c.clients == nil { + c.clients = map[string]*terminalClient{} + } + if old := c.clients[clientID]; old != nil { + close(old.done) + } + c.clients[clientID] = client + c.mu.Unlock() + go func() { + for { + select { + case <-client.done: + return + case msg := <-client.ch: + if write(msg) != nil { + return + } + } + } + }() +} + +func (c *terminalClients) detach(clientID string) { + c.mu.Lock() + defer c.mu.Unlock() + if client := c.clients[clientID]; client != nil { + close(client.done) + delete(c.clients, clientID) + } +} + +// subscribe sends terminalID's output frames to clientID from now on. +func (c *terminalClients) subscribe(clientID, terminalID string) { + c.mu.Lock() + defer c.mu.Unlock() + if client := c.clients[clientID]; client != nil { + client.subs[terminalID] = true + } +} + +// publish queues frame for its recipients without blocking: a lifecycle +// change for everyone, output for the terminal's subscribers. +func (c *terminalClients) publish(frame protocol.TerminalEventFrame) { + msg := protocol.Message{Type: protocol.TerminalEvent, TerminalEvent: &frame} + c.mu.Lock() + defer c.mu.Unlock() + for _, client := range c.clients { + if frame.Terminal == nil && !client.subs[frame.TerminalID] { + continue + } + if frame.Terminal != nil && frame.Terminal.Status == protocol.TerminalClosed { + delete(client.subs, frame.TerminalID) + } + select { + case client.ch <- msg: + default: + } + } +} diff --git a/internal/server/terminal_cwd.go b/internal/server/terminal_cwd.go new file mode 100644 index 00000000..1a002997 --- /dev/null +++ b/internal/server/terminal_cwd.go @@ -0,0 +1,41 @@ +package server + +import ( + "errors" + "os" + "path/filepath" +) + +// resolveTerminalCwd is the directory a terminal starts in: requested ("", +// "~", "~/sub" or an absolute path) must be an existing directory inside +// home once every symlink is resolved, the containment the Files panel and +// folder picker apply. It returns the path as the operator sees it, under +// home. +func resolveTerminalCwd(home, requested string) (string, error) { + home = filepath.Clean(home) + realHome, err := filepath.EvalSymlinks(home) + if err != nil { + return "", errors.New("resolve home directory: " + err.Error()) + } + target, err := expandHomePath(home, requested) + if err != nil { + return "", err + } + realTarget, err := filepath.EvalSymlinks(target) + if err != nil { + return "", describePathError(target, err) + } + rel, inside := relInside(realHome, realTarget) + if !inside { + return "", errors.New(target + " is outside the home directory " + home + "; a terminal starts only in a folder inside home") + } + shown := filepath.Join(home, rel) + info, err := os.Stat(realTarget) + if err != nil { + return "", describePathError(shown, err) + } + if !info.IsDir() { + return "", errors.New(shown + " is not a directory") + } + return shown, nil +} diff --git a/internal/server/terminal_handlers.go b/internal/server/terminal_handlers.go new file mode 100644 index 00000000..bd8bd219 --- /dev/null +++ b/internal/server/terminal_handlers.go @@ -0,0 +1,65 @@ +package server + +import ( + "context" + "errors" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// isTerminalRequest reports the request types handleTerminalRequest answers. +func isTerminalRequest(kind string) bool { + switch kind { + case protocol.TerminalOpen, protocol.TerminalList, protocol.TerminalInput, protocol.TerminalResize, + protocol.TerminalOutput, protocol.TerminalClose: + return true + } + return false +} + +// handleTerminalRequest answers one machine-terminal request from an +// authenticated connection. Opening or reading a terminal subscribes the +// connection to its output frames. +func handleTerminalRequest(ctx context.Context, request protocol.Message, m *terminalManager, clientID string, write func(protocol.Message) error) { + fail := func(err error) { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) } + if m == nil { + fail(errors.New("terminals are not configured on this server")) + return + } + body := request.TerminalRequest + if body == nil { + body = &protocol.TerminalRequest{} + } + if request.Type != protocol.TerminalOpen && request.Type != protocol.TerminalList && body.TerminalID == "" { + fail(errors.New("a terminal_id is required")) + return + } + reply := protocol.Message{Type: request.Type, ID: request.ID} + var record protocol.TerminalRecord + var err error + switch request.Type { + case protocol.TerminalOpen: + record, err = m.Open(ctx, clientID, body.Cwd, body.Cols, body.Rows) + case protocol.TerminalList: + var records []protocol.TerminalRecord + records, err = m.List(ctx) + reply.Terminals = records + case protocol.TerminalInput: + err = m.Input(body.TerminalID, body.Data) + case protocol.TerminalResize: + record, err = m.Resize(body.TerminalID, body.Cols, body.Rows) + case protocol.TerminalOutput: + m.clients.subscribe(clientID, body.TerminalID) + record, err = m.Output(ctx, body.TerminalID, body.After) + case protocol.TerminalClose: + record, err = m.Close(ctx, body.TerminalID) + } + if err != nil { + fail(err) + return + } + if record.ID != "" { + reply.Terminal = &record + } + _ = write(reply) +} diff --git a/internal/server/terminal_manager.go b/internal/server/terminal_manager.go new file mode 100644 index 00000000..146eae71 --- /dev/null +++ b/internal/server/terminal_manager.go @@ -0,0 +1,314 @@ +package server + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "log" + "os" + "os/exec" + "syscall" + "sync" + "time" + "unicode/utf8" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// terminalStore persists the machine's terminal list; *pipeline.Store +// implements it. +type terminalStore interface { + SaveTerminal(ctx context.Context, terminal protocol.TerminalRecord, keep int) error + DeleteTerminal(ctx context.Context, id string) error + Terminals(ctx context.Context) ([]protocol.TerminalRecord, error) +} + +const ( + terminalNoTmuxReason = "The Pipelines server restarted and this shell ended with it (tmux is not installed on this machine, so shells do not outlive the server)." + terminalGoneReason = "Its tmux session ended while the Pipelines server was stopped." + terminalStopReason = "The Pipelines server stopped and this shell ended with it (tmux is not installed on this machine)." + // terminalReattachLimit bounds how often a tmux client that exits while + // its session lives is replaced before the terminal is given up. + terminalReattachLimit = 5 +) + +// terminalManager owns the machine's terminals (terminal.go in protocol). +type terminalManager struct { + base context.Context + store terminalStore + clients *terminalClients + tmux *terminalTmux + shell func() string + home func() (string, error) + now func() time.Time + flush time.Duration + + mu sync.Mutex + terms map[string]*terminalProcess + wg sync.WaitGroup +} + +func newTerminalManager(base context.Context, store terminalStore, tmux *terminalTmux) *terminalManager { + return &terminalManager{ + base: base, store: store, clients: &terminalClients{}, tmux: tmux, + shell: chatShellProgram, home: OperatorHomeDirectory, now: time.Now, flush: 50 * time.Millisecond, + terms: map[string]*terminalProcess{}, + } +} + +func newTerminalID() (string, error) { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return "term_" + hex.EncodeToString(b[:]), nil +} + +// Open starts a login shell in cwd at cols x rows and returns its record. +// clientID (when set) is subscribed to its output before the shell writes. +func (m *terminalManager) Open(ctx context.Context, clientID, cwd string, cols, rows int) (protocol.TerminalRecord, error) { + if m.store == nil { + return protocol.TerminalRecord{}, errors.New("terminals need the server's store, which is not configured") + } + if !chatShellTerminals { + return protocol.TerminalRecord{}, errors.New(protocol.ChatShellTerminalUnsupported) + } + home, err := m.home() + if err != nil { + return protocol.TerminalRecord{}, err + } + dir, err := resolveTerminalCwd(home, cwd) + if err != nil { + return protocol.TerminalRecord{}, err + } + id, err := newTerminalID() + if err != nil { + return protocol.TerminalRecord{}, err + } + m.mu.Lock() + defer m.mu.Unlock() + if m.base.Err() != nil { + return protocol.TerminalRecord{}, errors.New("the server is stopping") + } + if running := m.runningLocked(); running >= protocol.TerminalOpenLimit { + return protocol.TerminalRecord{}, fmt.Errorf("this machine already has %d open terminals; close one first", running) + } + cols, rows = chatShellTerminalSize(cols, rows) + shell := m.shell() + record := protocol.TerminalRecord{ID: id, Cwd: dir, Shell: shell, CreatedAt: m.now().UnixMilli(), Status: protocol.TerminalRunning, Cols: cols, Rows: rows} + var cmd *exec.Cmd + if m.tmux != nil { + record.Tmux = terminalTmuxName(id) + if err := m.tmux.create(record.Tmux, dir, shell, cols, rows); err != nil { + return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err) + } + cmd = m.tmux.attach(record.Tmux) + } else { + cmd = exec.Command(shell, "-l") + cmd.Env = terminalEnv(os.Environ(), dir) + } + cmd.Dir = dir + p := newTerminalProcess(record) + terminal, err := p.startAttachment(cmd) + if err == nil { + err = m.store.SaveTerminal(ctx, record, protocol.TerminalEndedKept) + if err != nil { + p.signal(syscall.SIGKILL) + terminal.Close() + _ = cmd.Wait() + } + } + if err != nil { + if record.Tmux != "" { + _ = m.tmux.kill(record.Tmux) + } + return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err) + } + m.terms[id] = p + if clientID != "" { + m.clients.subscribe(clientID, id) + } + m.wg.Add(1) + go m.supervise(p, cmd, terminal) + go p.pumpInput() + m.publishRecord(record) + return record, nil +} + +func (m *terminalManager) runningLocked() int { + running := 0 + for _, p := range m.terms { + if p.running() { + running++ + } + } + return running +} + +// List is every recorded terminal, oldest first, with the live state of the +// ones this process holds. +func (m *terminalManager) List(ctx context.Context) ([]protocol.TerminalRecord, error) { + if m.store == nil { + return nil, errors.New("terminals need the server's store, which is not configured") + } + records, err := m.store.Terminals(ctx) + if err != nil { + return nil, err + } + for i, record := range records { + if p := m.process(record.ID); p != nil { + records[i] = p.snapshot(false, 0) + } + } + return records, nil +} + +func (m *terminalManager) process(id string) *terminalProcess { + m.mu.Lock() + defer m.mu.Unlock() + return m.terms[id] +} + +func (m *terminalManager) live(id string) (*terminalProcess, error) { + p := m.process(id) + if p == nil || !p.running() { + return nil, errors.New("this terminal is not running") + } + return p, nil +} + +// Input queues data for a running terminal, in arrival order. It is kept +// nowhere else: what the shell echoes is output. +func (m *terminalManager) Input(id, data string) error { + switch { + case data == "": + return errors.New("input is required") + case len(data) > protocol.ChatShellInputLimit: + return errors.New("the input exceeds 16 KiB; send it in parts") + case !utf8.ValidString(data): + return errors.New("input must be UTF-8 text") + } + p, err := m.live(id) + if err != nil { + return err + } + select { + case <-p.done: + return errors.New("this terminal is not running") + case p.input <- []byte(data): + return nil + default: + return errors.New("the terminal is not reading its input; wait for it or press Ctrl-C") + } +} + +// Resize sets a running terminal's size and returns its record. +func (m *terminalManager) Resize(id string, cols, rows int) (protocol.TerminalRecord, error) { + if cols <= 0 || rows <= 0 { + return protocol.TerminalRecord{}, errors.New("positive cols and rows are required") + } + p, err := m.live(id) + if err != nil { + return protocol.TerminalRecord{}, err + } + cols, rows = chatShellTerminalSize(cols, rows) + p.mu.Lock() + p.record.Cols, p.record.Rows = cols, rows + terminal := p.pty + p.mu.Unlock() + if terminal != nil { + if err := resizeChatShellTerminal(terminal, cols, rows); err != nil { + return protocol.TerminalRecord{}, err + } + } + return p.snapshot(false, 0), nil +} + +// Output is one terminal with its retained output from byte after on. A +// terminal ended before this process started has no output. +func (m *terminalManager) Output(ctx context.Context, id string, after int64) (protocol.TerminalRecord, error) { + if p := m.process(id); p != nil { + return p.snapshot(true, after), nil + } + records, err := m.List(ctx) + if err != nil { + return protocol.TerminalRecord{}, err + } + for _, record := range records { + if record.ID == id { + return record, nil + } + } + return protocol.TerminalRecord{}, errors.New("this machine has no such terminal") +} + +// Close ends a terminal (killing its shell and tmux session) and forgets +// it. It waits briefly for the shell to go and returns the closed record. +func (m *terminalManager) Close(ctx context.Context, id string) (protocol.TerminalRecord, error) { + if m.store == nil { + return protocol.TerminalRecord{}, errors.New("terminals need the server's store, which is not configured") + } + p := m.process(id) + if p != nil && p.running() { + p.mu.Lock() + p.closing = true + name := p.record.Tmux + p.mu.Unlock() + if name != "" && m.tmux != nil { + if err := m.tmux.kill(name); err != nil { + log.Printf("terminal close %s: %v", id, err) + } + } + p.signal(syscall.SIGHUP) + select { + case <-p.done: + case <-time.After(3 * time.Second): + p.signal(syscall.SIGKILL) + <-p.done + } + return p.snapshot(false, 0), nil + } + record, err := m.Output(ctx, id, 0) + if err != nil { + return protocol.TerminalRecord{}, err + } + if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) { + _ = m.tmux.kill(record.Tmux) + } + m.forget(id) + record.Output, record.OutputStart = "", 0 + record.Status = protocol.TerminalClosed + m.publishRecord(record) + return record, nil +} + +// forget drops a terminal's record and retained output. +func (m *terminalManager) forget(id string) { + m.mu.Lock() + delete(m.terms, id) + m.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := m.store.DeleteTerminal(ctx, id); err != nil { + log.Printf("terminal delete %s: %v", id, err) + } +} + +func (m *terminalManager) save(record protocol.TerminalRecord) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := m.store.SaveTerminal(ctx, record, protocol.TerminalEndedKept); err != nil { + log.Printf("terminal save %s: %v", record.ID, err) + } +} + +func (m *terminalManager) publishRecord(record protocol.TerminalRecord) { + record.Output, record.OutputStart = "", 0 + m.clients.publish(protocol.TerminalEventFrame{TerminalID: record.ID, Terminal: &record}) +} + +// Wait blocks until every supervised terminal has been released (tests, +// shutdown). +func (m *terminalManager) Wait() { m.wg.Wait() } diff --git a/internal/server/terminal_process.go b/internal/server/terminal_process.go new file mode 100644 index 00000000..36883dfe --- /dev/null +++ b/internal/server/terminal_process.go @@ -0,0 +1,173 @@ +package server + +import ( + "errors" + "os" + "os/exec" + "sync" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// terminalProcess is one terminal this server process holds: running (its +// attachment is the shell itself, or a tmux client showing the shell's +// session) or ended with its output still retained. +type terminalProcess struct { + mu sync.Mutex + record protocol.TerminalRecord + out *chatShellOutput + titles terminalTitles + // cmd and pty are the current attachment; a tmux client that exits while + // its session lives is replaced. + cmd *exec.Cmd + pty *os.File + input chan []byte + done chan struct{} + // closing is set by terminal_close: the end is a close, not an exit. + closing bool +} + +func newTerminalProcess(record protocol.TerminalRecord) *terminalProcess { + return &terminalProcess{ + record: record, + out: newChatShellOutput(protocol.TerminalRetained), + input: make(chan []byte, protocol.ChatShellInputQueue), + done: make(chan struct{}), + } +} + +func (p *terminalProcess) running() bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.record.Status == protocol.TerminalRunning +} + +// snapshot is the current record; with output, the retained bytes from +// absolute byte after on. +func (p *terminalProcess) snapshot(output bool, after int64) protocol.TerminalRecord { + p.mu.Lock() + defer p.mu.Unlock() + record := p.record + record.OutputEnd = p.out.total + if output { + text, start := p.out.tail(protocol.TerminalRetained) + if after > start { + skip := min(after-start, int64(len(text))) + cut := runeStartAfter([]byte(text), int(skip)) + text, start = text[cut:], start+int64(cut) + } + record.Output, record.OutputStart = text, start + } + return record +} + +// startAttachment starts cmd in a fresh pseudo-terminal at the record's +// size and makes it the current attachment. +func (p *terminalProcess) startAttachment(cmd *exec.Cmd) (*os.File, error) { + if !chatShellTerminals { + return nil, errors.New(protocol.ChatShellTerminalUnsupported) + } + p.mu.Lock() + cols, rows := p.record.Cols, p.record.Rows + p.mu.Unlock() + terminal, err := startChatShellTerminal(cmd, cols, rows) + if err != nil { + return nil, err + } + p.mu.Lock() + p.cmd, p.pty = cmd, terminal + p.mu.Unlock() + return terminal, nil +} + +// read copies one attachment's stream into the retained output and follows +// the window title; it returns when the attachment's terminal is closed. +func (p *terminalProcess) read(terminal *os.File, titled func(string)) { + buffer := make([]byte, 32<<10) + for { + n, err := terminal.Read(buffer) + if n > 0 { + p.mu.Lock() + p.out.write(buffer[:n]) + title, changed := p.titles.feed(buffer[:n]) + changed = changed && title != p.record.Title + if changed { + p.record.Title = title + } + p.mu.Unlock() + if changed { + titled(title) + } + } + if err != nil { + return + } + } +} + +// pumpInput writes queued input to the current attachment until the +// terminal ends. Input arriving while a tmux client is replaced is dropped. +func (p *terminalProcess) pumpInput() { + for { + select { + case <-p.done: + return + case data := <-p.input: + p.mu.Lock() + terminal := p.pty + p.mu.Unlock() + if terminal != nil { + _, _ = terminal.Write(data) + } + } + } +} + +// signal sends sig to the current attachment's process group. +func (p *terminalProcess) signal(sig syscall.Signal) { + p.mu.Lock() + cmd := p.cmd + p.mu.Unlock() + if cmd != nil && cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, sig) + } +} + +// waitAttachment waits for the attachment to exit and its stream to drain +// (a descendant holding the terminal is read briefly, then cut off), then +// kills its group and closes its terminal. It returns the exit error. +func (p *terminalProcess) waitAttachment(cmd *exec.Cmd, terminal *os.File, readDone <-chan struct{}) error { + err := cmd.Wait() + _ = terminal.SetReadDeadline(time.Now().Add(chatShellDrain)) + select { + case <-readDone: + case <-time.After(chatShellDrain + chatShellDrain/2): + } + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + p.mu.Lock() + terminal.Close() + if p.pty == terminal { + p.pty = nil + } + p.mu.Unlock() + return err +} + +// exitCode is the shell's exit status, when it exited by itself. +func terminalExitCode(err error) *int { + code := 0 + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() { + return nil + } + code = exitErr.ExitCode() + default: + return nil + } + return &code +} diff --git a/internal/server/terminal_restart_test.go b/internal/server/terminal_restart_test.go new file mode 100644 index 00000000..37bd774f --- /dev/null +++ b/internal/server/terminal_restart_test.go @@ -0,0 +1,223 @@ +//go:build darwin || linux + +package server + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// testTmux is a tmux server of this test alone, killed afterwards. +func testTmux(t *testing.T) *terminalTmux { + t.Helper() + tmux := findTerminalTmux(fmt.Sprintf("pipelines-test-%d-%d", os.Getpid(), time.Now().UnixNano())) + if tmux == nil { + t.Skip("tmux is not installed") + } + t.Cleanup(func() { + _ = tmux.command("kill-server").Run() + dir := os.Getenv("TMUX_TMPDIR") + if dir == "" { + dir = "/tmp" + } + _ = os.Remove(filepath.Join(dir, fmt.Sprintf("tmux-%d", os.Getuid()), tmux.socket)) + }) + return tmux +} + +// newTestTerminals is one server process's terminal manager over store. +func newTestTerminals(t *testing.T, store *pipeline.Store, tmux *terminalTmux) (*terminalManager, context.CancelFunc) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + m := newTerminalManager(ctx, store, tmux) + m.shell = func() string { return "/bin/sh" } + m.recover(ctx) + t.Cleanup(func() { cancel(); m.Wait() }) + return m, cancel +} + +// waitOutput polls a terminal's retained output until it contains want. +func waitTerminalOutput(t *testing.T, m *terminalManager, id, want string) protocol.TerminalRecord { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + record, err := m.Output(context.Background(), id, 0) + if err != nil { + t.Fatal(err) + } + if strings.Contains(record.Output, want) { + return record + } + if time.Now().After(deadline) { + t.Fatalf("output of %s never contained %q: %q", id, want, record.Output) + } + time.Sleep(50 * time.Millisecond) + } +} + +// A tmux-backed shell outlives its server process: the next one lists it as +// running, reattaches, redraws its screen and takes input again. Closing it +// ends the tmux session. +func TestTerminalTmuxSurvivesRestart(t *testing.T) { + terminalHome(t) + tmux := testTmux(t) + store := newOperatorStore(t) + first, stop := newTestTerminals(t, store, tmux) + term, err := first.Open(context.Background(), "", "~", 90, 20) + if err != nil { + t.Fatal(err) + } + if term.Tmux != terminalTmuxName(term.ID) { + t.Fatalf("tmux name = %q", term.Tmux) + } + if err := first.Input(term.ID, "export KEEP=still-here; echo marker-$((6*7))\r"); err != nil { + t.Fatal(err) + } + waitTerminalOutput(t, first, term.ID, "marker-42") + stop() + first.Wait() + if !tmux.alive(term.Tmux) { + t.Fatal("the tmux session ended with the server") + } + second, _ := newTestTerminals(t, store, tmux) + list, err := second.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(list) != 1 || list[0].ID != term.ID || list[0].Status != protocol.TerminalRunning { + t.Fatalf("list after restart = %+v", list) + } + // The reattached client redraws the screen, earlier output included, + // and the shell keeps its state. + waitTerminalOutput(t, second, term.ID, "marker-42") + if err := second.Input(term.ID, "echo $KEEP\r"); err != nil { + t.Fatal(err) + } + waitTerminalOutput(t, second, term.ID, "still-here") + closed, err := second.Close(context.Background(), term.ID) + if err != nil || closed.Status != protocol.TerminalClosed { + t.Fatalf("close = %+v, %v", closed, err) + } + if tmux.alive(term.Tmux) { + t.Fatal("the tmux session outlived close") + } + if list, _ := second.List(context.Background()); len(list) != 0 { + t.Fatalf("list after close = %+v", list) + } +} + +// A tmux client killed while its session lives is replaced. +func TestTerminalTmuxClientReplaced(t *testing.T) { + terminalHome(t) + tmux := testTmux(t) + m, _ := newTestTerminals(t, newOperatorStore(t), tmux) + term, err := m.Open(context.Background(), "", "", 0, 0) + if err != nil { + t.Fatal(err) + } + waitTerminalOutput(t, m, term.ID, "$") + if err := tmux.command("detach-client", "-s", "="+term.Tmux).Run(); err != nil { + t.Fatal(err) + } + time.Sleep(300 * time.Millisecond) + if err := m.Input(term.ID, "echo back-$((1+1))\r"); err != nil { + t.Fatal(err) + } + waitTerminalOutput(t, m, term.ID, "back-2") + if record, _ := m.Output(context.Background(), term.ID, 0); record.Status != protocol.TerminalRunning { + t.Fatalf("record = %+v", record) + } +} + +// Without tmux a shell ends with its server: the stop is recorded as +// ended, and a record a crashed server left running is marked ended too. +func TestTerminalWithoutTmuxEndsWithServer(t *testing.T) { + terminalHome(t) + store := newOperatorStore(t) + first, stop := newTestTerminals(t, store, nil) + term, err := first.Open(context.Background(), "", "", 0, 0) + if err != nil { + t.Fatal(err) + } + stop() + first.Wait() + crashed := protocol.TerminalRecord{ID: "term_crashed", Cwd: "/", CreatedAt: 1, Status: protocol.TerminalRunning, Cols: 80, Rows: 24} + gone := protocol.TerminalRecord{ID: "term_gone", Cwd: "/", CreatedAt: 2, Tmux: "pipelines-term_gone", Status: protocol.TerminalRunning} + for _, record := range []protocol.TerminalRecord{crashed, gone} { + if err := store.SaveTerminal(context.Background(), record, 0); err != nil { + t.Fatal(err) + } + } + second, _ := newTestTerminals(t, store, testTmux(t)) + list, err := second.List(context.Background()) + if err != nil { + t.Fatal(err) + } + reasons := map[string]string{} + for _, record := range list { + if record.Status != protocol.TerminalExited || record.EndedAt == 0 { + t.Fatalf("record = %+v", record) + } + reasons[record.ID] = record.Reason + } + if reasons[term.ID] != terminalStopReason || reasons["term_crashed"] != terminalNoTmuxReason || reasons["term_gone"] != terminalGoneReason { + t.Fatalf("reasons = %+v", reasons) + } + if _, err := second.Close(context.Background(), "term_gone"); err != nil { + t.Fatal(err) + } + if list, _ := second.List(context.Background()); len(list) != 2 { + t.Fatalf("list after closing an ended terminal = %+v", list) + } +} + +// The window title the shell sets is followed across reads and recorded. +func TestTerminalTitles(t *testing.T) { + var titles terminalTitles + if _, ok := titles.feed([]byte("plain \x1b[1mtext")); ok { + t.Fatal("title from plain text") + } + if _, ok := titles.feed([]byte("\x1b]0;~/src")); ok { + t.Fatal("title before its end") + } + if title, ok := titles.feed([]byte("/app\x07more")); !ok || title != "~/src/app" { + t.Fatalf("title = %q %v", title, ok) + } + if title, ok := titles.feed([]byte("\x1b]2;vim\x01 x\x1b\\")); !ok || title != "vim x" { + t.Fatalf("ST title = %q %v", title, ok) + } + if _, ok := titles.feed([]byte("\x1b]8;;https://x\x07")); ok { + t.Fatal("a hyperlink is not a title") + } + if title, _ := titles.feed([]byte("\x1b]0;" + strings.Repeat("é", protocol.TerminalTitleLimit) + "\x07")); len(title) > protocol.TerminalTitleLimit { + t.Fatalf("title of %d bytes", len(title)) + } + terminalHome(t) + m, _ := newTestTerminals(t, newOperatorStore(t), nil) + term, err := m.Open(context.Background(), "", "", 0, 0) + if err != nil { + t.Fatal(err) + } + if err := m.Input(term.ID, "printf '\\033]0;build-%s\\007' ok\r"); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for { + list, _ := m.List(context.Background()) + if len(list) == 1 && list[0].Title == "build-ok" { + break + } + if time.Now().After(deadline) { + t.Fatalf("list = %+v", list) + } + time.Sleep(50 * time.Millisecond) + } +} diff --git a/internal/server/terminal_supervise.go b/internal/server/terminal_supervise.go new file mode 100644 index 00000000..ba8b1239 --- /dev/null +++ b/internal/server/terminal_supervise.go @@ -0,0 +1,169 @@ +package server + +import ( + "context" + "log" + "os" + "os/exec" + "syscall" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// supervise streams one terminal until it ends: its first attachment is +// cmd on terminal. A tmux client that exits while its session lives (it was +// detached or killed) is replaced; a server shutdown leaves a tmux session +// running for the next server process and ends a direct shell. +func (m *terminalManager) supervise(p *terminalProcess, cmd *exec.Cmd, terminal *os.File) { + defer m.wg.Done() + defer close(p.done) + stopFlush, flushDone := make(chan struct{}), make(chan struct{}) + go m.flushLoop(p, stopFlush, flushDone) + titled := func(string) { + record := p.snapshot(false, 0) + m.save(record) + m.publishRecord(record) + } + var waitErr error + shutdown := false + for reattached := 0; ; reattached++ { + readDone := make(chan struct{}) + go func(terminal *os.File) { + defer close(readDone) + p.read(terminal, titled) + }(terminal) + attachDone := make(chan error, 1) + go func(cmd *exec.Cmd, terminal *os.File) { attachDone <- p.waitAttachment(cmd, terminal, readDone) }(cmd, terminal) + select { + case waitErr = <-attachDone: + case <-m.base.Done(): + p.signal(syscall.SIGHUP) + select { + case waitErr = <-attachDone: + case <-time.After(time.Second): + p.signal(syscall.SIGKILL) + waitErr = <-attachDone + } + } + if m.base.Err() != nil { + shutdown = true + break + } + p.mu.Lock() + closing, name := p.closing, p.record.Tmux + p.mu.Unlock() + if closing || name == "" || m.tmux == nil || reattached >= terminalReattachLimit || !m.tmux.alive(name) { + break + } + cmd = m.tmux.attach(name) + next, err := p.startAttachment(cmd) + if err != nil { + log.Printf("terminal reattach %s: %v", p.record.ID, err) + break + } + terminal = next + } + close(stopFlush) + <-flushDone + m.emit(p, true) + p.mu.Lock() + switch { + case shutdown && p.record.Tmux != "": + // Still running in tmux: the next server process reattaches it. + case p.closing: + p.record.Status = protocol.TerminalClosed + default: + p.record.Status = protocol.TerminalExited + p.record.EndedAt = m.now().UnixMilli() + if shutdown { + p.record.Reason = terminalStopReason + } else if p.record.Tmux == "" { + p.record.ExitCode = terminalExitCode(waitErr) + } + } + p.pty, p.cmd = nil, nil + p.mu.Unlock() + record := p.snapshot(false, 0) + switch record.Status { + case protocol.TerminalClosed: + m.forget(record.ID) + default: + m.save(record) + } + if !shutdown { + m.publishRecord(record) + } +} + +// flushLoop pushes new output in batches until stop is closed. +func (m *terminalManager) flushLoop(p *terminalProcess, stop <-chan struct{}, done chan<- struct{}) { + defer close(done) + ticker := time.NewTicker(m.flush) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + m.emit(p, false) + } + } +} + +// emit pushes the output written since the last frame. +func (m *terminalManager) emit(p *terminalProcess, final bool) { + p.mu.Lock() + text, offset, end, ok := p.out.pending(protocol.TerminalChunkLimit, final) + id := p.record.ID + p.mu.Unlock() + if ok { + m.clients.publish(protocol.TerminalEventFrame{TerminalID: id, Output: text, Offset: offset, End: end}) + } +} + +// recover lists what a previous server process left running: a tmux-backed +// shell whose session is alive is reattached; any other is recorded as +// ended, saying why. +func (m *terminalManager) recover(ctx context.Context) { + if m.store == nil { + return + } + records, err := m.store.Terminals(ctx) + if err != nil { + log.Printf("terminal recover: %v", err) + return + } + for _, record := range records { + if record.Status != protocol.TerminalRunning { + continue + } + if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) && m.reattach(record) { + continue + } + record.Status, record.EndedAt = protocol.TerminalExited, m.now().UnixMilli() + record.Reason = terminalNoTmuxReason + if record.Tmux != "" { + record.Reason = terminalGoneReason + } + m.save(record) + } +} + +// reattach shows a live tmux session in a new client and supervises it. +func (m *terminalManager) reattach(record protocol.TerminalRecord) bool { + cmd := m.tmux.attach(record.Tmux) + p := newTerminalProcess(record) + terminal, err := p.startAttachment(cmd) + if err != nil { + log.Printf("terminal reattach %s: %v", record.ID, err) + return false + } + m.mu.Lock() + m.terms[record.ID] = p + m.mu.Unlock() + m.wg.Add(1) + go m.supervise(p, cmd, terminal) + go p.pumpInput() + return true +} diff --git a/internal/server/terminal_title.go b/internal/server/terminal_title.go new file mode 100644 index 00000000..00def6ee --- /dev/null +++ b/internal/server/terminal_title.go @@ -0,0 +1,90 @@ +package server + +import ( + "strings" + "unicode/utf8" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// terminalTitles follows a terminal stream across reads and reports the +// window title the program sets (OSC 0 or OSC 2, ended by BEL or ST), so a +// listed terminal keeps a name before any client draws it. +type terminalTitles struct { + state int + osc []byte +} + +const ( + titleText = iota + titleEscape + titleOSC + titleOSCEscape +) + +// feed scans data and returns the last title it completed, if any. +func (t *terminalTitles) feed(data []byte) (string, bool) { + title, found := "", false + for _, b := range data { + switch t.state { + case titleText: + if b == 0x1b { + t.state = titleEscape + } + case titleEscape: + if b == ']' { + t.state, t.osc = titleOSC, t.osc[:0] + } else if b != 0x1b { + t.state = titleText + } + case titleOSC: + switch { + case b == 0x07: + if s, ok := t.finish(); ok { + title, found = s, true + } + case b == 0x1b: + t.state = titleOSCEscape + case len(t.osc) < 4*protocol.TerminalTitleLimit: + t.osc = append(t.osc, b) + } + case titleOSCEscape: + if b == '\\' { + if s, ok := t.finish(); ok { + title, found = s, true + } + } else { + t.state = titleEscape + } + } + } + return title, found +} + +func (t *terminalTitles) finish() (string, bool) { + t.state = titleText + body := string(t.osc) + for _, prefix := range []string{"0;", "2;"} { + if strings.HasPrefix(body, prefix) { + return cleanTerminalTitle(body[len(prefix):]), true + } + } + return "", false +} + +// cleanTerminalTitle keeps printable text within the title bound. +func cleanTerminalTitle(title string) string { + title = strings.ToValidUTF8(title, "") + title = strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return -1 + } + return r + }, title) + title = strings.TrimSpace(title) + for len(title) > protocol.TerminalTitleLimit { + _, size := utf8.DecodeLastRuneInString(title) + title = title[:len(title)-size] + } + return title +} diff --git a/internal/server/terminal_tmux.go b/internal/server/terminal_tmux.go new file mode 100644 index 00000000..5239174b --- /dev/null +++ b/internal/server/terminal_tmux.go @@ -0,0 +1,120 @@ +package server + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// terminalTmuxSocket is the tmux server Pipelines owns (`tmux -L pipelines`): +// its sessions never mix with the operator's own tmux server. +const terminalTmuxSocket = "pipelines" + +// terminalTmux runs one Pipelines-owned tmux server. Every terminal shell is +// the only pane of its own session there, so it outlives the Pipelines +// server; the server talks to it through an ordinary attached client in a +// pseudo-terminal. +type terminalTmux struct { + program string + socket string +} + +// findTerminalTmux is the tmux on PATH (or the usual package-manager +// locations a service's PATH may lack); nil when none is installed. +func findTerminalTmux(socket string) *terminalTmux { + if socket == "" { + socket = terminalTmuxSocket + } + program, err := exec.LookPath("tmux") + if err != nil { + for _, dir := range []string{"/opt/homebrew/bin", "/usr/local/bin", "/usr/bin"} { + candidate := filepath.Join(dir, "tmux") + if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() && info.Mode()&0o111 != 0 { + program, err = candidate, nil + break + } + } + } + if err != nil { + return nil + } + return &terminalTmux{program: program, socket: socket} +} + +// terminalTmuxName is the tmux session of terminal id. +func terminalTmuxName(id string) string { return "pipelines-" + id } + +// command is one tmux client invocation against the Pipelines server, with +// no configuration file and outside any tmux the server itself runs in. +func (t *terminalTmux) command(args ...string) *exec.Cmd { + cmd := exec.Command(t.program, append([]string{"-L", t.socket, "-f", os.DevNull}, args...)...) + cmd.Env = terminalEnv(os.Environ(), "") + return cmd +} + +// create starts session name running shell -l in cwd at cols x rows, +// detached. The first call also starts the tmux server; the options make +// it invisible (no status line), pass the shell's window title through, +// keep a bounded scrollback reachable by the mouse wheel, and leave Esc +// prompt. +func (t *terminalTmux) create(name, cwd, shell string, cols, rows int) error { + cmd := t.command( + "start-server", ";", + "set-option", "-g", "history-limit", strconv.Itoa(protocol.TerminalTmuxHistory), ";", + "set-option", "-g", "status", "off", ";", + "set-option", "-g", "mouse", "on", ";", + "set-option", "-g", "escape-time", "10", ";", + "set-option", "-g", "set-titles", "on", ";", + "set-option", "-g", "set-titles-string", "#{pane_title}", ";", + "new-session", "-d", "-s", name, "-c", cwd, "-x", strconv.Itoa(cols), "-y", strconv.Itoa(rows), "--", shell, "-l", ";", + "select-pane", "-t", "="+name+":", "-T", "", + ) + cmd.Dir = cwd + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("tmux new-session: %v: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +// alive reports whether session name still exists. +func (t *terminalTmux) alive(name string) bool { + return t.command("has-session", "-t", "="+name).Run() == nil +} + +// attach is the client command showing session name in the server's +// pseudo-terminal. +func (t *terminalTmux) attach(name string) *exec.Cmd { + return t.command("attach-session", "-t", "="+name) +} + +// kill ends session name and its shell. +func (t *terminalTmux) kill(name string) error { + if out, err := t.command("kill-session", "-t", "="+name).CombinedOutput(); err != nil && t.alive(name) { + return fmt.Errorf("tmux kill-session: %v: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +// terminalEnv is env for a terminal's process: TERM names the emulator the +// clients draw, PWD the start directory (when set), and no TMUX variables +// leak in from a tmux the server runs inside. +func terminalEnv(env []string, cwd string) []string { + out := make([]string, 0, len(env)+2) + for _, entry := range env { + if strings.HasPrefix(entry, "TERM=") || strings.HasPrefix(entry, "TMUX=") || strings.HasPrefix(entry, "TMUX_PANE=") || + (cwd != "" && strings.HasPrefix(entry, "PWD=")) { + continue + } + out = append(out, entry) + } + out = append(out, "TERM=xterm-256color") + if cwd != "" { + out = append(out, "PWD="+cwd) + } + return out +} diff --git a/internal/server/terminal_ws_test.go b/internal/server/terminal_ws_test.go new file mode 100644 index 00000000..5bb4403f --- /dev/null +++ b/internal/server/terminal_ws_test.go @@ -0,0 +1,173 @@ +//go:build darwin || linux + +package server + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/coder/websocket" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// terminalHome points the operator's home at a fresh directory and the +// login shell at /bin/sh, so tests never read the operator's own profile. +func terminalHome(t *testing.T) string { + t.Helper() + home, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + original := operatorHomeDirectory + operatorHomeDirectory = func() (string, error) { return home, nil } + t.Cleanup(func() { operatorHomeDirectory = original }) + t.Setenv("SHELL", "/bin/sh") + t.Setenv("ENV", "") + return home +} + +func terminalRequest(kind, id string, body protocol.TerminalRequest) protocol.Message { + return protocol.Message{Type: kind, ID: id, TerminalRequest: &body} +} + +// terminalReply sends request and returns its reply (skipping pushes). +func terminalReply(t *testing.T, ctx context.Context, conn *websocket.Conn, request protocol.Message) protocol.Message { + t.Helper() + writeChat(t, ctx, conn, request) + return readShellFrame(t, ctx, conn, func(m protocol.Message) bool { return m.ID == request.ID }) +} + +// readTerminalUntil reads output frames of id from end on, in order, until +// the collected text contains want; it returns the text and the new end. +func readTerminalUntil(t *testing.T, ctx context.Context, conn *websocket.Conn, id string, end int64, want string) (string, int64) { + t.Helper() + var seen strings.Builder + for !strings.Contains(seen.String(), want) { + frame := readShellFrame(t, ctx, conn, func(m protocol.Message) bool { + return m.Type == protocol.TerminalEvent && m.TerminalEvent.TerminalID == id && m.TerminalEvent.Terminal == nil + }).TerminalEvent + if frame.Offset != end { + t.Fatalf("frame at %d after %d", frame.Offset, end) + } + seen.WriteString(frame.Output) + end = frame.End + } + return seen.String(), end +} + +// One client opens a shell, types into it, resizes it; a second client +// lists it, resumes its output by offset and closes it. +func TestTerminalOverWebsocket(t *testing.T) { + home := terminalHome(t) + ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 2) + opened := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{Cols: 80, Rows: 24})) + term := opened.Terminal + if opened.Type != protocol.TerminalOpen || term == nil || term.Cwd != home || term.Status != protocol.TerminalRunning || term.Cols != 80 || term.Tmux != "" || term.Shell != "/bin/sh" { + t.Fatalf("open reply = %+v", opened) + } + if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "in", protocol.TerminalRequest{TerminalID: term.ID, Data: "pwd; echo sum-$((40+2))\r"})); got.Type != protocol.TerminalInput { + t.Fatalf("input reply = %+v", got) + } + _, end := readTerminalUntil(t, ctx, conns[0], term.ID, 0, "sum-42") + resized := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalResize, "size", protocol.TerminalRequest{TerminalID: term.ID, Cols: 120, Rows: 33})) + if resized.Type != protocol.TerminalResize || resized.Terminal.Cols != 120 || resized.Terminal.Rows != 33 { + t.Fatalf("resize reply = %+v", resized) + } + list := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalList, "list", protocol.TerminalRequest{})) + if len(list.Terminals) != 1 || list.Terminals[0].ID != term.ID || list.Terminals[0].Cols != 120 || list.Terminals[0].Output != "" { + t.Fatalf("list = %+v", list.Terminals) + } + // The second client reads everything once, then only what follows. + all := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalOutput, "all", protocol.TerminalRequest{TerminalID: term.ID})).Terminal + if all.OutputStart != 0 || !strings.Contains(all.Output, home) || !strings.Contains(all.Output, "sum-42") { + t.Fatalf("output = %+v", all) + } + terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalInput, "in2", protocol.TerminalRequest{TerminalID: term.ID, Data: "stty size\r"})) + readTerminalUntil(t, ctx, conns[0], term.ID, end, "33 120") + resume := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalOutput, "resume", protocol.TerminalRequest{TerminalID: term.ID, After: end})).Terminal + if resume.OutputStart != end || strings.Contains(resume.Output, "sum-42") || !strings.Contains(resume.Output, "33 120") { + t.Fatalf("resume = %+v", resume) + } + closed := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: term.ID})) + if closed.Type != protocol.TerminalClose || closed.Terminal.Status != protocol.TerminalClosed { + t.Fatalf("close reply = %+v", closed) + } + // Every connection hears the close; the list forgets it. + event := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { + return m.Type == protocol.TerminalEvent && m.TerminalEvent.Terminal != nil && m.TerminalEvent.Terminal.Status == protocol.TerminalClosed + }) + if event.TerminalEvent.TerminalID != term.ID { + t.Fatalf("close event = %+v", event) + } + if list := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalList, "list2", protocol.TerminalRequest{})); len(list.Terminals) != 0 { + t.Fatalf("list after close = %+v", list.Terminals) + } + if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "late", protocol.TerminalRequest{TerminalID: term.ID, Data: "x"})); got.Type != protocol.Error { + t.Fatalf("late input = %+v", got) + } +} + +// A shell that exits by itself is listed as exited with its status and +// keeps its output until it is closed. +func TestTerminalShellExit(t *testing.T) { + terminalHome(t) + ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 1) + term := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{})).Terminal + terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "in", protocol.TerminalRequest{TerminalID: term.ID, Data: "echo bye; exit 3\r"})) + exited := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool { + return m.Type == protocol.TerminalEvent && m.TerminalEvent.Terminal != nil && m.TerminalEvent.Terminal.Status == protocol.TerminalExited + }).TerminalEvent.Terminal + if exited.ExitCode == nil || *exited.ExitCode != 3 || exited.EndedAt == 0 { + t.Fatalf("exited = %+v", exited) + } + out := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOutput, "out", protocol.TerminalRequest{TerminalID: term.ID})).Terminal + if !strings.Contains(out.Output, "bye") || out.Status != protocol.TerminalExited { + t.Fatalf("output = %+v", out) + } + if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: term.ID})); got.Terminal == nil || got.Terminal.Status != protocol.TerminalClosed { + t.Fatalf("close = %+v", got) + } +} + +// Requests outside the bounds are refused: directories outside home or +// missing, input bounds, unknown terminals, the open limit. +func TestTerminalLimitsAndValidation(t *testing.T) { + home := terminalHome(t) + ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 1) + conn := conns[0] + for i, cwd := range []string{"/", filepath.Join(home, "missing"), "relative", "~other"} { + if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "bad", protocol.TerminalRequest{Cwd: cwd})); got.Type != protocol.Error { + t.Fatalf("cwd %d %q = %+v", i, cwd, got) + } + } + if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOutput, "none", protocol.TerminalRequest{TerminalID: "term_nope"})); got.Type != protocol.Error { + t.Fatalf("unknown = %+v", got) + } + if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalInput, "noid", protocol.TerminalRequest{Data: "x"})); got.Type != protocol.Error { + t.Fatalf("no id = %+v", got) + } + var ids []string + for i := 0; i < protocol.TerminalOpenLimit; i++ { + got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{Cwd: "~"})) + if got.Terminal == nil { + t.Fatalf("open %d = %+v", i, got) + } + ids = append(ids, got.Terminal.ID) + } + over := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "over", protocol.TerminalRequest{})) + if over.Type != protocol.Error || !strings.Contains(over.Error, "8 open terminals") { + t.Fatalf("over limit = %+v", over) + } + for _, bad := range []string{"", strings.Repeat("x", protocol.ChatShellInputLimit+1)} { + if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalInput, "bad-in", protocol.TerminalRequest{TerminalID: ids[0], Data: bad})); got.Type != protocol.Error { + t.Fatalf("input %q = %+v", bad[:min(len(bad), 4)], got) + } + } + terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: ids[0]})) + if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "again", protocol.TerminalRequest{})); got.Terminal == nil { + t.Fatalf("open after close = %+v", got) + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index fec278bf..06d619b6 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -66,6 +66,12 @@ type Options struct { // False keeps the previous behaviour (tests, embedded servers): a stopping // server interrupts its runs and the next start marks them interrupted. DetachSessions bool + // TerminalTmuxSocket names the Pipelines-owned tmux server machine + // terminals live in (empty: "pipelines"); TerminalWithoutTmux runs + // shells as direct children even where tmux is installed. Tests set + // them so they never share the real server's tmux sessions. + TerminalTmuxSocket string + TerminalWithoutTmux bool } // Handler serves the persistent connection with default options. Canceling ctx @@ -143,6 +149,20 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } else if store, ok := registry.(chatShellRunStore); ok { hub.startShellRunner(ctx, store) } + // Machine terminals are recorded beside the operator sessions, in the + // same store; a previous process's tmux-backed shells are reattached. + var terminalRecords terminalStore + if len(stores) > 0 { + terminalRecords = stores[0] + } else if store, ok := registry.(terminalStore); ok { + terminalRecords = store + } + var terminalTmuxServer *terminalTmux + if !options.TerminalWithoutTmux { + terminalTmuxServer = findTerminalTmux(options.TerminalTmuxSocket) + } + terminals := newTerminalManager(ctx, terminalRecords, terminalTmuxServer) + terminals.recover(ctx) acks := newAttentionAckHub() var ackStore attentionAckStore var notes *appNotifier @@ -237,6 +257,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin // connection the same way. hub.sessionChanges.attach(clientID, write) defer hub.sessionChanges.detach(clientID) + // Terminal lifecycle changes reach every connection; output only the + // connections that opened or read that terminal. + terminals.clients.attach(clientID, write) + defer terminals.clients.detach(clientID) if notes != nil { defer notes.detach(clientID) } @@ -250,7 +274,8 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if err := wsjson.Read(connCtx, conn, &request); err != nil { return } - if request.Type != protocol.ChatShellInput && request.Type != protocol.ChatShellResize { + if request.Type != protocol.ChatShellInput && request.Type != protocol.ChatShellResize && + request.Type != protocol.TerminalInput && request.Type != protocol.TerminalResize { // Keystrokes would be one line each; their data is never logged. log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version) } @@ -291,6 +316,16 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } continue } + if isTerminalRequest(request.Type) { + // Closing waits for the shell to go; it never delays input + // to the other terminals behind it. + if request.Type == protocol.TerminalClose { + go handleTerminalRequest(connCtx, request, terminals, clientID, write) + } else { + handleTerminalRequest(connCtx, request, terminals, clientID, write) + } + continue + } if handleDirectoryListing(request, write) { continue }