Add persistent machine terminals with tmux-backed reattach

This commit is contained in:
2026-11-09 21:30:15 +00:00
parent 8f06f12d49
commit 277384c864
16 changed files with 1757 additions and 1 deletions
+4
View File
@@ -109,6 +109,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
db.Close()
return nil, err
}
if err := s.initTerminals(); err != nil {
db.Close()
return nil, err
}
if err := s.initAppNotifications(); err != nil {
db.Close()
return nil, err
+88
View File
@@ -0,0 +1,88 @@
package pipeline
import (
"context"
"database/sql"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// initTerminals creates the additive table of machine terminals. It records
// only what a restarted server needs to list and reattach them (id, title,
// cwd, shell, tmux session, status). Output and typed input are never stored.
func (s *Store) initTerminals() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS terminals (
id TEXT PRIMARY KEY,
title TEXT NOT NULL DEFAULT '',
cwd TEXT NOT NULL,
shell TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL,
tmux TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL,
ended_at INTEGER NOT NULL DEFAULT 0,
exit_code INTEGER,
reason TEXT NOT NULL DEFAULT '',
cols INTEGER NOT NULL DEFAULT 0,
rows INTEGER NOT NULL DEFAULT 0);`)
return err
}
// SaveTerminal inserts or replaces one terminal. Saving an ended terminal
// prunes the oldest ended ones beyond keep (keep <= 0 keeps everything).
func (s *Store) SaveTerminal(ctx context.Context, t protocol.TerminalRecord, keep int) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
var exit any
if t.ExitCode != nil {
exit = *t.ExitCode
}
if _, err := tx.ExecContext(ctx, `INSERT INTO terminals
(id, title, cwd, shell, created_at, tmux, status, ended_at, exit_code, reason, cols, rows)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET title = excluded.title, status = excluded.status, ended_at = excluded.ended_at,
exit_code = excluded.exit_code, reason = excluded.reason, cols = excluded.cols, rows = excluded.rows`,
t.ID, t.Title, t.Cwd, t.Shell, t.CreatedAt, t.Tmux, t.Status, t.EndedAt, exit, t.Reason, t.Cols, t.Rows); err != nil {
return err
}
if keep > 0 && t.Status != protocol.TerminalRunning {
if _, err := tx.ExecContext(ctx, `DELETE FROM terminals WHERE status != ? AND id NOT IN
(SELECT id FROM terminals WHERE status != ? ORDER BY ended_at DESC, id DESC LIMIT ?)`,
protocol.TerminalRunning, protocol.TerminalRunning, keep); err != nil {
return err
}
}
return tx.Commit()
}
// DeleteTerminal removes one terminal's record.
func (s *Store) DeleteTerminal(ctx context.Context, id string) error {
_, err := s.db.ExecContext(ctx, `DELETE FROM terminals WHERE id = ?`, id)
return err
}
// Terminals lists every recorded terminal, oldest first.
func (s *Store) Terminals(ctx context.Context) ([]protocol.TerminalRecord, error) {
rows, err := s.db.QueryContext(ctx, `SELECT id, title, cwd, shell, created_at, tmux, status, ended_at, exit_code, reason, cols, rows
FROM terminals ORDER BY created_at, id`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []protocol.TerminalRecord
for rows.Next() {
var t protocol.TerminalRecord
var exit sql.NullInt64
if err := rows.Scan(&t.ID, &t.Title, &t.Cwd, &t.Shell, &t.CreatedAt, &t.Tmux, &t.Status, &t.EndedAt, &exit, &t.Reason, &t.Cols, &t.Rows); err != nil {
return nil, err
}
if exit.Valid {
code := int(exit.Int64)
t.ExitCode = &code
}
out = append(out, t)
}
return out, rows.Err()
}
+54
View File
@@ -0,0 +1,54 @@
package pipeline
import (
"context"
"fmt"
"path/filepath"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func TestTerminalsPersistPruneAndDelete(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "terminals.db")
store, err := Open(path)
if err != nil {
t.Fatal(err)
}
live := protocol.TerminalRecord{ID: "term_live", Cwd: "/home/a", Shell: "/bin/zsh", CreatedAt: 1, Tmux: "pipelines-term_live", Status: protocol.TerminalRunning, Cols: 80, Rows: 24}
if err := store.SaveTerminal(ctx, live, 2); err != nil {
t.Fatal(err)
}
code := 0
for i := 0; i < 3; i++ {
ended := protocol.TerminalRecord{ID: fmt.Sprintf("term_%d", i), Cwd: "/home/a", CreatedAt: int64(10 + i), Status: protocol.TerminalExited, EndedAt: int64(20 + i), ExitCode: &code}
if err := store.SaveTerminal(ctx, ended, 2); err != nil {
t.Fatal(err)
}
}
live.Title, live.Cols = "vim", 120
if err := store.SaveTerminal(ctx, live, 2); err != nil {
t.Fatal(err)
}
store.Close()
store, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer store.Close()
list, err := store.Terminals(ctx)
if err != nil {
t.Fatal(err)
}
if len(list) != 3 || list[0].ID != "term_live" || list[0].Title != "vim" || list[0].Cols != 120 || list[0].Tmux != "pipelines-term_live" ||
list[1].ID != "term_1" || list[2].ID != "term_2" || list[2].ExitCode == nil || *list[2].ExitCode != 0 {
t.Fatalf("terminals = %+v", list)
}
if err := store.DeleteTerminal(ctx, "term_live"); err != nil {
t.Fatal(err)
}
if list, _ := store.Terminals(ctx); len(list) != 2 {
t.Fatalf("after delete = %+v", list)
}
}
+6
View File
@@ -280,6 +280,12 @@ type Message struct {
ChatShellRun *ChatShellRunRecord `json:"chat_shell_run,omitempty"`
ChatShellRuns []ChatShellRunRecord `json:"chat_shell_runs,omitempty"`
ChatShellRunEvent *ChatShellRunEventFrame `json:"chat_shell_run_event,omitempty"`
// Machine terminals (terminal.go): the request body, one terminal, the
// machine's terminals and the pushed change.
TerminalRequest *TerminalRequest `json:"terminal_request,omitempty"`
Terminal *TerminalRecord `json:"terminal,omitempty"`
Terminals []TerminalRecord `json:"terminals,omitempty"`
TerminalEvent *TerminalEventFrame `json:"terminal_event,omitempty"`
// ListDirectory is the list_directory body and DirectoryListing its reply:
// the home-restricted folder picker listing (see directories.go).
ListDirectory *DirectoryListRequest `json:"list_directory,omitempty"`
+110
View File
@@ -0,0 +1,110 @@
package protocol
// Machine terminals: interactive login shells the operator opens on a
// server's machine, independent of any chat. Each runs `$SHELL -l` (else
// /bin/sh) in a pseudo-terminal; where tmux is installed the shell lives in
// a Pipelines-owned tmux server (socket name "pipelines", never the
// operator's own), so it survives a Pipelines server restart and the next
// process reattaches it. Without tmux a shell ends with the server and is
// listed as ended. Typed input is never logged or stored.
const (
// TerminalOpen starts a shell. The body is TerminalRequest (cwd, cols,
// rows); cwd defaults to the operator's home and must be an existing
// directory inside it. The reply carries the new terminal; the
// connection is then subscribed to its terminal_event output frames.
TerminalOpen = "terminal_open"
// TerminalList lists this machine's terminals (running and ended),
// oldest first, without output. The body is optional.
TerminalList = "terminal_list"
// TerminalInput writes Data (UTF-8, control characters included, at most
// ChatShellInputLimit bytes) to a running terminal. No reply body.
TerminalInput = "terminal_input"
// TerminalResize sets a running terminal's size (cols, rows); the reply
// echoes the terminal.
TerminalResize = "terminal_resize"
// TerminalOutput reads one terminal with its retained output from the
// absolute byte After on (all of it when After precedes the retained
// tail; OutputStart says where it starts). It also subscribes the
// connection to the terminal's terminal_event output frames.
TerminalOutput = "terminal_output"
// TerminalClose ends a terminal: its shell (and tmux session) is killed
// and its record removed. The reply echoes the last record.
TerminalClose = "terminal_close"
// TerminalEvent is pushed: an output chunk to the connections subscribed
// to that terminal (Output, Offset, End), or a lifecycle change (Terminal
// set: opened, title, exited, closed) to every connection.
TerminalEvent = "terminal_event"
)
// Terminal status values. Running is the only live status.
const (
TerminalRunning = "running"
TerminalExited = "exited"
TerminalClosed = "closed"
)
// Terminal bounds.
const (
// TerminalOpenLimit bounds the running terminals of one machine.
TerminalOpenLimit = 8
// TerminalRetained is the output tail kept for reconnecting clients.
TerminalRetained = 1 << 20
// TerminalChunkLimit bounds one live output frame.
TerminalChunkLimit = 64 << 10
// TerminalEndedKept bounds the ended terminals still listed.
TerminalEndedKept = 16
// TerminalTmuxHistory is the tmux scrollback of one tmux-backed shell.
TerminalTmuxHistory = 10000
// TerminalTitleLimit bounds a recorded title.
TerminalTitleLimit = 200
)
// TerminalRequest is the body of every terminal request.
type TerminalRequest struct {
TerminalID string `json:"terminal_id,omitempty"`
Cwd string `json:"cwd,omitempty"`
Cols int `json:"cols,omitempty"`
Rows int `json:"rows,omitempty"`
Data string `json:"data,omitempty"`
After int64 `json:"after,omitempty"`
}
// TerminalRecord is one terminal. Output and OutputStart are set only on
// terminal_output replies.
type TerminalRecord struct {
ID string `json:"id"`
// Title is the last terminal title (OSC 0/2) the shell set, or empty;
// clients show the cwd then.
Title string `json:"title,omitempty"`
Cwd string `json:"cwd"`
Shell string `json:"shell"`
CreatedAt int64 `json:"created_at"`
// Tmux is the tmux session name keeping the shell alive across server
// restarts; empty when the shell is a direct child of the server and
// ends with it.
Tmux string `json:"tmux,omitempty"`
Status string `json:"status"`
EndedAt int64 `json:"ended_at,omitempty"`
ExitCode *int `json:"exit_code,omitempty"`
// Reason says why an ended terminal ended when it was not simply its
// shell exiting (the server restarted without tmux, its tmux session
// was gone).
Reason string `json:"reason,omitempty"`
Cols int `json:"cols"`
Rows int `json:"rows"`
// OutputEnd is the absolute byte count this server process has read;
// it restarts at zero when a restarted server reattaches.
OutputEnd int64 `json:"output_end"`
Output string `json:"output,omitempty"`
OutputStart int64 `json:"output_start,omitempty"`
}
// TerminalEventFrame is one pushed terminal change: either Terminal (a
// lifecycle change) or one output chunk covering bytes [Offset, End).
type TerminalEventFrame struct {
TerminalID string `json:"terminal_id"`
Terminal *TerminalRecord `json:"terminal,omitempty"`
Output string `json:"output,omitempty"`
Offset int64 `json:"offset,omitempty"`
End int64 `json:"end,omitempty"`
}
+91
View File
@@ -0,0 +1,91 @@
package server
import (
"sync"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// terminalClientQueue bounds the frames waiting for one slow connection.
// A full queue drops output frames; the client sees the gap in the next
// frame's offset and reads the retained tail again (terminal_output after).
const terminalClientQueue = 256
// terminalClients delivers terminal_event frames to open connections, in
// order per connection: lifecycle changes to every connection, output only
// to connections subscribed to that terminal (by terminal_open or
// terminal_output). It holds each connection's serialized write function,
// never the socket. The zero value is ready to use.
type terminalClients struct {
mu sync.Mutex
clients map[string]*terminalClient
}
type terminalClient struct {
ch chan protocol.Message
done chan struct{}
subs map[string]bool
}
func (c *terminalClients) attach(clientID string, write func(protocol.Message) error) {
client := &terminalClient{ch: make(chan protocol.Message, terminalClientQueue), done: make(chan struct{}), subs: map[string]bool{}}
c.mu.Lock()
if c.clients == nil {
c.clients = map[string]*terminalClient{}
}
if old := c.clients[clientID]; old != nil {
close(old.done)
}
c.clients[clientID] = client
c.mu.Unlock()
go func() {
for {
select {
case <-client.done:
return
case msg := <-client.ch:
if write(msg) != nil {
return
}
}
}
}()
}
func (c *terminalClients) detach(clientID string) {
c.mu.Lock()
defer c.mu.Unlock()
if client := c.clients[clientID]; client != nil {
close(client.done)
delete(c.clients, clientID)
}
}
// subscribe sends terminalID's output frames to clientID from now on.
func (c *terminalClients) subscribe(clientID, terminalID string) {
c.mu.Lock()
defer c.mu.Unlock()
if client := c.clients[clientID]; client != nil {
client.subs[terminalID] = true
}
}
// publish queues frame for its recipients without blocking: a lifecycle
// change for everyone, output for the terminal's subscribers.
func (c *terminalClients) publish(frame protocol.TerminalEventFrame) {
msg := protocol.Message{Type: protocol.TerminalEvent, TerminalEvent: &frame}
c.mu.Lock()
defer c.mu.Unlock()
for _, client := range c.clients {
if frame.Terminal == nil && !client.subs[frame.TerminalID] {
continue
}
if frame.Terminal != nil && frame.Terminal.Status == protocol.TerminalClosed {
delete(client.subs, frame.TerminalID)
}
select {
case client.ch <- msg:
default:
}
}
}
+41
View File
@@ -0,0 +1,41 @@
package server
import (
"errors"
"os"
"path/filepath"
)
// resolveTerminalCwd is the directory a terminal starts in: requested ("",
// "~", "~/sub" or an absolute path) must be an existing directory inside
// home once every symlink is resolved, the containment the Files panel and
// folder picker apply. It returns the path as the operator sees it, under
// home.
func resolveTerminalCwd(home, requested string) (string, error) {
home = filepath.Clean(home)
realHome, err := filepath.EvalSymlinks(home)
if err != nil {
return "", errors.New("resolve home directory: " + err.Error())
}
target, err := expandHomePath(home, requested)
if err != nil {
return "", err
}
realTarget, err := filepath.EvalSymlinks(target)
if err != nil {
return "", describePathError(target, err)
}
rel, inside := relInside(realHome, realTarget)
if !inside {
return "", errors.New(target + " is outside the home directory " + home + "; a terminal starts only in a folder inside home")
}
shown := filepath.Join(home, rel)
info, err := os.Stat(realTarget)
if err != nil {
return "", describePathError(shown, err)
}
if !info.IsDir() {
return "", errors.New(shown + " is not a directory")
}
return shown, nil
}
+65
View File
@@ -0,0 +1,65 @@
package server
import (
"context"
"errors"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// isTerminalRequest reports the request types handleTerminalRequest answers.
func isTerminalRequest(kind string) bool {
switch kind {
case protocol.TerminalOpen, protocol.TerminalList, protocol.TerminalInput, protocol.TerminalResize,
protocol.TerminalOutput, protocol.TerminalClose:
return true
}
return false
}
// handleTerminalRequest answers one machine-terminal request from an
// authenticated connection. Opening or reading a terminal subscribes the
// connection to its output frames.
func handleTerminalRequest(ctx context.Context, request protocol.Message, m *terminalManager, clientID string, write func(protocol.Message) error) {
fail := func(err error) { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) }
if m == nil {
fail(errors.New("terminals are not configured on this server"))
return
}
body := request.TerminalRequest
if body == nil {
body = &protocol.TerminalRequest{}
}
if request.Type != protocol.TerminalOpen && request.Type != protocol.TerminalList && body.TerminalID == "" {
fail(errors.New("a terminal_id is required"))
return
}
reply := protocol.Message{Type: request.Type, ID: request.ID}
var record protocol.TerminalRecord
var err error
switch request.Type {
case protocol.TerminalOpen:
record, err = m.Open(ctx, clientID, body.Cwd, body.Cols, body.Rows)
case protocol.TerminalList:
var records []protocol.TerminalRecord
records, err = m.List(ctx)
reply.Terminals = records
case protocol.TerminalInput:
err = m.Input(body.TerminalID, body.Data)
case protocol.TerminalResize:
record, err = m.Resize(body.TerminalID, body.Cols, body.Rows)
case protocol.TerminalOutput:
m.clients.subscribe(clientID, body.TerminalID)
record, err = m.Output(ctx, body.TerminalID, body.After)
case protocol.TerminalClose:
record, err = m.Close(ctx, body.TerminalID)
}
if err != nil {
fail(err)
return
}
if record.ID != "" {
reply.Terminal = &record
}
_ = write(reply)
}
+314
View File
@@ -0,0 +1,314 @@
package server
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"log"
"os"
"os/exec"
"syscall"
"sync"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// terminalStore persists the machine's terminal list; *pipeline.Store
// implements it.
type terminalStore interface {
SaveTerminal(ctx context.Context, terminal protocol.TerminalRecord, keep int) error
DeleteTerminal(ctx context.Context, id string) error
Terminals(ctx context.Context) ([]protocol.TerminalRecord, error)
}
const (
terminalNoTmuxReason = "The Pipelines server restarted and this shell ended with it (tmux is not installed on this machine, so shells do not outlive the server)."
terminalGoneReason = "Its tmux session ended while the Pipelines server was stopped."
terminalStopReason = "The Pipelines server stopped and this shell ended with it (tmux is not installed on this machine)."
// terminalReattachLimit bounds how often a tmux client that exits while
// its session lives is replaced before the terminal is given up.
terminalReattachLimit = 5
)
// terminalManager owns the machine's terminals (terminal.go in protocol).
type terminalManager struct {
base context.Context
store terminalStore
clients *terminalClients
tmux *terminalTmux
shell func() string
home func() (string, error)
now func() time.Time
flush time.Duration
mu sync.Mutex
terms map[string]*terminalProcess
wg sync.WaitGroup
}
func newTerminalManager(base context.Context, store terminalStore, tmux *terminalTmux) *terminalManager {
return &terminalManager{
base: base, store: store, clients: &terminalClients{}, tmux: tmux,
shell: chatShellProgram, home: OperatorHomeDirectory, now: time.Now, flush: 50 * time.Millisecond,
terms: map[string]*terminalProcess{},
}
}
func newTerminalID() (string, error) {
var b [8]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
return "term_" + hex.EncodeToString(b[:]), nil
}
// Open starts a login shell in cwd at cols x rows and returns its record.
// clientID (when set) is subscribed to its output before the shell writes.
func (m *terminalManager) Open(ctx context.Context, clientID, cwd string, cols, rows int) (protocol.TerminalRecord, error) {
if m.store == nil {
return protocol.TerminalRecord{}, errors.New("terminals need the server's store, which is not configured")
}
if !chatShellTerminals {
return protocol.TerminalRecord{}, errors.New(protocol.ChatShellTerminalUnsupported)
}
home, err := m.home()
if err != nil {
return protocol.TerminalRecord{}, err
}
dir, err := resolveTerminalCwd(home, cwd)
if err != nil {
return protocol.TerminalRecord{}, err
}
id, err := newTerminalID()
if err != nil {
return protocol.TerminalRecord{}, err
}
m.mu.Lock()
defer m.mu.Unlock()
if m.base.Err() != nil {
return protocol.TerminalRecord{}, errors.New("the server is stopping")
}
if running := m.runningLocked(); running >= protocol.TerminalOpenLimit {
return protocol.TerminalRecord{}, fmt.Errorf("this machine already has %d open terminals; close one first", running)
}
cols, rows = chatShellTerminalSize(cols, rows)
shell := m.shell()
record := protocol.TerminalRecord{ID: id, Cwd: dir, Shell: shell, CreatedAt: m.now().UnixMilli(), Status: protocol.TerminalRunning, Cols: cols, Rows: rows}
var cmd *exec.Cmd
if m.tmux != nil {
record.Tmux = terminalTmuxName(id)
if err := m.tmux.create(record.Tmux, dir, shell, cols, rows); err != nil {
return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err)
}
cmd = m.tmux.attach(record.Tmux)
} else {
cmd = exec.Command(shell, "-l")
cmd.Env = terminalEnv(os.Environ(), dir)
}
cmd.Dir = dir
p := newTerminalProcess(record)
terminal, err := p.startAttachment(cmd)
if err == nil {
err = m.store.SaveTerminal(ctx, record, protocol.TerminalEndedKept)
if err != nil {
p.signal(syscall.SIGKILL)
terminal.Close()
_ = cmd.Wait()
}
}
if err != nil {
if record.Tmux != "" {
_ = m.tmux.kill(record.Tmux)
}
return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err)
}
m.terms[id] = p
if clientID != "" {
m.clients.subscribe(clientID, id)
}
m.wg.Add(1)
go m.supervise(p, cmd, terminal)
go p.pumpInput()
m.publishRecord(record)
return record, nil
}
func (m *terminalManager) runningLocked() int {
running := 0
for _, p := range m.terms {
if p.running() {
running++
}
}
return running
}
// List is every recorded terminal, oldest first, with the live state of the
// ones this process holds.
func (m *terminalManager) List(ctx context.Context) ([]protocol.TerminalRecord, error) {
if m.store == nil {
return nil, errors.New("terminals need the server's store, which is not configured")
}
records, err := m.store.Terminals(ctx)
if err != nil {
return nil, err
}
for i, record := range records {
if p := m.process(record.ID); p != nil {
records[i] = p.snapshot(false, 0)
}
}
return records, nil
}
func (m *terminalManager) process(id string) *terminalProcess {
m.mu.Lock()
defer m.mu.Unlock()
return m.terms[id]
}
func (m *terminalManager) live(id string) (*terminalProcess, error) {
p := m.process(id)
if p == nil || !p.running() {
return nil, errors.New("this terminal is not running")
}
return p, nil
}
// Input queues data for a running terminal, in arrival order. It is kept
// nowhere else: what the shell echoes is output.
func (m *terminalManager) Input(id, data string) error {
switch {
case data == "":
return errors.New("input is required")
case len(data) > protocol.ChatShellInputLimit:
return errors.New("the input exceeds 16 KiB; send it in parts")
case !utf8.ValidString(data):
return errors.New("input must be UTF-8 text")
}
p, err := m.live(id)
if err != nil {
return err
}
select {
case <-p.done:
return errors.New("this terminal is not running")
case p.input <- []byte(data):
return nil
default:
return errors.New("the terminal is not reading its input; wait for it or press Ctrl-C")
}
}
// Resize sets a running terminal's size and returns its record.
func (m *terminalManager) Resize(id string, cols, rows int) (protocol.TerminalRecord, error) {
if cols <= 0 || rows <= 0 {
return protocol.TerminalRecord{}, errors.New("positive cols and rows are required")
}
p, err := m.live(id)
if err != nil {
return protocol.TerminalRecord{}, err
}
cols, rows = chatShellTerminalSize(cols, rows)
p.mu.Lock()
p.record.Cols, p.record.Rows = cols, rows
terminal := p.pty
p.mu.Unlock()
if terminal != nil {
if err := resizeChatShellTerminal(terminal, cols, rows); err != nil {
return protocol.TerminalRecord{}, err
}
}
return p.snapshot(false, 0), nil
}
// Output is one terminal with its retained output from byte after on. A
// terminal ended before this process started has no output.
func (m *terminalManager) Output(ctx context.Context, id string, after int64) (protocol.TerminalRecord, error) {
if p := m.process(id); p != nil {
return p.snapshot(true, after), nil
}
records, err := m.List(ctx)
if err != nil {
return protocol.TerminalRecord{}, err
}
for _, record := range records {
if record.ID == id {
return record, nil
}
}
return protocol.TerminalRecord{}, errors.New("this machine has no such terminal")
}
// Close ends a terminal (killing its shell and tmux session) and forgets
// it. It waits briefly for the shell to go and returns the closed record.
func (m *terminalManager) Close(ctx context.Context, id string) (protocol.TerminalRecord, error) {
if m.store == nil {
return protocol.TerminalRecord{}, errors.New("terminals need the server's store, which is not configured")
}
p := m.process(id)
if p != nil && p.running() {
p.mu.Lock()
p.closing = true
name := p.record.Tmux
p.mu.Unlock()
if name != "" && m.tmux != nil {
if err := m.tmux.kill(name); err != nil {
log.Printf("terminal close %s: %v", id, err)
}
}
p.signal(syscall.SIGHUP)
select {
case <-p.done:
case <-time.After(3 * time.Second):
p.signal(syscall.SIGKILL)
<-p.done
}
return p.snapshot(false, 0), nil
}
record, err := m.Output(ctx, id, 0)
if err != nil {
return protocol.TerminalRecord{}, err
}
if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) {
_ = m.tmux.kill(record.Tmux)
}
m.forget(id)
record.Output, record.OutputStart = "", 0
record.Status = protocol.TerminalClosed
m.publishRecord(record)
return record, nil
}
// forget drops a terminal's record and retained output.
func (m *terminalManager) forget(id string) {
m.mu.Lock()
delete(m.terms, id)
m.mu.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := m.store.DeleteTerminal(ctx, id); err != nil {
log.Printf("terminal delete %s: %v", id, err)
}
}
func (m *terminalManager) save(record protocol.TerminalRecord) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := m.store.SaveTerminal(ctx, record, protocol.TerminalEndedKept); err != nil {
log.Printf("terminal save %s: %v", record.ID, err)
}
}
func (m *terminalManager) publishRecord(record protocol.TerminalRecord) {
record.Output, record.OutputStart = "", 0
m.clients.publish(protocol.TerminalEventFrame{TerminalID: record.ID, Terminal: &record})
}
// Wait blocks until every supervised terminal has been released (tests,
// shutdown).
func (m *terminalManager) Wait() { m.wg.Wait() }
+173
View File
@@ -0,0 +1,173 @@
package server
import (
"errors"
"os"
"os/exec"
"sync"
"syscall"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// terminalProcess is one terminal this server process holds: running (its
// attachment is the shell itself, or a tmux client showing the shell's
// session) or ended with its output still retained.
type terminalProcess struct {
mu sync.Mutex
record protocol.TerminalRecord
out *chatShellOutput
titles terminalTitles
// cmd and pty are the current attachment; a tmux client that exits while
// its session lives is replaced.
cmd *exec.Cmd
pty *os.File
input chan []byte
done chan struct{}
// closing is set by terminal_close: the end is a close, not an exit.
closing bool
}
func newTerminalProcess(record protocol.TerminalRecord) *terminalProcess {
return &terminalProcess{
record: record,
out: newChatShellOutput(protocol.TerminalRetained),
input: make(chan []byte, protocol.ChatShellInputQueue),
done: make(chan struct{}),
}
}
func (p *terminalProcess) running() bool {
p.mu.Lock()
defer p.mu.Unlock()
return p.record.Status == protocol.TerminalRunning
}
// snapshot is the current record; with output, the retained bytes from
// absolute byte after on.
func (p *terminalProcess) snapshot(output bool, after int64) protocol.TerminalRecord {
p.mu.Lock()
defer p.mu.Unlock()
record := p.record
record.OutputEnd = p.out.total
if output {
text, start := p.out.tail(protocol.TerminalRetained)
if after > start {
skip := min(after-start, int64(len(text)))
cut := runeStartAfter([]byte(text), int(skip))
text, start = text[cut:], start+int64(cut)
}
record.Output, record.OutputStart = text, start
}
return record
}
// startAttachment starts cmd in a fresh pseudo-terminal at the record's
// size and makes it the current attachment.
func (p *terminalProcess) startAttachment(cmd *exec.Cmd) (*os.File, error) {
if !chatShellTerminals {
return nil, errors.New(protocol.ChatShellTerminalUnsupported)
}
p.mu.Lock()
cols, rows := p.record.Cols, p.record.Rows
p.mu.Unlock()
terminal, err := startChatShellTerminal(cmd, cols, rows)
if err != nil {
return nil, err
}
p.mu.Lock()
p.cmd, p.pty = cmd, terminal
p.mu.Unlock()
return terminal, nil
}
// read copies one attachment's stream into the retained output and follows
// the window title; it returns when the attachment's terminal is closed.
func (p *terminalProcess) read(terminal *os.File, titled func(string)) {
buffer := make([]byte, 32<<10)
for {
n, err := terminal.Read(buffer)
if n > 0 {
p.mu.Lock()
p.out.write(buffer[:n])
title, changed := p.titles.feed(buffer[:n])
changed = changed && title != p.record.Title
if changed {
p.record.Title = title
}
p.mu.Unlock()
if changed {
titled(title)
}
}
if err != nil {
return
}
}
}
// pumpInput writes queued input to the current attachment until the
// terminal ends. Input arriving while a tmux client is replaced is dropped.
func (p *terminalProcess) pumpInput() {
for {
select {
case <-p.done:
return
case data := <-p.input:
p.mu.Lock()
terminal := p.pty
p.mu.Unlock()
if terminal != nil {
_, _ = terminal.Write(data)
}
}
}
}
// signal sends sig to the current attachment's process group.
func (p *terminalProcess) signal(sig syscall.Signal) {
p.mu.Lock()
cmd := p.cmd
p.mu.Unlock()
if cmd != nil && cmd.Process != nil {
_ = syscall.Kill(-cmd.Process.Pid, sig)
}
}
// waitAttachment waits for the attachment to exit and its stream to drain
// (a descendant holding the terminal is read briefly, then cut off), then
// kills its group and closes its terminal. It returns the exit error.
func (p *terminalProcess) waitAttachment(cmd *exec.Cmd, terminal *os.File, readDone <-chan struct{}) error {
err := cmd.Wait()
_ = terminal.SetReadDeadline(time.Now().Add(chatShellDrain))
select {
case <-readDone:
case <-time.After(chatShellDrain + chatShellDrain/2):
}
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
p.mu.Lock()
terminal.Close()
if p.pty == terminal {
p.pty = nil
}
p.mu.Unlock()
return err
}
// exitCode is the shell's exit status, when it exited by itself.
func terminalExitCode(err error) *int {
code := 0
var exitErr *exec.ExitError
switch {
case err == nil:
case errors.As(err, &exitErr):
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() {
return nil
}
code = exitErr.ExitCode()
default:
return nil
}
return &code
}
+223
View File
@@ -0,0 +1,223 @@
//go:build darwin || linux
package server
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// testTmux is a tmux server of this test alone, killed afterwards.
func testTmux(t *testing.T) *terminalTmux {
t.Helper()
tmux := findTerminalTmux(fmt.Sprintf("pipelines-test-%d-%d", os.Getpid(), time.Now().UnixNano()))
if tmux == nil {
t.Skip("tmux is not installed")
}
t.Cleanup(func() {
_ = tmux.command("kill-server").Run()
dir := os.Getenv("TMUX_TMPDIR")
if dir == "" {
dir = "/tmp"
}
_ = os.Remove(filepath.Join(dir, fmt.Sprintf("tmux-%d", os.Getuid()), tmux.socket))
})
return tmux
}
// newTestTerminals is one server process's terminal manager over store.
func newTestTerminals(t *testing.T, store *pipeline.Store, tmux *terminalTmux) (*terminalManager, context.CancelFunc) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
m := newTerminalManager(ctx, store, tmux)
m.shell = func() string { return "/bin/sh" }
m.recover(ctx)
t.Cleanup(func() { cancel(); m.Wait() })
return m, cancel
}
// waitOutput polls a terminal's retained output until it contains want.
func waitTerminalOutput(t *testing.T, m *terminalManager, id, want string) protocol.TerminalRecord {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
record, err := m.Output(context.Background(), id, 0)
if err != nil {
t.Fatal(err)
}
if strings.Contains(record.Output, want) {
return record
}
if time.Now().After(deadline) {
t.Fatalf("output of %s never contained %q: %q", id, want, record.Output)
}
time.Sleep(50 * time.Millisecond)
}
}
// A tmux-backed shell outlives its server process: the next one lists it as
// running, reattaches, redraws its screen and takes input again. Closing it
// ends the tmux session.
func TestTerminalTmuxSurvivesRestart(t *testing.T) {
terminalHome(t)
tmux := testTmux(t)
store := newOperatorStore(t)
first, stop := newTestTerminals(t, store, tmux)
term, err := first.Open(context.Background(), "", "~", 90, 20)
if err != nil {
t.Fatal(err)
}
if term.Tmux != terminalTmuxName(term.ID) {
t.Fatalf("tmux name = %q", term.Tmux)
}
if err := first.Input(term.ID, "export KEEP=still-here; echo marker-$((6*7))\r"); err != nil {
t.Fatal(err)
}
waitTerminalOutput(t, first, term.ID, "marker-42")
stop()
first.Wait()
if !tmux.alive(term.Tmux) {
t.Fatal("the tmux session ended with the server")
}
second, _ := newTestTerminals(t, store, tmux)
list, err := second.List(context.Background())
if err != nil {
t.Fatal(err)
}
if len(list) != 1 || list[0].ID != term.ID || list[0].Status != protocol.TerminalRunning {
t.Fatalf("list after restart = %+v", list)
}
// The reattached client redraws the screen, earlier output included,
// and the shell keeps its state.
waitTerminalOutput(t, second, term.ID, "marker-42")
if err := second.Input(term.ID, "echo $KEEP\r"); err != nil {
t.Fatal(err)
}
waitTerminalOutput(t, second, term.ID, "still-here")
closed, err := second.Close(context.Background(), term.ID)
if err != nil || closed.Status != protocol.TerminalClosed {
t.Fatalf("close = %+v, %v", closed, err)
}
if tmux.alive(term.Tmux) {
t.Fatal("the tmux session outlived close")
}
if list, _ := second.List(context.Background()); len(list) != 0 {
t.Fatalf("list after close = %+v", list)
}
}
// A tmux client killed while its session lives is replaced.
func TestTerminalTmuxClientReplaced(t *testing.T) {
terminalHome(t)
tmux := testTmux(t)
m, _ := newTestTerminals(t, newOperatorStore(t), tmux)
term, err := m.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
waitTerminalOutput(t, m, term.ID, "$")
if err := tmux.command("detach-client", "-s", "="+term.Tmux).Run(); err != nil {
t.Fatal(err)
}
time.Sleep(300 * time.Millisecond)
if err := m.Input(term.ID, "echo back-$((1+1))\r"); err != nil {
t.Fatal(err)
}
waitTerminalOutput(t, m, term.ID, "back-2")
if record, _ := m.Output(context.Background(), term.ID, 0); record.Status != protocol.TerminalRunning {
t.Fatalf("record = %+v", record)
}
}
// Without tmux a shell ends with its server: the stop is recorded as
// ended, and a record a crashed server left running is marked ended too.
func TestTerminalWithoutTmuxEndsWithServer(t *testing.T) {
terminalHome(t)
store := newOperatorStore(t)
first, stop := newTestTerminals(t, store, nil)
term, err := first.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
stop()
first.Wait()
crashed := protocol.TerminalRecord{ID: "term_crashed", Cwd: "/", CreatedAt: 1, Status: protocol.TerminalRunning, Cols: 80, Rows: 24}
gone := protocol.TerminalRecord{ID: "term_gone", Cwd: "/", CreatedAt: 2, Tmux: "pipelines-term_gone", Status: protocol.TerminalRunning}
for _, record := range []protocol.TerminalRecord{crashed, gone} {
if err := store.SaveTerminal(context.Background(), record, 0); err != nil {
t.Fatal(err)
}
}
second, _ := newTestTerminals(t, store, testTmux(t))
list, err := second.List(context.Background())
if err != nil {
t.Fatal(err)
}
reasons := map[string]string{}
for _, record := range list {
if record.Status != protocol.TerminalExited || record.EndedAt == 0 {
t.Fatalf("record = %+v", record)
}
reasons[record.ID] = record.Reason
}
if reasons[term.ID] != terminalStopReason || reasons["term_crashed"] != terminalNoTmuxReason || reasons["term_gone"] != terminalGoneReason {
t.Fatalf("reasons = %+v", reasons)
}
if _, err := second.Close(context.Background(), "term_gone"); err != nil {
t.Fatal(err)
}
if list, _ := second.List(context.Background()); len(list) != 2 {
t.Fatalf("list after closing an ended terminal = %+v", list)
}
}
// The window title the shell sets is followed across reads and recorded.
func TestTerminalTitles(t *testing.T) {
var titles terminalTitles
if _, ok := titles.feed([]byte("plain \x1b[1mtext")); ok {
t.Fatal("title from plain text")
}
if _, ok := titles.feed([]byte("\x1b]0;~/src")); ok {
t.Fatal("title before its end")
}
if title, ok := titles.feed([]byte("/app\x07more")); !ok || title != "~/src/app" {
t.Fatalf("title = %q %v", title, ok)
}
if title, ok := titles.feed([]byte("\x1b]2;vim\x01 x\x1b\\")); !ok || title != "vim x" {
t.Fatalf("ST title = %q %v", title, ok)
}
if _, ok := titles.feed([]byte("\x1b]8;;https://x\x07")); ok {
t.Fatal("a hyperlink is not a title")
}
if title, _ := titles.feed([]byte("\x1b]0;" + strings.Repeat("é", protocol.TerminalTitleLimit) + "\x07")); len(title) > protocol.TerminalTitleLimit {
t.Fatalf("title of %d bytes", len(title))
}
terminalHome(t)
m, _ := newTestTerminals(t, newOperatorStore(t), nil)
term, err := m.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
if err := m.Input(term.ID, "printf '\\033]0;build-%s\\007' ok\r"); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for {
list, _ := m.List(context.Background())
if len(list) == 1 && list[0].Title == "build-ok" {
break
}
if time.Now().After(deadline) {
t.Fatalf("list = %+v", list)
}
time.Sleep(50 * time.Millisecond)
}
}
+169
View File
@@ -0,0 +1,169 @@
package server
import (
"context"
"log"
"os"
"os/exec"
"syscall"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// supervise streams one terminal until it ends: its first attachment is
// cmd on terminal. A tmux client that exits while its session lives (it was
// detached or killed) is replaced; a server shutdown leaves a tmux session
// running for the next server process and ends a direct shell.
func (m *terminalManager) supervise(p *terminalProcess, cmd *exec.Cmd, terminal *os.File) {
defer m.wg.Done()
defer close(p.done)
stopFlush, flushDone := make(chan struct{}), make(chan struct{})
go m.flushLoop(p, stopFlush, flushDone)
titled := func(string) {
record := p.snapshot(false, 0)
m.save(record)
m.publishRecord(record)
}
var waitErr error
shutdown := false
for reattached := 0; ; reattached++ {
readDone := make(chan struct{})
go func(terminal *os.File) {
defer close(readDone)
p.read(terminal, titled)
}(terminal)
attachDone := make(chan error, 1)
go func(cmd *exec.Cmd, terminal *os.File) { attachDone <- p.waitAttachment(cmd, terminal, readDone) }(cmd, terminal)
select {
case waitErr = <-attachDone:
case <-m.base.Done():
p.signal(syscall.SIGHUP)
select {
case waitErr = <-attachDone:
case <-time.After(time.Second):
p.signal(syscall.SIGKILL)
waitErr = <-attachDone
}
}
if m.base.Err() != nil {
shutdown = true
break
}
p.mu.Lock()
closing, name := p.closing, p.record.Tmux
p.mu.Unlock()
if closing || name == "" || m.tmux == nil || reattached >= terminalReattachLimit || !m.tmux.alive(name) {
break
}
cmd = m.tmux.attach(name)
next, err := p.startAttachment(cmd)
if err != nil {
log.Printf("terminal reattach %s: %v", p.record.ID, err)
break
}
terminal = next
}
close(stopFlush)
<-flushDone
m.emit(p, true)
p.mu.Lock()
switch {
case shutdown && p.record.Tmux != "":
// Still running in tmux: the next server process reattaches it.
case p.closing:
p.record.Status = protocol.TerminalClosed
default:
p.record.Status = protocol.TerminalExited
p.record.EndedAt = m.now().UnixMilli()
if shutdown {
p.record.Reason = terminalStopReason
} else if p.record.Tmux == "" {
p.record.ExitCode = terminalExitCode(waitErr)
}
}
p.pty, p.cmd = nil, nil
p.mu.Unlock()
record := p.snapshot(false, 0)
switch record.Status {
case protocol.TerminalClosed:
m.forget(record.ID)
default:
m.save(record)
}
if !shutdown {
m.publishRecord(record)
}
}
// flushLoop pushes new output in batches until stop is closed.
func (m *terminalManager) flushLoop(p *terminalProcess, stop <-chan struct{}, done chan<- struct{}) {
defer close(done)
ticker := time.NewTicker(m.flush)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
m.emit(p, false)
}
}
}
// emit pushes the output written since the last frame.
func (m *terminalManager) emit(p *terminalProcess, final bool) {
p.mu.Lock()
text, offset, end, ok := p.out.pending(protocol.TerminalChunkLimit, final)
id := p.record.ID
p.mu.Unlock()
if ok {
m.clients.publish(protocol.TerminalEventFrame{TerminalID: id, Output: text, Offset: offset, End: end})
}
}
// recover lists what a previous server process left running: a tmux-backed
// shell whose session is alive is reattached; any other is recorded as
// ended, saying why.
func (m *terminalManager) recover(ctx context.Context) {
if m.store == nil {
return
}
records, err := m.store.Terminals(ctx)
if err != nil {
log.Printf("terminal recover: %v", err)
return
}
for _, record := range records {
if record.Status != protocol.TerminalRunning {
continue
}
if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) && m.reattach(record) {
continue
}
record.Status, record.EndedAt = protocol.TerminalExited, m.now().UnixMilli()
record.Reason = terminalNoTmuxReason
if record.Tmux != "" {
record.Reason = terminalGoneReason
}
m.save(record)
}
}
// reattach shows a live tmux session in a new client and supervises it.
func (m *terminalManager) reattach(record protocol.TerminalRecord) bool {
cmd := m.tmux.attach(record.Tmux)
p := newTerminalProcess(record)
terminal, err := p.startAttachment(cmd)
if err != nil {
log.Printf("terminal reattach %s: %v", record.ID, err)
return false
}
m.mu.Lock()
m.terms[record.ID] = p
m.mu.Unlock()
m.wg.Add(1)
go m.supervise(p, cmd, terminal)
go p.pumpInput()
return true
}
+90
View File
@@ -0,0 +1,90 @@
package server
import (
"strings"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// terminalTitles follows a terminal stream across reads and reports the
// window title the program sets (OSC 0 or OSC 2, ended by BEL or ST), so a
// listed terminal keeps a name before any client draws it.
type terminalTitles struct {
state int
osc []byte
}
const (
titleText = iota
titleEscape
titleOSC
titleOSCEscape
)
// feed scans data and returns the last title it completed, if any.
func (t *terminalTitles) feed(data []byte) (string, bool) {
title, found := "", false
for _, b := range data {
switch t.state {
case titleText:
if b == 0x1b {
t.state = titleEscape
}
case titleEscape:
if b == ']' {
t.state, t.osc = titleOSC, t.osc[:0]
} else if b != 0x1b {
t.state = titleText
}
case titleOSC:
switch {
case b == 0x07:
if s, ok := t.finish(); ok {
title, found = s, true
}
case b == 0x1b:
t.state = titleOSCEscape
case len(t.osc) < 4*protocol.TerminalTitleLimit:
t.osc = append(t.osc, b)
}
case titleOSCEscape:
if b == '\\' {
if s, ok := t.finish(); ok {
title, found = s, true
}
} else {
t.state = titleEscape
}
}
}
return title, found
}
func (t *terminalTitles) finish() (string, bool) {
t.state = titleText
body := string(t.osc)
for _, prefix := range []string{"0;", "2;"} {
if strings.HasPrefix(body, prefix) {
return cleanTerminalTitle(body[len(prefix):]), true
}
}
return "", false
}
// cleanTerminalTitle keeps printable text within the title bound.
func cleanTerminalTitle(title string) string {
title = strings.ToValidUTF8(title, "")
title = strings.Map(func(r rune) rune {
if r < 0x20 || r == 0x7f {
return -1
}
return r
}, title)
title = strings.TrimSpace(title)
for len(title) > protocol.TerminalTitleLimit {
_, size := utf8.DecodeLastRuneInString(title)
title = title[:len(title)-size]
}
return title
}
+120
View File
@@ -0,0 +1,120 @@
package server
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// terminalTmuxSocket is the tmux server Pipelines owns (`tmux -L pipelines`):
// its sessions never mix with the operator's own tmux server.
const terminalTmuxSocket = "pipelines"
// terminalTmux runs one Pipelines-owned tmux server. Every terminal shell is
// the only pane of its own session there, so it outlives the Pipelines
// server; the server talks to it through an ordinary attached client in a
// pseudo-terminal.
type terminalTmux struct {
program string
socket string
}
// findTerminalTmux is the tmux on PATH (or the usual package-manager
// locations a service's PATH may lack); nil when none is installed.
func findTerminalTmux(socket string) *terminalTmux {
if socket == "" {
socket = terminalTmuxSocket
}
program, err := exec.LookPath("tmux")
if err != nil {
for _, dir := range []string{"/opt/homebrew/bin", "/usr/local/bin", "/usr/bin"} {
candidate := filepath.Join(dir, "tmux")
if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() && info.Mode()&0o111 != 0 {
program, err = candidate, nil
break
}
}
}
if err != nil {
return nil
}
return &terminalTmux{program: program, socket: socket}
}
// terminalTmuxName is the tmux session of terminal id.
func terminalTmuxName(id string) string { return "pipelines-" + id }
// command is one tmux client invocation against the Pipelines server, with
// no configuration file and outside any tmux the server itself runs in.
func (t *terminalTmux) command(args ...string) *exec.Cmd {
cmd := exec.Command(t.program, append([]string{"-L", t.socket, "-f", os.DevNull}, args...)...)
cmd.Env = terminalEnv(os.Environ(), "")
return cmd
}
// create starts session name running shell -l in cwd at cols x rows,
// detached. The first call also starts the tmux server; the options make
// it invisible (no status line), pass the shell's window title through,
// keep a bounded scrollback reachable by the mouse wheel, and leave Esc
// prompt.
func (t *terminalTmux) create(name, cwd, shell string, cols, rows int) error {
cmd := t.command(
"start-server", ";",
"set-option", "-g", "history-limit", strconv.Itoa(protocol.TerminalTmuxHistory), ";",
"set-option", "-g", "status", "off", ";",
"set-option", "-g", "mouse", "on", ";",
"set-option", "-g", "escape-time", "10", ";",
"set-option", "-g", "set-titles", "on", ";",
"set-option", "-g", "set-titles-string", "#{pane_title}", ";",
"new-session", "-d", "-s", name, "-c", cwd, "-x", strconv.Itoa(cols), "-y", strconv.Itoa(rows), "--", shell, "-l", ";",
"select-pane", "-t", "="+name+":", "-T", "",
)
cmd.Dir = cwd
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("tmux new-session: %v: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
// alive reports whether session name still exists.
func (t *terminalTmux) alive(name string) bool {
return t.command("has-session", "-t", "="+name).Run() == nil
}
// attach is the client command showing session name in the server's
// pseudo-terminal.
func (t *terminalTmux) attach(name string) *exec.Cmd {
return t.command("attach-session", "-t", "="+name)
}
// kill ends session name and its shell.
func (t *terminalTmux) kill(name string) error {
if out, err := t.command("kill-session", "-t", "="+name).CombinedOutput(); err != nil && t.alive(name) {
return fmt.Errorf("tmux kill-session: %v: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
// terminalEnv is env for a terminal's process: TERM names the emulator the
// clients draw, PWD the start directory (when set), and no TMUX variables
// leak in from a tmux the server runs inside.
func terminalEnv(env []string, cwd string) []string {
out := make([]string, 0, len(env)+2)
for _, entry := range env {
if strings.HasPrefix(entry, "TERM=") || strings.HasPrefix(entry, "TMUX=") || strings.HasPrefix(entry, "TMUX_PANE=") ||
(cwd != "" && strings.HasPrefix(entry, "PWD=")) {
continue
}
out = append(out, entry)
}
out = append(out, "TERM=xterm-256color")
if cwd != "" {
out = append(out, "PWD="+cwd)
}
return out
}
+173
View File
@@ -0,0 +1,173 @@
//go:build darwin || linux
package server
import (
"context"
"path/filepath"
"strings"
"testing"
"github.com/coder/websocket"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// terminalHome points the operator's home at a fresh directory and the
// login shell at /bin/sh, so tests never read the operator's own profile.
func terminalHome(t *testing.T) string {
t.Helper()
home, err := filepath.EvalSymlinks(t.TempDir())
if err != nil {
t.Fatal(err)
}
original := operatorHomeDirectory
operatorHomeDirectory = func() (string, error) { return home, nil }
t.Cleanup(func() { operatorHomeDirectory = original })
t.Setenv("SHELL", "/bin/sh")
t.Setenv("ENV", "")
return home
}
func terminalRequest(kind, id string, body protocol.TerminalRequest) protocol.Message {
return protocol.Message{Type: kind, ID: id, TerminalRequest: &body}
}
// terminalReply sends request and returns its reply (skipping pushes).
func terminalReply(t *testing.T, ctx context.Context, conn *websocket.Conn, request protocol.Message) protocol.Message {
t.Helper()
writeChat(t, ctx, conn, request)
return readShellFrame(t, ctx, conn, func(m protocol.Message) bool { return m.ID == request.ID })
}
// readTerminalUntil reads output frames of id from end on, in order, until
// the collected text contains want; it returns the text and the new end.
func readTerminalUntil(t *testing.T, ctx context.Context, conn *websocket.Conn, id string, end int64, want string) (string, int64) {
t.Helper()
var seen strings.Builder
for !strings.Contains(seen.String(), want) {
frame := readShellFrame(t, ctx, conn, func(m protocol.Message) bool {
return m.Type == protocol.TerminalEvent && m.TerminalEvent.TerminalID == id && m.TerminalEvent.Terminal == nil
}).TerminalEvent
if frame.Offset != end {
t.Fatalf("frame at %d after %d", frame.Offset, end)
}
seen.WriteString(frame.Output)
end = frame.End
}
return seen.String(), end
}
// One client opens a shell, types into it, resizes it; a second client
// lists it, resumes its output by offset and closes it.
func TestTerminalOverWebsocket(t *testing.T) {
home := terminalHome(t)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 2)
opened := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{Cols: 80, Rows: 24}))
term := opened.Terminal
if opened.Type != protocol.TerminalOpen || term == nil || term.Cwd != home || term.Status != protocol.TerminalRunning || term.Cols != 80 || term.Tmux != "" || term.Shell != "/bin/sh" {
t.Fatalf("open reply = %+v", opened)
}
if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "in", protocol.TerminalRequest{TerminalID: term.ID, Data: "pwd; echo sum-$((40+2))\r"})); got.Type != protocol.TerminalInput {
t.Fatalf("input reply = %+v", got)
}
_, end := readTerminalUntil(t, ctx, conns[0], term.ID, 0, "sum-42")
resized := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalResize, "size", protocol.TerminalRequest{TerminalID: term.ID, Cols: 120, Rows: 33}))
if resized.Type != protocol.TerminalResize || resized.Terminal.Cols != 120 || resized.Terminal.Rows != 33 {
t.Fatalf("resize reply = %+v", resized)
}
list := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalList, "list", protocol.TerminalRequest{}))
if len(list.Terminals) != 1 || list.Terminals[0].ID != term.ID || list.Terminals[0].Cols != 120 || list.Terminals[0].Output != "" {
t.Fatalf("list = %+v", list.Terminals)
}
// The second client reads everything once, then only what follows.
all := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalOutput, "all", protocol.TerminalRequest{TerminalID: term.ID})).Terminal
if all.OutputStart != 0 || !strings.Contains(all.Output, home) || !strings.Contains(all.Output, "sum-42") {
t.Fatalf("output = %+v", all)
}
terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalInput, "in2", protocol.TerminalRequest{TerminalID: term.ID, Data: "stty size\r"}))
readTerminalUntil(t, ctx, conns[0], term.ID, end, "33 120")
resume := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalOutput, "resume", protocol.TerminalRequest{TerminalID: term.ID, After: end})).Terminal
if resume.OutputStart != end || strings.Contains(resume.Output, "sum-42") || !strings.Contains(resume.Output, "33 120") {
t.Fatalf("resume = %+v", resume)
}
closed := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: term.ID}))
if closed.Type != protocol.TerminalClose || closed.Terminal.Status != protocol.TerminalClosed {
t.Fatalf("close reply = %+v", closed)
}
// Every connection hears the close; the list forgets it.
event := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool {
return m.Type == protocol.TerminalEvent && m.TerminalEvent.Terminal != nil && m.TerminalEvent.Terminal.Status == protocol.TerminalClosed
})
if event.TerminalEvent.TerminalID != term.ID {
t.Fatalf("close event = %+v", event)
}
if list := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalList, "list2", protocol.TerminalRequest{})); len(list.Terminals) != 0 {
t.Fatalf("list after close = %+v", list.Terminals)
}
if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "late", protocol.TerminalRequest{TerminalID: term.ID, Data: "x"})); got.Type != protocol.Error {
t.Fatalf("late input = %+v", got)
}
}
// A shell that exits by itself is listed as exited with its status and
// keeps its output until it is closed.
func TestTerminalShellExit(t *testing.T) {
terminalHome(t)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 1)
term := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{})).Terminal
terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "in", protocol.TerminalRequest{TerminalID: term.ID, Data: "echo bye; exit 3\r"}))
exited := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool {
return m.Type == protocol.TerminalEvent && m.TerminalEvent.Terminal != nil && m.TerminalEvent.Terminal.Status == protocol.TerminalExited
}).TerminalEvent.Terminal
if exited.ExitCode == nil || *exited.ExitCode != 3 || exited.EndedAt == 0 {
t.Fatalf("exited = %+v", exited)
}
out := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOutput, "out", protocol.TerminalRequest{TerminalID: term.ID})).Terminal
if !strings.Contains(out.Output, "bye") || out.Status != protocol.TerminalExited {
t.Fatalf("output = %+v", out)
}
if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: term.ID})); got.Terminal == nil || got.Terminal.Status != protocol.TerminalClosed {
t.Fatalf("close = %+v", got)
}
}
// Requests outside the bounds are refused: directories outside home or
// missing, input bounds, unknown terminals, the open limit.
func TestTerminalLimitsAndValidation(t *testing.T) {
home := terminalHome(t)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 1)
conn := conns[0]
for i, cwd := range []string{"/", filepath.Join(home, "missing"), "relative", "~other"} {
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "bad", protocol.TerminalRequest{Cwd: cwd})); got.Type != protocol.Error {
t.Fatalf("cwd %d %q = %+v", i, cwd, got)
}
}
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOutput, "none", protocol.TerminalRequest{TerminalID: "term_nope"})); got.Type != protocol.Error {
t.Fatalf("unknown = %+v", got)
}
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalInput, "noid", protocol.TerminalRequest{Data: "x"})); got.Type != protocol.Error {
t.Fatalf("no id = %+v", got)
}
var ids []string
for i := 0; i < protocol.TerminalOpenLimit; i++ {
got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{Cwd: "~"}))
if got.Terminal == nil {
t.Fatalf("open %d = %+v", i, got)
}
ids = append(ids, got.Terminal.ID)
}
over := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "over", protocol.TerminalRequest{}))
if over.Type != protocol.Error || !strings.Contains(over.Error, "8 open terminals") {
t.Fatalf("over limit = %+v", over)
}
for _, bad := range []string{"", strings.Repeat("x", protocol.ChatShellInputLimit+1)} {
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalInput, "bad-in", protocol.TerminalRequest{TerminalID: ids[0], Data: bad})); got.Type != protocol.Error {
t.Fatalf("input %q = %+v", bad[:min(len(bad), 4)], got)
}
}
terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: ids[0]}))
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "again", protocol.TerminalRequest{})); got.Terminal == nil {
t.Fatalf("open after close = %+v", got)
}
}
+36 -1
View File
@@ -66,6 +66,12 @@ type Options struct {
// False keeps the previous behaviour (tests, embedded servers): a stopping
// server interrupts its runs and the next start marks them interrupted.
DetachSessions bool
// TerminalTmuxSocket names the Pipelines-owned tmux server machine
// terminals live in (empty: "pipelines"); TerminalWithoutTmux runs
// shells as direct children even where tmux is installed. Tests set
// them so they never share the real server's tmux sessions.
TerminalTmuxSocket string
TerminalWithoutTmux bool
}
// Handler serves the persistent connection with default options. Canceling ctx
@@ -143,6 +149,20 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
} else if store, ok := registry.(chatShellRunStore); ok {
hub.startShellRunner(ctx, store)
}
// Machine terminals are recorded beside the operator sessions, in the
// same store; a previous process's tmux-backed shells are reattached.
var terminalRecords terminalStore
if len(stores) > 0 {
terminalRecords = stores[0]
} else if store, ok := registry.(terminalStore); ok {
terminalRecords = store
}
var terminalTmuxServer *terminalTmux
if !options.TerminalWithoutTmux {
terminalTmuxServer = findTerminalTmux(options.TerminalTmuxSocket)
}
terminals := newTerminalManager(ctx, terminalRecords, terminalTmuxServer)
terminals.recover(ctx)
acks := newAttentionAckHub()
var ackStore attentionAckStore
var notes *appNotifier
@@ -237,6 +257,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
// connection the same way.
hub.sessionChanges.attach(clientID, write)
defer hub.sessionChanges.detach(clientID)
// Terminal lifecycle changes reach every connection; output only the
// connections that opened or read that terminal.
terminals.clients.attach(clientID, write)
defer terminals.clients.detach(clientID)
if notes != nil {
defer notes.detach(clientID)
}
@@ -250,7 +274,8 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
if err := wsjson.Read(connCtx, conn, &request); err != nil {
return
}
if request.Type != protocol.ChatShellInput && request.Type != protocol.ChatShellResize {
if request.Type != protocol.ChatShellInput && request.Type != protocol.ChatShellResize &&
request.Type != protocol.TerminalInput && request.Type != protocol.TerminalResize {
// Keystrokes would be one line each; their data is never logged.
log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version)
}
@@ -291,6 +316,16 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
continue
}
if isTerminalRequest(request.Type) {
// Closing waits for the shell to go; it never delays input
// to the other terminals behind it.
if request.Type == protocol.TerminalClose {
go handleTerminalRequest(connCtx, request, terminals, clientID, write)
} else {
handleTerminalRequest(connCtx, request, terminals, clientID, write)
}
continue
}
if handleDirectoryListing(request, write) {
continue
}