Add persistent machine terminals with tmux-backed reattach
This commit is contained in:
@@ -109,6 +109,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initTerminals(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initAppNotifications(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// initTerminals creates the additive table of machine terminals. It records
|
||||
// only what a restarted server needs to list and reattach them (id, title,
|
||||
// cwd, shell, tmux session, status). Output and typed input are never stored.
|
||||
func (s *Store) initTerminals() error {
|
||||
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS terminals (
|
||||
id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
cwd TEXT NOT NULL,
|
||||
shell TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL,
|
||||
tmux TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL,
|
||||
ended_at INTEGER NOT NULL DEFAULT 0,
|
||||
exit_code INTEGER,
|
||||
reason TEXT NOT NULL DEFAULT '',
|
||||
cols INTEGER NOT NULL DEFAULT 0,
|
||||
rows INTEGER NOT NULL DEFAULT 0);`)
|
||||
return err
|
||||
}
|
||||
|
||||
// SaveTerminal inserts or replaces one terminal. Saving an ended terminal
|
||||
// prunes the oldest ended ones beyond keep (keep <= 0 keeps everything).
|
||||
func (s *Store) SaveTerminal(ctx context.Context, t protocol.TerminalRecord, keep int) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var exit any
|
||||
if t.ExitCode != nil {
|
||||
exit = *t.ExitCode
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO terminals
|
||||
(id, title, cwd, shell, created_at, tmux, status, ended_at, exit_code, reason, cols, rows)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET title = excluded.title, status = excluded.status, ended_at = excluded.ended_at,
|
||||
exit_code = excluded.exit_code, reason = excluded.reason, cols = excluded.cols, rows = excluded.rows`,
|
||||
t.ID, t.Title, t.Cwd, t.Shell, t.CreatedAt, t.Tmux, t.Status, t.EndedAt, exit, t.Reason, t.Cols, t.Rows); err != nil {
|
||||
return err
|
||||
}
|
||||
if keep > 0 && t.Status != protocol.TerminalRunning {
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM terminals WHERE status != ? AND id NOT IN
|
||||
(SELECT id FROM terminals WHERE status != ? ORDER BY ended_at DESC, id DESC LIMIT ?)`,
|
||||
protocol.TerminalRunning, protocol.TerminalRunning, keep); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteTerminal removes one terminal's record.
|
||||
func (s *Store) DeleteTerminal(ctx context.Context, id string) error {
|
||||
_, err := s.db.ExecContext(ctx, `DELETE FROM terminals WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// Terminals lists every recorded terminal, oldest first.
|
||||
func (s *Store) Terminals(ctx context.Context) ([]protocol.TerminalRecord, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT id, title, cwd, shell, created_at, tmux, status, ended_at, exit_code, reason, cols, rows
|
||||
FROM terminals ORDER BY created_at, id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []protocol.TerminalRecord
|
||||
for rows.Next() {
|
||||
var t protocol.TerminalRecord
|
||||
var exit sql.NullInt64
|
||||
if err := rows.Scan(&t.ID, &t.Title, &t.Cwd, &t.Shell, &t.CreatedAt, &t.Tmux, &t.Status, &t.EndedAt, &exit, &t.Reason, &t.Cols, &t.Rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if exit.Valid {
|
||||
code := int(exit.Int64)
|
||||
t.ExitCode = &code
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestTerminalsPersistPruneAndDelete(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "terminals.db")
|
||||
store, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
live := protocol.TerminalRecord{ID: "term_live", Cwd: "/home/a", Shell: "/bin/zsh", CreatedAt: 1, Tmux: "pipelines-term_live", Status: protocol.TerminalRunning, Cols: 80, Rows: 24}
|
||||
if err := store.SaveTerminal(ctx, live, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code := 0
|
||||
for i := 0; i < 3; i++ {
|
||||
ended := protocol.TerminalRecord{ID: fmt.Sprintf("term_%d", i), Cwd: "/home/a", CreatedAt: int64(10 + i), Status: protocol.TerminalExited, EndedAt: int64(20 + i), ExitCode: &code}
|
||||
if err := store.SaveTerminal(ctx, ended, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
live.Title, live.Cols = "vim", 120
|
||||
if err := store.SaveTerminal(ctx, live, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store.Close()
|
||||
store, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
list, err := store.Terminals(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(list) != 3 || list[0].ID != "term_live" || list[0].Title != "vim" || list[0].Cols != 120 || list[0].Tmux != "pipelines-term_live" ||
|
||||
list[1].ID != "term_1" || list[2].ID != "term_2" || list[2].ExitCode == nil || *list[2].ExitCode != 0 {
|
||||
t.Fatalf("terminals = %+v", list)
|
||||
}
|
||||
if err := store.DeleteTerminal(ctx, "term_live"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if list, _ := store.Terminals(ctx); len(list) != 2 {
|
||||
t.Fatalf("after delete = %+v", list)
|
||||
}
|
||||
}
|
||||
@@ -280,6 +280,12 @@ type Message struct {
|
||||
ChatShellRun *ChatShellRunRecord `json:"chat_shell_run,omitempty"`
|
||||
ChatShellRuns []ChatShellRunRecord `json:"chat_shell_runs,omitempty"`
|
||||
ChatShellRunEvent *ChatShellRunEventFrame `json:"chat_shell_run_event,omitempty"`
|
||||
// Machine terminals (terminal.go): the request body, one terminal, the
|
||||
// machine's terminals and the pushed change.
|
||||
TerminalRequest *TerminalRequest `json:"terminal_request,omitempty"`
|
||||
Terminal *TerminalRecord `json:"terminal,omitempty"`
|
||||
Terminals []TerminalRecord `json:"terminals,omitempty"`
|
||||
TerminalEvent *TerminalEventFrame `json:"terminal_event,omitempty"`
|
||||
// ListDirectory is the list_directory body and DirectoryListing its reply:
|
||||
// the home-restricted folder picker listing (see directories.go).
|
||||
ListDirectory *DirectoryListRequest `json:"list_directory,omitempty"`
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package protocol
|
||||
|
||||
// Machine terminals: interactive login shells the operator opens on a
|
||||
// server's machine, independent of any chat. Each runs `$SHELL -l` (else
|
||||
// /bin/sh) in a pseudo-terminal; where tmux is installed the shell lives in
|
||||
// a Pipelines-owned tmux server (socket name "pipelines", never the
|
||||
// operator's own), so it survives a Pipelines server restart and the next
|
||||
// process reattaches it. Without tmux a shell ends with the server and is
|
||||
// listed as ended. Typed input is never logged or stored.
|
||||
const (
|
||||
// TerminalOpen starts a shell. The body is TerminalRequest (cwd, cols,
|
||||
// rows); cwd defaults to the operator's home and must be an existing
|
||||
// directory inside it. The reply carries the new terminal; the
|
||||
// connection is then subscribed to its terminal_event output frames.
|
||||
TerminalOpen = "terminal_open"
|
||||
// TerminalList lists this machine's terminals (running and ended),
|
||||
// oldest first, without output. The body is optional.
|
||||
TerminalList = "terminal_list"
|
||||
// TerminalInput writes Data (UTF-8, control characters included, at most
|
||||
// ChatShellInputLimit bytes) to a running terminal. No reply body.
|
||||
TerminalInput = "terminal_input"
|
||||
// TerminalResize sets a running terminal's size (cols, rows); the reply
|
||||
// echoes the terminal.
|
||||
TerminalResize = "terminal_resize"
|
||||
// TerminalOutput reads one terminal with its retained output from the
|
||||
// absolute byte After on (all of it when After precedes the retained
|
||||
// tail; OutputStart says where it starts). It also subscribes the
|
||||
// connection to the terminal's terminal_event output frames.
|
||||
TerminalOutput = "terminal_output"
|
||||
// TerminalClose ends a terminal: its shell (and tmux session) is killed
|
||||
// and its record removed. The reply echoes the last record.
|
||||
TerminalClose = "terminal_close"
|
||||
// TerminalEvent is pushed: an output chunk to the connections subscribed
|
||||
// to that terminal (Output, Offset, End), or a lifecycle change (Terminal
|
||||
// set: opened, title, exited, closed) to every connection.
|
||||
TerminalEvent = "terminal_event"
|
||||
)
|
||||
|
||||
// Terminal status values. Running is the only live status.
|
||||
const (
|
||||
TerminalRunning = "running"
|
||||
TerminalExited = "exited"
|
||||
TerminalClosed = "closed"
|
||||
)
|
||||
|
||||
// Terminal bounds.
|
||||
const (
|
||||
// TerminalOpenLimit bounds the running terminals of one machine.
|
||||
TerminalOpenLimit = 8
|
||||
// TerminalRetained is the output tail kept for reconnecting clients.
|
||||
TerminalRetained = 1 << 20
|
||||
// TerminalChunkLimit bounds one live output frame.
|
||||
TerminalChunkLimit = 64 << 10
|
||||
// TerminalEndedKept bounds the ended terminals still listed.
|
||||
TerminalEndedKept = 16
|
||||
// TerminalTmuxHistory is the tmux scrollback of one tmux-backed shell.
|
||||
TerminalTmuxHistory = 10000
|
||||
// TerminalTitleLimit bounds a recorded title.
|
||||
TerminalTitleLimit = 200
|
||||
)
|
||||
|
||||
// TerminalRequest is the body of every terminal request.
|
||||
type TerminalRequest struct {
|
||||
TerminalID string `json:"terminal_id,omitempty"`
|
||||
Cwd string `json:"cwd,omitempty"`
|
||||
Cols int `json:"cols,omitempty"`
|
||||
Rows int `json:"rows,omitempty"`
|
||||
Data string `json:"data,omitempty"`
|
||||
After int64 `json:"after,omitempty"`
|
||||
}
|
||||
|
||||
// TerminalRecord is one terminal. Output and OutputStart are set only on
|
||||
// terminal_output replies.
|
||||
type TerminalRecord struct {
|
||||
ID string `json:"id"`
|
||||
// Title is the last terminal title (OSC 0/2) the shell set, or empty;
|
||||
// clients show the cwd then.
|
||||
Title string `json:"title,omitempty"`
|
||||
Cwd string `json:"cwd"`
|
||||
Shell string `json:"shell"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
// Tmux is the tmux session name keeping the shell alive across server
|
||||
// restarts; empty when the shell is a direct child of the server and
|
||||
// ends with it.
|
||||
Tmux string `json:"tmux,omitempty"`
|
||||
Status string `json:"status"`
|
||||
EndedAt int64 `json:"ended_at,omitempty"`
|
||||
ExitCode *int `json:"exit_code,omitempty"`
|
||||
// Reason says why an ended terminal ended when it was not simply its
|
||||
// shell exiting (the server restarted without tmux, its tmux session
|
||||
// was gone).
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Cols int `json:"cols"`
|
||||
Rows int `json:"rows"`
|
||||
// OutputEnd is the absolute byte count this server process has read;
|
||||
// it restarts at zero when a restarted server reattaches.
|
||||
OutputEnd int64 `json:"output_end"`
|
||||
Output string `json:"output,omitempty"`
|
||||
OutputStart int64 `json:"output_start,omitempty"`
|
||||
}
|
||||
|
||||
// TerminalEventFrame is one pushed terminal change: either Terminal (a
|
||||
// lifecycle change) or one output chunk covering bytes [Offset, End).
|
||||
type TerminalEventFrame struct {
|
||||
TerminalID string `json:"terminal_id"`
|
||||
Terminal *TerminalRecord `json:"terminal,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
Offset int64 `json:"offset,omitempty"`
|
||||
End int64 `json:"end,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// terminalClientQueue bounds the frames waiting for one slow connection.
|
||||
// A full queue drops output frames; the client sees the gap in the next
|
||||
// frame's offset and reads the retained tail again (terminal_output after).
|
||||
const terminalClientQueue = 256
|
||||
|
||||
// terminalClients delivers terminal_event frames to open connections, in
|
||||
// order per connection: lifecycle changes to every connection, output only
|
||||
// to connections subscribed to that terminal (by terminal_open or
|
||||
// terminal_output). It holds each connection's serialized write function,
|
||||
// never the socket. The zero value is ready to use.
|
||||
type terminalClients struct {
|
||||
mu sync.Mutex
|
||||
clients map[string]*terminalClient
|
||||
}
|
||||
|
||||
type terminalClient struct {
|
||||
ch chan protocol.Message
|
||||
done chan struct{}
|
||||
subs map[string]bool
|
||||
}
|
||||
|
||||
func (c *terminalClients) attach(clientID string, write func(protocol.Message) error) {
|
||||
client := &terminalClient{ch: make(chan protocol.Message, terminalClientQueue), done: make(chan struct{}), subs: map[string]bool{}}
|
||||
c.mu.Lock()
|
||||
if c.clients == nil {
|
||||
c.clients = map[string]*terminalClient{}
|
||||
}
|
||||
if old := c.clients[clientID]; old != nil {
|
||||
close(old.done)
|
||||
}
|
||||
c.clients[clientID] = client
|
||||
c.mu.Unlock()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-client.done:
|
||||
return
|
||||
case msg := <-client.ch:
|
||||
if write(msg) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (c *terminalClients) detach(clientID string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if client := c.clients[clientID]; client != nil {
|
||||
close(client.done)
|
||||
delete(c.clients, clientID)
|
||||
}
|
||||
}
|
||||
|
||||
// subscribe sends terminalID's output frames to clientID from now on.
|
||||
func (c *terminalClients) subscribe(clientID, terminalID string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if client := c.clients[clientID]; client != nil {
|
||||
client.subs[terminalID] = true
|
||||
}
|
||||
}
|
||||
|
||||
// publish queues frame for its recipients without blocking: a lifecycle
|
||||
// change for everyone, output for the terminal's subscribers.
|
||||
func (c *terminalClients) publish(frame protocol.TerminalEventFrame) {
|
||||
msg := protocol.Message{Type: protocol.TerminalEvent, TerminalEvent: &frame}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for _, client := range c.clients {
|
||||
if frame.Terminal == nil && !client.subs[frame.TerminalID] {
|
||||
continue
|
||||
}
|
||||
if frame.Terminal != nil && frame.Terminal.Status == protocol.TerminalClosed {
|
||||
delete(client.subs, frame.TerminalID)
|
||||
}
|
||||
select {
|
||||
case client.ch <- msg:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// resolveTerminalCwd is the directory a terminal starts in: requested ("",
|
||||
// "~", "~/sub" or an absolute path) must be an existing directory inside
|
||||
// home once every symlink is resolved, the containment the Files panel and
|
||||
// folder picker apply. It returns the path as the operator sees it, under
|
||||
// home.
|
||||
func resolveTerminalCwd(home, requested string) (string, error) {
|
||||
home = filepath.Clean(home)
|
||||
realHome, err := filepath.EvalSymlinks(home)
|
||||
if err != nil {
|
||||
return "", errors.New("resolve home directory: " + err.Error())
|
||||
}
|
||||
target, err := expandHomePath(home, requested)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
realTarget, err := filepath.EvalSymlinks(target)
|
||||
if err != nil {
|
||||
return "", describePathError(target, err)
|
||||
}
|
||||
rel, inside := relInside(realHome, realTarget)
|
||||
if !inside {
|
||||
return "", errors.New(target + " is outside the home directory " + home + "; a terminal starts only in a folder inside home")
|
||||
}
|
||||
shown := filepath.Join(home, rel)
|
||||
info, err := os.Stat(realTarget)
|
||||
if err != nil {
|
||||
return "", describePathError(shown, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", errors.New(shown + " is not a directory")
|
||||
}
|
||||
return shown, nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// isTerminalRequest reports the request types handleTerminalRequest answers.
|
||||
func isTerminalRequest(kind string) bool {
|
||||
switch kind {
|
||||
case protocol.TerminalOpen, protocol.TerminalList, protocol.TerminalInput, protocol.TerminalResize,
|
||||
protocol.TerminalOutput, protocol.TerminalClose:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// handleTerminalRequest answers one machine-terminal request from an
|
||||
// authenticated connection. Opening or reading a terminal subscribes the
|
||||
// connection to its output frames.
|
||||
func handleTerminalRequest(ctx context.Context, request protocol.Message, m *terminalManager, clientID string, write func(protocol.Message) error) {
|
||||
fail := func(err error) { _ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()}) }
|
||||
if m == nil {
|
||||
fail(errors.New("terminals are not configured on this server"))
|
||||
return
|
||||
}
|
||||
body := request.TerminalRequest
|
||||
if body == nil {
|
||||
body = &protocol.TerminalRequest{}
|
||||
}
|
||||
if request.Type != protocol.TerminalOpen && request.Type != protocol.TerminalList && body.TerminalID == "" {
|
||||
fail(errors.New("a terminal_id is required"))
|
||||
return
|
||||
}
|
||||
reply := protocol.Message{Type: request.Type, ID: request.ID}
|
||||
var record protocol.TerminalRecord
|
||||
var err error
|
||||
switch request.Type {
|
||||
case protocol.TerminalOpen:
|
||||
record, err = m.Open(ctx, clientID, body.Cwd, body.Cols, body.Rows)
|
||||
case protocol.TerminalList:
|
||||
var records []protocol.TerminalRecord
|
||||
records, err = m.List(ctx)
|
||||
reply.Terminals = records
|
||||
case protocol.TerminalInput:
|
||||
err = m.Input(body.TerminalID, body.Data)
|
||||
case protocol.TerminalResize:
|
||||
record, err = m.Resize(body.TerminalID, body.Cols, body.Rows)
|
||||
case protocol.TerminalOutput:
|
||||
m.clients.subscribe(clientID, body.TerminalID)
|
||||
record, err = m.Output(ctx, body.TerminalID, body.After)
|
||||
case protocol.TerminalClose:
|
||||
record, err = m.Close(ctx, body.TerminalID)
|
||||
}
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
if record.ID != "" {
|
||||
reply.Terminal = &record
|
||||
}
|
||||
_ = write(reply)
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// terminalStore persists the machine's terminal list; *pipeline.Store
|
||||
// implements it.
|
||||
type terminalStore interface {
|
||||
SaveTerminal(ctx context.Context, terminal protocol.TerminalRecord, keep int) error
|
||||
DeleteTerminal(ctx context.Context, id string) error
|
||||
Terminals(ctx context.Context) ([]protocol.TerminalRecord, error)
|
||||
}
|
||||
|
||||
const (
|
||||
terminalNoTmuxReason = "The Pipelines server restarted and this shell ended with it (tmux is not installed on this machine, so shells do not outlive the server)."
|
||||
terminalGoneReason = "Its tmux session ended while the Pipelines server was stopped."
|
||||
terminalStopReason = "The Pipelines server stopped and this shell ended with it (tmux is not installed on this machine)."
|
||||
// terminalReattachLimit bounds how often a tmux client that exits while
|
||||
// its session lives is replaced before the terminal is given up.
|
||||
terminalReattachLimit = 5
|
||||
)
|
||||
|
||||
// terminalManager owns the machine's terminals (terminal.go in protocol).
|
||||
type terminalManager struct {
|
||||
base context.Context
|
||||
store terminalStore
|
||||
clients *terminalClients
|
||||
tmux *terminalTmux
|
||||
shell func() string
|
||||
home func() (string, error)
|
||||
now func() time.Time
|
||||
flush time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
terms map[string]*terminalProcess
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func newTerminalManager(base context.Context, store terminalStore, tmux *terminalTmux) *terminalManager {
|
||||
return &terminalManager{
|
||||
base: base, store: store, clients: &terminalClients{}, tmux: tmux,
|
||||
shell: chatShellProgram, home: OperatorHomeDirectory, now: time.Now, flush: 50 * time.Millisecond,
|
||||
terms: map[string]*terminalProcess{},
|
||||
}
|
||||
}
|
||||
|
||||
func newTerminalID() (string, error) {
|
||||
var b [8]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "term_" + hex.EncodeToString(b[:]), nil
|
||||
}
|
||||
|
||||
// Open starts a login shell in cwd at cols x rows and returns its record.
|
||||
// clientID (when set) is subscribed to its output before the shell writes.
|
||||
func (m *terminalManager) Open(ctx context.Context, clientID, cwd string, cols, rows int) (protocol.TerminalRecord, error) {
|
||||
if m.store == nil {
|
||||
return protocol.TerminalRecord{}, errors.New("terminals need the server's store, which is not configured")
|
||||
}
|
||||
if !chatShellTerminals {
|
||||
return protocol.TerminalRecord{}, errors.New(protocol.ChatShellTerminalUnsupported)
|
||||
}
|
||||
home, err := m.home()
|
||||
if err != nil {
|
||||
return protocol.TerminalRecord{}, err
|
||||
}
|
||||
dir, err := resolveTerminalCwd(home, cwd)
|
||||
if err != nil {
|
||||
return protocol.TerminalRecord{}, err
|
||||
}
|
||||
id, err := newTerminalID()
|
||||
if err != nil {
|
||||
return protocol.TerminalRecord{}, err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.base.Err() != nil {
|
||||
return protocol.TerminalRecord{}, errors.New("the server is stopping")
|
||||
}
|
||||
if running := m.runningLocked(); running >= protocol.TerminalOpenLimit {
|
||||
return protocol.TerminalRecord{}, fmt.Errorf("this machine already has %d open terminals; close one first", running)
|
||||
}
|
||||
cols, rows = chatShellTerminalSize(cols, rows)
|
||||
shell := m.shell()
|
||||
record := protocol.TerminalRecord{ID: id, Cwd: dir, Shell: shell, CreatedAt: m.now().UnixMilli(), Status: protocol.TerminalRunning, Cols: cols, Rows: rows}
|
||||
var cmd *exec.Cmd
|
||||
if m.tmux != nil {
|
||||
record.Tmux = terminalTmuxName(id)
|
||||
if err := m.tmux.create(record.Tmux, dir, shell, cols, rows); err != nil {
|
||||
return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err)
|
||||
}
|
||||
cmd = m.tmux.attach(record.Tmux)
|
||||
} else {
|
||||
cmd = exec.Command(shell, "-l")
|
||||
cmd.Env = terminalEnv(os.Environ(), dir)
|
||||
}
|
||||
cmd.Dir = dir
|
||||
p := newTerminalProcess(record)
|
||||
terminal, err := p.startAttachment(cmd)
|
||||
if err == nil {
|
||||
err = m.store.SaveTerminal(ctx, record, protocol.TerminalEndedKept)
|
||||
if err != nil {
|
||||
p.signal(syscall.SIGKILL)
|
||||
terminal.Close()
|
||||
_ = cmd.Wait()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if record.Tmux != "" {
|
||||
_ = m.tmux.kill(record.Tmux)
|
||||
}
|
||||
return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err)
|
||||
}
|
||||
m.terms[id] = p
|
||||
if clientID != "" {
|
||||
m.clients.subscribe(clientID, id)
|
||||
}
|
||||
m.wg.Add(1)
|
||||
go m.supervise(p, cmd, terminal)
|
||||
go p.pumpInput()
|
||||
m.publishRecord(record)
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func (m *terminalManager) runningLocked() int {
|
||||
running := 0
|
||||
for _, p := range m.terms {
|
||||
if p.running() {
|
||||
running++
|
||||
}
|
||||
}
|
||||
return running
|
||||
}
|
||||
|
||||
// List is every recorded terminal, oldest first, with the live state of the
|
||||
// ones this process holds.
|
||||
func (m *terminalManager) List(ctx context.Context) ([]protocol.TerminalRecord, error) {
|
||||
if m.store == nil {
|
||||
return nil, errors.New("terminals need the server's store, which is not configured")
|
||||
}
|
||||
records, err := m.store.Terminals(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, record := range records {
|
||||
if p := m.process(record.ID); p != nil {
|
||||
records[i] = p.snapshot(false, 0)
|
||||
}
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (m *terminalManager) process(id string) *terminalProcess {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.terms[id]
|
||||
}
|
||||
|
||||
func (m *terminalManager) live(id string) (*terminalProcess, error) {
|
||||
p := m.process(id)
|
||||
if p == nil || !p.running() {
|
||||
return nil, errors.New("this terminal is not running")
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Input queues data for a running terminal, in arrival order. It is kept
|
||||
// nowhere else: what the shell echoes is output.
|
||||
func (m *terminalManager) Input(id, data string) error {
|
||||
switch {
|
||||
case data == "":
|
||||
return errors.New("input is required")
|
||||
case len(data) > protocol.ChatShellInputLimit:
|
||||
return errors.New("the input exceeds 16 KiB; send it in parts")
|
||||
case !utf8.ValidString(data):
|
||||
return errors.New("input must be UTF-8 text")
|
||||
}
|
||||
p, err := m.live(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-p.done:
|
||||
return errors.New("this terminal is not running")
|
||||
case p.input <- []byte(data):
|
||||
return nil
|
||||
default:
|
||||
return errors.New("the terminal is not reading its input; wait for it or press Ctrl-C")
|
||||
}
|
||||
}
|
||||
|
||||
// Resize sets a running terminal's size and returns its record.
|
||||
func (m *terminalManager) Resize(id string, cols, rows int) (protocol.TerminalRecord, error) {
|
||||
if cols <= 0 || rows <= 0 {
|
||||
return protocol.TerminalRecord{}, errors.New("positive cols and rows are required")
|
||||
}
|
||||
p, err := m.live(id)
|
||||
if err != nil {
|
||||
return protocol.TerminalRecord{}, err
|
||||
}
|
||||
cols, rows = chatShellTerminalSize(cols, rows)
|
||||
p.mu.Lock()
|
||||
p.record.Cols, p.record.Rows = cols, rows
|
||||
terminal := p.pty
|
||||
p.mu.Unlock()
|
||||
if terminal != nil {
|
||||
if err := resizeChatShellTerminal(terminal, cols, rows); err != nil {
|
||||
return protocol.TerminalRecord{}, err
|
||||
}
|
||||
}
|
||||
return p.snapshot(false, 0), nil
|
||||
}
|
||||
|
||||
// Output is one terminal with its retained output from byte after on. A
|
||||
// terminal ended before this process started has no output.
|
||||
func (m *terminalManager) Output(ctx context.Context, id string, after int64) (protocol.TerminalRecord, error) {
|
||||
if p := m.process(id); p != nil {
|
||||
return p.snapshot(true, after), nil
|
||||
}
|
||||
records, err := m.List(ctx)
|
||||
if err != nil {
|
||||
return protocol.TerminalRecord{}, err
|
||||
}
|
||||
for _, record := range records {
|
||||
if record.ID == id {
|
||||
return record, nil
|
||||
}
|
||||
}
|
||||
return protocol.TerminalRecord{}, errors.New("this machine has no such terminal")
|
||||
}
|
||||
|
||||
// Close ends a terminal (killing its shell and tmux session) and forgets
|
||||
// it. It waits briefly for the shell to go and returns the closed record.
|
||||
func (m *terminalManager) Close(ctx context.Context, id string) (protocol.TerminalRecord, error) {
|
||||
if m.store == nil {
|
||||
return protocol.TerminalRecord{}, errors.New("terminals need the server's store, which is not configured")
|
||||
}
|
||||
p := m.process(id)
|
||||
if p != nil && p.running() {
|
||||
p.mu.Lock()
|
||||
p.closing = true
|
||||
name := p.record.Tmux
|
||||
p.mu.Unlock()
|
||||
if name != "" && m.tmux != nil {
|
||||
if err := m.tmux.kill(name); err != nil {
|
||||
log.Printf("terminal close %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
p.signal(syscall.SIGHUP)
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(3 * time.Second):
|
||||
p.signal(syscall.SIGKILL)
|
||||
<-p.done
|
||||
}
|
||||
return p.snapshot(false, 0), nil
|
||||
}
|
||||
record, err := m.Output(ctx, id, 0)
|
||||
if err != nil {
|
||||
return protocol.TerminalRecord{}, err
|
||||
}
|
||||
if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) {
|
||||
_ = m.tmux.kill(record.Tmux)
|
||||
}
|
||||
m.forget(id)
|
||||
record.Output, record.OutputStart = "", 0
|
||||
record.Status = protocol.TerminalClosed
|
||||
m.publishRecord(record)
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// forget drops a terminal's record and retained output.
|
||||
func (m *terminalManager) forget(id string) {
|
||||
m.mu.Lock()
|
||||
delete(m.terms, id)
|
||||
m.mu.Unlock()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := m.store.DeleteTerminal(ctx, id); err != nil {
|
||||
log.Printf("terminal delete %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *terminalManager) save(record protocol.TerminalRecord) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := m.store.SaveTerminal(ctx, record, protocol.TerminalEndedKept); err != nil {
|
||||
log.Printf("terminal save %s: %v", record.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *terminalManager) publishRecord(record protocol.TerminalRecord) {
|
||||
record.Output, record.OutputStart = "", 0
|
||||
m.clients.publish(protocol.TerminalEventFrame{TerminalID: record.ID, Terminal: &record})
|
||||
}
|
||||
|
||||
// Wait blocks until every supervised terminal has been released (tests,
|
||||
// shutdown).
|
||||
func (m *terminalManager) Wait() { m.wg.Wait() }
|
||||
@@ -0,0 +1,173 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// terminalProcess is one terminal this server process holds: running (its
|
||||
// attachment is the shell itself, or a tmux client showing the shell's
|
||||
// session) or ended with its output still retained.
|
||||
type terminalProcess struct {
|
||||
mu sync.Mutex
|
||||
record protocol.TerminalRecord
|
||||
out *chatShellOutput
|
||||
titles terminalTitles
|
||||
// cmd and pty are the current attachment; a tmux client that exits while
|
||||
// its session lives is replaced.
|
||||
cmd *exec.Cmd
|
||||
pty *os.File
|
||||
input chan []byte
|
||||
done chan struct{}
|
||||
// closing is set by terminal_close: the end is a close, not an exit.
|
||||
closing bool
|
||||
}
|
||||
|
||||
func newTerminalProcess(record protocol.TerminalRecord) *terminalProcess {
|
||||
return &terminalProcess{
|
||||
record: record,
|
||||
out: newChatShellOutput(protocol.TerminalRetained),
|
||||
input: make(chan []byte, protocol.ChatShellInputQueue),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (p *terminalProcess) running() bool {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.record.Status == protocol.TerminalRunning
|
||||
}
|
||||
|
||||
// snapshot is the current record; with output, the retained bytes from
|
||||
// absolute byte after on.
|
||||
func (p *terminalProcess) snapshot(output bool, after int64) protocol.TerminalRecord {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
record := p.record
|
||||
record.OutputEnd = p.out.total
|
||||
if output {
|
||||
text, start := p.out.tail(protocol.TerminalRetained)
|
||||
if after > start {
|
||||
skip := min(after-start, int64(len(text)))
|
||||
cut := runeStartAfter([]byte(text), int(skip))
|
||||
text, start = text[cut:], start+int64(cut)
|
||||
}
|
||||
record.Output, record.OutputStart = text, start
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
// startAttachment starts cmd in a fresh pseudo-terminal at the record's
|
||||
// size and makes it the current attachment.
|
||||
func (p *terminalProcess) startAttachment(cmd *exec.Cmd) (*os.File, error) {
|
||||
if !chatShellTerminals {
|
||||
return nil, errors.New(protocol.ChatShellTerminalUnsupported)
|
||||
}
|
||||
p.mu.Lock()
|
||||
cols, rows := p.record.Cols, p.record.Rows
|
||||
p.mu.Unlock()
|
||||
terminal, err := startChatShellTerminal(cmd, cols, rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.cmd, p.pty = cmd, terminal
|
||||
p.mu.Unlock()
|
||||
return terminal, nil
|
||||
}
|
||||
|
||||
// read copies one attachment's stream into the retained output and follows
|
||||
// the window title; it returns when the attachment's terminal is closed.
|
||||
func (p *terminalProcess) read(terminal *os.File, titled func(string)) {
|
||||
buffer := make([]byte, 32<<10)
|
||||
for {
|
||||
n, err := terminal.Read(buffer)
|
||||
if n > 0 {
|
||||
p.mu.Lock()
|
||||
p.out.write(buffer[:n])
|
||||
title, changed := p.titles.feed(buffer[:n])
|
||||
changed = changed && title != p.record.Title
|
||||
if changed {
|
||||
p.record.Title = title
|
||||
}
|
||||
p.mu.Unlock()
|
||||
if changed {
|
||||
titled(title)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pumpInput writes queued input to the current attachment until the
|
||||
// terminal ends. Input arriving while a tmux client is replaced is dropped.
|
||||
func (p *terminalProcess) pumpInput() {
|
||||
for {
|
||||
select {
|
||||
case <-p.done:
|
||||
return
|
||||
case data := <-p.input:
|
||||
p.mu.Lock()
|
||||
terminal := p.pty
|
||||
p.mu.Unlock()
|
||||
if terminal != nil {
|
||||
_, _ = terminal.Write(data)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// signal sends sig to the current attachment's process group.
|
||||
func (p *terminalProcess) signal(sig syscall.Signal) {
|
||||
p.mu.Lock()
|
||||
cmd := p.cmd
|
||||
p.mu.Unlock()
|
||||
if cmd != nil && cmd.Process != nil {
|
||||
_ = syscall.Kill(-cmd.Process.Pid, sig)
|
||||
}
|
||||
}
|
||||
|
||||
// waitAttachment waits for the attachment to exit and its stream to drain
|
||||
// (a descendant holding the terminal is read briefly, then cut off), then
|
||||
// kills its group and closes its terminal. It returns the exit error.
|
||||
func (p *terminalProcess) waitAttachment(cmd *exec.Cmd, terminal *os.File, readDone <-chan struct{}) error {
|
||||
err := cmd.Wait()
|
||||
_ = terminal.SetReadDeadline(time.Now().Add(chatShellDrain))
|
||||
select {
|
||||
case <-readDone:
|
||||
case <-time.After(chatShellDrain + chatShellDrain/2):
|
||||
}
|
||||
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||
p.mu.Lock()
|
||||
terminal.Close()
|
||||
if p.pty == terminal {
|
||||
p.pty = nil
|
||||
}
|
||||
p.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
// exitCode is the shell's exit status, when it exited by itself.
|
||||
func terminalExitCode(err error) *int {
|
||||
code := 0
|
||||
var exitErr *exec.ExitError
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.As(err, &exitErr):
|
||||
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok && status.Signaled() {
|
||||
return nil
|
||||
}
|
||||
code = exitErr.ExitCode()
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return &code
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
//go:build darwin || linux
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// testTmux is a tmux server of this test alone, killed afterwards.
|
||||
func testTmux(t *testing.T) *terminalTmux {
|
||||
t.Helper()
|
||||
tmux := findTerminalTmux(fmt.Sprintf("pipelines-test-%d-%d", os.Getpid(), time.Now().UnixNano()))
|
||||
if tmux == nil {
|
||||
t.Skip("tmux is not installed")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = tmux.command("kill-server").Run()
|
||||
dir := os.Getenv("TMUX_TMPDIR")
|
||||
if dir == "" {
|
||||
dir = "/tmp"
|
||||
}
|
||||
_ = os.Remove(filepath.Join(dir, fmt.Sprintf("tmux-%d", os.Getuid()), tmux.socket))
|
||||
})
|
||||
return tmux
|
||||
}
|
||||
|
||||
// newTestTerminals is one server process's terminal manager over store.
|
||||
func newTestTerminals(t *testing.T, store *pipeline.Store, tmux *terminalTmux) (*terminalManager, context.CancelFunc) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
m := newTerminalManager(ctx, store, tmux)
|
||||
m.shell = func() string { return "/bin/sh" }
|
||||
m.recover(ctx)
|
||||
t.Cleanup(func() { cancel(); m.Wait() })
|
||||
return m, cancel
|
||||
}
|
||||
|
||||
// waitOutput polls a terminal's retained output until it contains want.
|
||||
func waitTerminalOutput(t *testing.T, m *terminalManager, id, want string) protocol.TerminalRecord {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
record, err := m.Output(context.Background(), id, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(record.Output, want) {
|
||||
return record
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("output of %s never contained %q: %q", id, want, record.Output)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// A tmux-backed shell outlives its server process: the next one lists it as
|
||||
// running, reattaches, redraws its screen and takes input again. Closing it
|
||||
// ends the tmux session.
|
||||
func TestTerminalTmuxSurvivesRestart(t *testing.T) {
|
||||
terminalHome(t)
|
||||
tmux := testTmux(t)
|
||||
store := newOperatorStore(t)
|
||||
first, stop := newTestTerminals(t, store, tmux)
|
||||
term, err := first.Open(context.Background(), "", "~", 90, 20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if term.Tmux != terminalTmuxName(term.ID) {
|
||||
t.Fatalf("tmux name = %q", term.Tmux)
|
||||
}
|
||||
if err := first.Input(term.ID, "export KEEP=still-here; echo marker-$((6*7))\r"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTerminalOutput(t, first, term.ID, "marker-42")
|
||||
stop()
|
||||
first.Wait()
|
||||
if !tmux.alive(term.Tmux) {
|
||||
t.Fatal("the tmux session ended with the server")
|
||||
}
|
||||
second, _ := newTestTerminals(t, store, tmux)
|
||||
list, err := second.List(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(list) != 1 || list[0].ID != term.ID || list[0].Status != protocol.TerminalRunning {
|
||||
t.Fatalf("list after restart = %+v", list)
|
||||
}
|
||||
// The reattached client redraws the screen, earlier output included,
|
||||
// and the shell keeps its state.
|
||||
waitTerminalOutput(t, second, term.ID, "marker-42")
|
||||
if err := second.Input(term.ID, "echo $KEEP\r"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTerminalOutput(t, second, term.ID, "still-here")
|
||||
closed, err := second.Close(context.Background(), term.ID)
|
||||
if err != nil || closed.Status != protocol.TerminalClosed {
|
||||
t.Fatalf("close = %+v, %v", closed, err)
|
||||
}
|
||||
if tmux.alive(term.Tmux) {
|
||||
t.Fatal("the tmux session outlived close")
|
||||
}
|
||||
if list, _ := second.List(context.Background()); len(list) != 0 {
|
||||
t.Fatalf("list after close = %+v", list)
|
||||
}
|
||||
}
|
||||
|
||||
// A tmux client killed while its session lives is replaced.
|
||||
func TestTerminalTmuxClientReplaced(t *testing.T) {
|
||||
terminalHome(t)
|
||||
tmux := testTmux(t)
|
||||
m, _ := newTestTerminals(t, newOperatorStore(t), tmux)
|
||||
term, err := m.Open(context.Background(), "", "", 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTerminalOutput(t, m, term.ID, "$")
|
||||
if err := tmux.command("detach-client", "-s", "="+term.Tmux).Run(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if err := m.Input(term.ID, "echo back-$((1+1))\r"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitTerminalOutput(t, m, term.ID, "back-2")
|
||||
if record, _ := m.Output(context.Background(), term.ID, 0); record.Status != protocol.TerminalRunning {
|
||||
t.Fatalf("record = %+v", record)
|
||||
}
|
||||
}
|
||||
|
||||
// Without tmux a shell ends with its server: the stop is recorded as
|
||||
// ended, and a record a crashed server left running is marked ended too.
|
||||
func TestTerminalWithoutTmuxEndsWithServer(t *testing.T) {
|
||||
terminalHome(t)
|
||||
store := newOperatorStore(t)
|
||||
first, stop := newTestTerminals(t, store, nil)
|
||||
term, err := first.Open(context.Background(), "", "", 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop()
|
||||
first.Wait()
|
||||
crashed := protocol.TerminalRecord{ID: "term_crashed", Cwd: "/", CreatedAt: 1, Status: protocol.TerminalRunning, Cols: 80, Rows: 24}
|
||||
gone := protocol.TerminalRecord{ID: "term_gone", Cwd: "/", CreatedAt: 2, Tmux: "pipelines-term_gone", Status: protocol.TerminalRunning}
|
||||
for _, record := range []protocol.TerminalRecord{crashed, gone} {
|
||||
if err := store.SaveTerminal(context.Background(), record, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
second, _ := newTestTerminals(t, store, testTmux(t))
|
||||
list, err := second.List(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reasons := map[string]string{}
|
||||
for _, record := range list {
|
||||
if record.Status != protocol.TerminalExited || record.EndedAt == 0 {
|
||||
t.Fatalf("record = %+v", record)
|
||||
}
|
||||
reasons[record.ID] = record.Reason
|
||||
}
|
||||
if reasons[term.ID] != terminalStopReason || reasons["term_crashed"] != terminalNoTmuxReason || reasons["term_gone"] != terminalGoneReason {
|
||||
t.Fatalf("reasons = %+v", reasons)
|
||||
}
|
||||
if _, err := second.Close(context.Background(), "term_gone"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if list, _ := second.List(context.Background()); len(list) != 2 {
|
||||
t.Fatalf("list after closing an ended terminal = %+v", list)
|
||||
}
|
||||
}
|
||||
|
||||
// The window title the shell sets is followed across reads and recorded.
|
||||
func TestTerminalTitles(t *testing.T) {
|
||||
var titles terminalTitles
|
||||
if _, ok := titles.feed([]byte("plain \x1b[1mtext")); ok {
|
||||
t.Fatal("title from plain text")
|
||||
}
|
||||
if _, ok := titles.feed([]byte("\x1b]0;~/src")); ok {
|
||||
t.Fatal("title before its end")
|
||||
}
|
||||
if title, ok := titles.feed([]byte("/app\x07more")); !ok || title != "~/src/app" {
|
||||
t.Fatalf("title = %q %v", title, ok)
|
||||
}
|
||||
if title, ok := titles.feed([]byte("\x1b]2;vim\x01 x\x1b\\")); !ok || title != "vim x" {
|
||||
t.Fatalf("ST title = %q %v", title, ok)
|
||||
}
|
||||
if _, ok := titles.feed([]byte("\x1b]8;;https://x\x07")); ok {
|
||||
t.Fatal("a hyperlink is not a title")
|
||||
}
|
||||
if title, _ := titles.feed([]byte("\x1b]0;" + strings.Repeat("é", protocol.TerminalTitleLimit) + "\x07")); len(title) > protocol.TerminalTitleLimit {
|
||||
t.Fatalf("title of %d bytes", len(title))
|
||||
}
|
||||
terminalHome(t)
|
||||
m, _ := newTestTerminals(t, newOperatorStore(t), nil)
|
||||
term, err := m.Open(context.Background(), "", "", 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.Input(term.ID, "printf '\\033]0;build-%s\\007' ok\r"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
list, _ := m.List(context.Background())
|
||||
if len(list) == 1 && list[0].Title == "build-ok" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("list = %+v", list)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// supervise streams one terminal until it ends: its first attachment is
|
||||
// cmd on terminal. A tmux client that exits while its session lives (it was
|
||||
// detached or killed) is replaced; a server shutdown leaves a tmux session
|
||||
// running for the next server process and ends a direct shell.
|
||||
func (m *terminalManager) supervise(p *terminalProcess, cmd *exec.Cmd, terminal *os.File) {
|
||||
defer m.wg.Done()
|
||||
defer close(p.done)
|
||||
stopFlush, flushDone := make(chan struct{}), make(chan struct{})
|
||||
go m.flushLoop(p, stopFlush, flushDone)
|
||||
titled := func(string) {
|
||||
record := p.snapshot(false, 0)
|
||||
m.save(record)
|
||||
m.publishRecord(record)
|
||||
}
|
||||
var waitErr error
|
||||
shutdown := false
|
||||
for reattached := 0; ; reattached++ {
|
||||
readDone := make(chan struct{})
|
||||
go func(terminal *os.File) {
|
||||
defer close(readDone)
|
||||
p.read(terminal, titled)
|
||||
}(terminal)
|
||||
attachDone := make(chan error, 1)
|
||||
go func(cmd *exec.Cmd, terminal *os.File) { attachDone <- p.waitAttachment(cmd, terminal, readDone) }(cmd, terminal)
|
||||
select {
|
||||
case waitErr = <-attachDone:
|
||||
case <-m.base.Done():
|
||||
p.signal(syscall.SIGHUP)
|
||||
select {
|
||||
case waitErr = <-attachDone:
|
||||
case <-time.After(time.Second):
|
||||
p.signal(syscall.SIGKILL)
|
||||
waitErr = <-attachDone
|
||||
}
|
||||
}
|
||||
if m.base.Err() != nil {
|
||||
shutdown = true
|
||||
break
|
||||
}
|
||||
p.mu.Lock()
|
||||
closing, name := p.closing, p.record.Tmux
|
||||
p.mu.Unlock()
|
||||
if closing || name == "" || m.tmux == nil || reattached >= terminalReattachLimit || !m.tmux.alive(name) {
|
||||
break
|
||||
}
|
||||
cmd = m.tmux.attach(name)
|
||||
next, err := p.startAttachment(cmd)
|
||||
if err != nil {
|
||||
log.Printf("terminal reattach %s: %v", p.record.ID, err)
|
||||
break
|
||||
}
|
||||
terminal = next
|
||||
}
|
||||
close(stopFlush)
|
||||
<-flushDone
|
||||
m.emit(p, true)
|
||||
p.mu.Lock()
|
||||
switch {
|
||||
case shutdown && p.record.Tmux != "":
|
||||
// Still running in tmux: the next server process reattaches it.
|
||||
case p.closing:
|
||||
p.record.Status = protocol.TerminalClosed
|
||||
default:
|
||||
p.record.Status = protocol.TerminalExited
|
||||
p.record.EndedAt = m.now().UnixMilli()
|
||||
if shutdown {
|
||||
p.record.Reason = terminalStopReason
|
||||
} else if p.record.Tmux == "" {
|
||||
p.record.ExitCode = terminalExitCode(waitErr)
|
||||
}
|
||||
}
|
||||
p.pty, p.cmd = nil, nil
|
||||
p.mu.Unlock()
|
||||
record := p.snapshot(false, 0)
|
||||
switch record.Status {
|
||||
case protocol.TerminalClosed:
|
||||
m.forget(record.ID)
|
||||
default:
|
||||
m.save(record)
|
||||
}
|
||||
if !shutdown {
|
||||
m.publishRecord(record)
|
||||
}
|
||||
}
|
||||
|
||||
// flushLoop pushes new output in batches until stop is closed.
|
||||
func (m *terminalManager) flushLoop(p *terminalProcess, stop <-chan struct{}, done chan<- struct{}) {
|
||||
defer close(done)
|
||||
ticker := time.NewTicker(m.flush)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
m.emit(p, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// emit pushes the output written since the last frame.
|
||||
func (m *terminalManager) emit(p *terminalProcess, final bool) {
|
||||
p.mu.Lock()
|
||||
text, offset, end, ok := p.out.pending(protocol.TerminalChunkLimit, final)
|
||||
id := p.record.ID
|
||||
p.mu.Unlock()
|
||||
if ok {
|
||||
m.clients.publish(protocol.TerminalEventFrame{TerminalID: id, Output: text, Offset: offset, End: end})
|
||||
}
|
||||
}
|
||||
|
||||
// recover lists what a previous server process left running: a tmux-backed
|
||||
// shell whose session is alive is reattached; any other is recorded as
|
||||
// ended, saying why.
|
||||
func (m *terminalManager) recover(ctx context.Context) {
|
||||
if m.store == nil {
|
||||
return
|
||||
}
|
||||
records, err := m.store.Terminals(ctx)
|
||||
if err != nil {
|
||||
log.Printf("terminal recover: %v", err)
|
||||
return
|
||||
}
|
||||
for _, record := range records {
|
||||
if record.Status != protocol.TerminalRunning {
|
||||
continue
|
||||
}
|
||||
if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) && m.reattach(record) {
|
||||
continue
|
||||
}
|
||||
record.Status, record.EndedAt = protocol.TerminalExited, m.now().UnixMilli()
|
||||
record.Reason = terminalNoTmuxReason
|
||||
if record.Tmux != "" {
|
||||
record.Reason = terminalGoneReason
|
||||
}
|
||||
m.save(record)
|
||||
}
|
||||
}
|
||||
|
||||
// reattach shows a live tmux session in a new client and supervises it.
|
||||
func (m *terminalManager) reattach(record protocol.TerminalRecord) bool {
|
||||
cmd := m.tmux.attach(record.Tmux)
|
||||
p := newTerminalProcess(record)
|
||||
terminal, err := p.startAttachment(cmd)
|
||||
if err != nil {
|
||||
log.Printf("terminal reattach %s: %v", record.ID, err)
|
||||
return false
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.terms[record.ID] = p
|
||||
m.mu.Unlock()
|
||||
m.wg.Add(1)
|
||||
go m.supervise(p, cmd, terminal)
|
||||
go p.pumpInput()
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// terminalTitles follows a terminal stream across reads and reports the
|
||||
// window title the program sets (OSC 0 or OSC 2, ended by BEL or ST), so a
|
||||
// listed terminal keeps a name before any client draws it.
|
||||
type terminalTitles struct {
|
||||
state int
|
||||
osc []byte
|
||||
}
|
||||
|
||||
const (
|
||||
titleText = iota
|
||||
titleEscape
|
||||
titleOSC
|
||||
titleOSCEscape
|
||||
)
|
||||
|
||||
// feed scans data and returns the last title it completed, if any.
|
||||
func (t *terminalTitles) feed(data []byte) (string, bool) {
|
||||
title, found := "", false
|
||||
for _, b := range data {
|
||||
switch t.state {
|
||||
case titleText:
|
||||
if b == 0x1b {
|
||||
t.state = titleEscape
|
||||
}
|
||||
case titleEscape:
|
||||
if b == ']' {
|
||||
t.state, t.osc = titleOSC, t.osc[:0]
|
||||
} else if b != 0x1b {
|
||||
t.state = titleText
|
||||
}
|
||||
case titleOSC:
|
||||
switch {
|
||||
case b == 0x07:
|
||||
if s, ok := t.finish(); ok {
|
||||
title, found = s, true
|
||||
}
|
||||
case b == 0x1b:
|
||||
t.state = titleOSCEscape
|
||||
case len(t.osc) < 4*protocol.TerminalTitleLimit:
|
||||
t.osc = append(t.osc, b)
|
||||
}
|
||||
case titleOSCEscape:
|
||||
if b == '\\' {
|
||||
if s, ok := t.finish(); ok {
|
||||
title, found = s, true
|
||||
}
|
||||
} else {
|
||||
t.state = titleEscape
|
||||
}
|
||||
}
|
||||
}
|
||||
return title, found
|
||||
}
|
||||
|
||||
func (t *terminalTitles) finish() (string, bool) {
|
||||
t.state = titleText
|
||||
body := string(t.osc)
|
||||
for _, prefix := range []string{"0;", "2;"} {
|
||||
if strings.HasPrefix(body, prefix) {
|
||||
return cleanTerminalTitle(body[len(prefix):]), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// cleanTerminalTitle keeps printable text within the title bound.
|
||||
func cleanTerminalTitle(title string) string {
|
||||
title = strings.ToValidUTF8(title, "")
|
||||
title = strings.Map(func(r rune) rune {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, title)
|
||||
title = strings.TrimSpace(title)
|
||||
for len(title) > protocol.TerminalTitleLimit {
|
||||
_, size := utf8.DecodeLastRuneInString(title)
|
||||
title = title[:len(title)-size]
|
||||
}
|
||||
return title
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// terminalTmuxSocket is the tmux server Pipelines owns (`tmux -L pipelines`):
|
||||
// its sessions never mix with the operator's own tmux server.
|
||||
const terminalTmuxSocket = "pipelines"
|
||||
|
||||
// terminalTmux runs one Pipelines-owned tmux server. Every terminal shell is
|
||||
// the only pane of its own session there, so it outlives the Pipelines
|
||||
// server; the server talks to it through an ordinary attached client in a
|
||||
// pseudo-terminal.
|
||||
type terminalTmux struct {
|
||||
program string
|
||||
socket string
|
||||
}
|
||||
|
||||
// findTerminalTmux is the tmux on PATH (or the usual package-manager
|
||||
// locations a service's PATH may lack); nil when none is installed.
|
||||
func findTerminalTmux(socket string) *terminalTmux {
|
||||
if socket == "" {
|
||||
socket = terminalTmuxSocket
|
||||
}
|
||||
program, err := exec.LookPath("tmux")
|
||||
if err != nil {
|
||||
for _, dir := range []string{"/opt/homebrew/bin", "/usr/local/bin", "/usr/bin"} {
|
||||
candidate := filepath.Join(dir, "tmux")
|
||||
if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() && info.Mode()&0o111 != 0 {
|
||||
program, err = candidate, nil
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return &terminalTmux{program: program, socket: socket}
|
||||
}
|
||||
|
||||
// terminalTmuxName is the tmux session of terminal id.
|
||||
func terminalTmuxName(id string) string { return "pipelines-" + id }
|
||||
|
||||
// command is one tmux client invocation against the Pipelines server, with
|
||||
// no configuration file and outside any tmux the server itself runs in.
|
||||
func (t *terminalTmux) command(args ...string) *exec.Cmd {
|
||||
cmd := exec.Command(t.program, append([]string{"-L", t.socket, "-f", os.DevNull}, args...)...)
|
||||
cmd.Env = terminalEnv(os.Environ(), "")
|
||||
return cmd
|
||||
}
|
||||
|
||||
// create starts session name running shell -l in cwd at cols x rows,
|
||||
// detached. The first call also starts the tmux server; the options make
|
||||
// it invisible (no status line), pass the shell's window title through,
|
||||
// keep a bounded scrollback reachable by the mouse wheel, and leave Esc
|
||||
// prompt.
|
||||
func (t *terminalTmux) create(name, cwd, shell string, cols, rows int) error {
|
||||
cmd := t.command(
|
||||
"start-server", ";",
|
||||
"set-option", "-g", "history-limit", strconv.Itoa(protocol.TerminalTmuxHistory), ";",
|
||||
"set-option", "-g", "status", "off", ";",
|
||||
"set-option", "-g", "mouse", "on", ";",
|
||||
"set-option", "-g", "escape-time", "10", ";",
|
||||
"set-option", "-g", "set-titles", "on", ";",
|
||||
"set-option", "-g", "set-titles-string", "#{pane_title}", ";",
|
||||
"new-session", "-d", "-s", name, "-c", cwd, "-x", strconv.Itoa(cols), "-y", strconv.Itoa(rows), "--", shell, "-l", ";",
|
||||
"select-pane", "-t", "="+name+":", "-T", "",
|
||||
)
|
||||
cmd.Dir = cwd
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("tmux new-session: %v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// alive reports whether session name still exists.
|
||||
func (t *terminalTmux) alive(name string) bool {
|
||||
return t.command("has-session", "-t", "="+name).Run() == nil
|
||||
}
|
||||
|
||||
// attach is the client command showing session name in the server's
|
||||
// pseudo-terminal.
|
||||
func (t *terminalTmux) attach(name string) *exec.Cmd {
|
||||
return t.command("attach-session", "-t", "="+name)
|
||||
}
|
||||
|
||||
// kill ends session name and its shell.
|
||||
func (t *terminalTmux) kill(name string) error {
|
||||
if out, err := t.command("kill-session", "-t", "="+name).CombinedOutput(); err != nil && t.alive(name) {
|
||||
return fmt.Errorf("tmux kill-session: %v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// terminalEnv is env for a terminal's process: TERM names the emulator the
|
||||
// clients draw, PWD the start directory (when set), and no TMUX variables
|
||||
// leak in from a tmux the server runs inside.
|
||||
func terminalEnv(env []string, cwd string) []string {
|
||||
out := make([]string, 0, len(env)+2)
|
||||
for _, entry := range env {
|
||||
if strings.HasPrefix(entry, "TERM=") || strings.HasPrefix(entry, "TMUX=") || strings.HasPrefix(entry, "TMUX_PANE=") ||
|
||||
(cwd != "" && strings.HasPrefix(entry, "PWD=")) {
|
||||
continue
|
||||
}
|
||||
out = append(out, entry)
|
||||
}
|
||||
out = append(out, "TERM=xterm-256color")
|
||||
if cwd != "" {
|
||||
out = append(out, "PWD="+cwd)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
//go:build darwin || linux
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// terminalHome points the operator's home at a fresh directory and the
|
||||
// login shell at /bin/sh, so tests never read the operator's own profile.
|
||||
func terminalHome(t *testing.T) string {
|
||||
t.Helper()
|
||||
home, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original := operatorHomeDirectory
|
||||
operatorHomeDirectory = func() (string, error) { return home, nil }
|
||||
t.Cleanup(func() { operatorHomeDirectory = original })
|
||||
t.Setenv("SHELL", "/bin/sh")
|
||||
t.Setenv("ENV", "")
|
||||
return home
|
||||
}
|
||||
|
||||
func terminalRequest(kind, id string, body protocol.TerminalRequest) protocol.Message {
|
||||
return protocol.Message{Type: kind, ID: id, TerminalRequest: &body}
|
||||
}
|
||||
|
||||
// terminalReply sends request and returns its reply (skipping pushes).
|
||||
func terminalReply(t *testing.T, ctx context.Context, conn *websocket.Conn, request protocol.Message) protocol.Message {
|
||||
t.Helper()
|
||||
writeChat(t, ctx, conn, request)
|
||||
return readShellFrame(t, ctx, conn, func(m protocol.Message) bool { return m.ID == request.ID })
|
||||
}
|
||||
|
||||
// readTerminalUntil reads output frames of id from end on, in order, until
|
||||
// the collected text contains want; it returns the text and the new end.
|
||||
func readTerminalUntil(t *testing.T, ctx context.Context, conn *websocket.Conn, id string, end int64, want string) (string, int64) {
|
||||
t.Helper()
|
||||
var seen strings.Builder
|
||||
for !strings.Contains(seen.String(), want) {
|
||||
frame := readShellFrame(t, ctx, conn, func(m protocol.Message) bool {
|
||||
return m.Type == protocol.TerminalEvent && m.TerminalEvent.TerminalID == id && m.TerminalEvent.Terminal == nil
|
||||
}).TerminalEvent
|
||||
if frame.Offset != end {
|
||||
t.Fatalf("frame at %d after %d", frame.Offset, end)
|
||||
}
|
||||
seen.WriteString(frame.Output)
|
||||
end = frame.End
|
||||
}
|
||||
return seen.String(), end
|
||||
}
|
||||
|
||||
// One client opens a shell, types into it, resizes it; a second client
|
||||
// lists it, resumes its output by offset and closes it.
|
||||
func TestTerminalOverWebsocket(t *testing.T) {
|
||||
home := terminalHome(t)
|
||||
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 2)
|
||||
opened := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{Cols: 80, Rows: 24}))
|
||||
term := opened.Terminal
|
||||
if opened.Type != protocol.TerminalOpen || term == nil || term.Cwd != home || term.Status != protocol.TerminalRunning || term.Cols != 80 || term.Tmux != "" || term.Shell != "/bin/sh" {
|
||||
t.Fatalf("open reply = %+v", opened)
|
||||
}
|
||||
if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "in", protocol.TerminalRequest{TerminalID: term.ID, Data: "pwd; echo sum-$((40+2))\r"})); got.Type != protocol.TerminalInput {
|
||||
t.Fatalf("input reply = %+v", got)
|
||||
}
|
||||
_, end := readTerminalUntil(t, ctx, conns[0], term.ID, 0, "sum-42")
|
||||
resized := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalResize, "size", protocol.TerminalRequest{TerminalID: term.ID, Cols: 120, Rows: 33}))
|
||||
if resized.Type != protocol.TerminalResize || resized.Terminal.Cols != 120 || resized.Terminal.Rows != 33 {
|
||||
t.Fatalf("resize reply = %+v", resized)
|
||||
}
|
||||
list := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalList, "list", protocol.TerminalRequest{}))
|
||||
if len(list.Terminals) != 1 || list.Terminals[0].ID != term.ID || list.Terminals[0].Cols != 120 || list.Terminals[0].Output != "" {
|
||||
t.Fatalf("list = %+v", list.Terminals)
|
||||
}
|
||||
// The second client reads everything once, then only what follows.
|
||||
all := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalOutput, "all", protocol.TerminalRequest{TerminalID: term.ID})).Terminal
|
||||
if all.OutputStart != 0 || !strings.Contains(all.Output, home) || !strings.Contains(all.Output, "sum-42") {
|
||||
t.Fatalf("output = %+v", all)
|
||||
}
|
||||
terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalInput, "in2", protocol.TerminalRequest{TerminalID: term.ID, Data: "stty size\r"}))
|
||||
readTerminalUntil(t, ctx, conns[0], term.ID, end, "33 120")
|
||||
resume := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalOutput, "resume", protocol.TerminalRequest{TerminalID: term.ID, After: end})).Terminal
|
||||
if resume.OutputStart != end || strings.Contains(resume.Output, "sum-42") || !strings.Contains(resume.Output, "33 120") {
|
||||
t.Fatalf("resume = %+v", resume)
|
||||
}
|
||||
closed := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: term.ID}))
|
||||
if closed.Type != protocol.TerminalClose || closed.Terminal.Status != protocol.TerminalClosed {
|
||||
t.Fatalf("close reply = %+v", closed)
|
||||
}
|
||||
// Every connection hears the close; the list forgets it.
|
||||
event := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool {
|
||||
return m.Type == protocol.TerminalEvent && m.TerminalEvent.Terminal != nil && m.TerminalEvent.Terminal.Status == protocol.TerminalClosed
|
||||
})
|
||||
if event.TerminalEvent.TerminalID != term.ID {
|
||||
t.Fatalf("close event = %+v", event)
|
||||
}
|
||||
if list := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalList, "list2", protocol.TerminalRequest{})); len(list.Terminals) != 0 {
|
||||
t.Fatalf("list after close = %+v", list.Terminals)
|
||||
}
|
||||
if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "late", protocol.TerminalRequest{TerminalID: term.ID, Data: "x"})); got.Type != protocol.Error {
|
||||
t.Fatalf("late input = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A shell that exits by itself is listed as exited with its status and
|
||||
// keeps its output until it is closed.
|
||||
func TestTerminalShellExit(t *testing.T) {
|
||||
terminalHome(t)
|
||||
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 1)
|
||||
term := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{})).Terminal
|
||||
terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "in", protocol.TerminalRequest{TerminalID: term.ID, Data: "echo bye; exit 3\r"}))
|
||||
exited := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool {
|
||||
return m.Type == protocol.TerminalEvent && m.TerminalEvent.Terminal != nil && m.TerminalEvent.Terminal.Status == protocol.TerminalExited
|
||||
}).TerminalEvent.Terminal
|
||||
if exited.ExitCode == nil || *exited.ExitCode != 3 || exited.EndedAt == 0 {
|
||||
t.Fatalf("exited = %+v", exited)
|
||||
}
|
||||
out := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOutput, "out", protocol.TerminalRequest{TerminalID: term.ID})).Terminal
|
||||
if !strings.Contains(out.Output, "bye") || out.Status != protocol.TerminalExited {
|
||||
t.Fatalf("output = %+v", out)
|
||||
}
|
||||
if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: term.ID})); got.Terminal == nil || got.Terminal.Status != protocol.TerminalClosed {
|
||||
t.Fatalf("close = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Requests outside the bounds are refused: directories outside home or
|
||||
// missing, input bounds, unknown terminals, the open limit.
|
||||
func TestTerminalLimitsAndValidation(t *testing.T) {
|
||||
home := terminalHome(t)
|
||||
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 1)
|
||||
conn := conns[0]
|
||||
for i, cwd := range []string{"/", filepath.Join(home, "missing"), "relative", "~other"} {
|
||||
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "bad", protocol.TerminalRequest{Cwd: cwd})); got.Type != protocol.Error {
|
||||
t.Fatalf("cwd %d %q = %+v", i, cwd, got)
|
||||
}
|
||||
}
|
||||
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOutput, "none", protocol.TerminalRequest{TerminalID: "term_nope"})); got.Type != protocol.Error {
|
||||
t.Fatalf("unknown = %+v", got)
|
||||
}
|
||||
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalInput, "noid", protocol.TerminalRequest{Data: "x"})); got.Type != protocol.Error {
|
||||
t.Fatalf("no id = %+v", got)
|
||||
}
|
||||
var ids []string
|
||||
for i := 0; i < protocol.TerminalOpenLimit; i++ {
|
||||
got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{Cwd: "~"}))
|
||||
if got.Terminal == nil {
|
||||
t.Fatalf("open %d = %+v", i, got)
|
||||
}
|
||||
ids = append(ids, got.Terminal.ID)
|
||||
}
|
||||
over := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "over", protocol.TerminalRequest{}))
|
||||
if over.Type != protocol.Error || !strings.Contains(over.Error, "8 open terminals") {
|
||||
t.Fatalf("over limit = %+v", over)
|
||||
}
|
||||
for _, bad := range []string{"", strings.Repeat("x", protocol.ChatShellInputLimit+1)} {
|
||||
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalInput, "bad-in", protocol.TerminalRequest{TerminalID: ids[0], Data: bad})); got.Type != protocol.Error {
|
||||
t.Fatalf("input %q = %+v", bad[:min(len(bad), 4)], got)
|
||||
}
|
||||
}
|
||||
terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalClose, "close", protocol.TerminalRequest{TerminalID: ids[0]}))
|
||||
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "again", protocol.TerminalRequest{})); got.Terminal == nil {
|
||||
t.Fatalf("open after close = %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,12 @@ type Options struct {
|
||||
// False keeps the previous behaviour (tests, embedded servers): a stopping
|
||||
// server interrupts its runs and the next start marks them interrupted.
|
||||
DetachSessions bool
|
||||
// TerminalTmuxSocket names the Pipelines-owned tmux server machine
|
||||
// terminals live in (empty: "pipelines"); TerminalWithoutTmux runs
|
||||
// shells as direct children even where tmux is installed. Tests set
|
||||
// them so they never share the real server's tmux sessions.
|
||||
TerminalTmuxSocket string
|
||||
TerminalWithoutTmux bool
|
||||
}
|
||||
|
||||
// Handler serves the persistent connection with default options. Canceling ctx
|
||||
@@ -143,6 +149,20 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
} else if store, ok := registry.(chatShellRunStore); ok {
|
||||
hub.startShellRunner(ctx, store)
|
||||
}
|
||||
// Machine terminals are recorded beside the operator sessions, in the
|
||||
// same store; a previous process's tmux-backed shells are reattached.
|
||||
var terminalRecords terminalStore
|
||||
if len(stores) > 0 {
|
||||
terminalRecords = stores[0]
|
||||
} else if store, ok := registry.(terminalStore); ok {
|
||||
terminalRecords = store
|
||||
}
|
||||
var terminalTmuxServer *terminalTmux
|
||||
if !options.TerminalWithoutTmux {
|
||||
terminalTmuxServer = findTerminalTmux(options.TerminalTmuxSocket)
|
||||
}
|
||||
terminals := newTerminalManager(ctx, terminalRecords, terminalTmuxServer)
|
||||
terminals.recover(ctx)
|
||||
acks := newAttentionAckHub()
|
||||
var ackStore attentionAckStore
|
||||
var notes *appNotifier
|
||||
@@ -237,6 +257,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
// connection the same way.
|
||||
hub.sessionChanges.attach(clientID, write)
|
||||
defer hub.sessionChanges.detach(clientID)
|
||||
// Terminal lifecycle changes reach every connection; output only the
|
||||
// connections that opened or read that terminal.
|
||||
terminals.clients.attach(clientID, write)
|
||||
defer terminals.clients.detach(clientID)
|
||||
if notes != nil {
|
||||
defer notes.detach(clientID)
|
||||
}
|
||||
@@ -250,7 +274,8 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
if err := wsjson.Read(connCtx, conn, &request); err != nil {
|
||||
return
|
||||
}
|
||||
if request.Type != protocol.ChatShellInput && request.Type != protocol.ChatShellResize {
|
||||
if request.Type != protocol.ChatShellInput && request.Type != protocol.ChatShellResize &&
|
||||
request.Type != protocol.TerminalInput && request.Type != protocol.TerminalResize {
|
||||
// Keystrokes would be one line each; their data is never logged.
|
||||
log.Printf("command received: client=%s type=%q id=%q pipeline=%d version=%d", r.RemoteAddr, request.Type, request.ID, request.PipelineID, request.Version)
|
||||
}
|
||||
@@ -291,6 +316,16 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
}
|
||||
continue
|
||||
}
|
||||
if isTerminalRequest(request.Type) {
|
||||
// Closing waits for the shell to go; it never delays input
|
||||
// to the other terminals behind it.
|
||||
if request.Type == protocol.TerminalClose {
|
||||
go handleTerminalRequest(connCtx, request, terminals, clientID, write)
|
||||
} else {
|
||||
handleTerminalRequest(connCtx, request, terminals, clientID, write)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if handleDirectoryListing(request, write) {
|
||||
continue
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user