03123c23fd
chat_shell_dismiss hides finished runs from every readback and pushes a dismissal frame to every watcher; chat_shell_restore undoes it within a 30 s window, after which the command, output and error are purged and only a tombstone remains, reported by chat_shell_runs for clients that missed the event.
273 lines
9.3 KiB
Go
273 lines
9.3 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// chatShellRunStore persists operator shell runs; *pipeline.Store implements it.
|
|
type chatShellRunStore interface {
|
|
SaveChatShellRun(ctx context.Context, run protocol.ChatShellRunRecord, keep int) error
|
|
ChatShellRuns(ctx context.Context, sessionID string) ([]protocol.ChatShellRunRecord, error)
|
|
ChatShellRun(ctx context.Context, sessionID, id string) (protocol.ChatShellRunRecord, bool, error)
|
|
InterruptChatShellRuns(ctx context.Context, endedAt int64, reason string) (int64, error)
|
|
chatShellDismissStore
|
|
}
|
|
|
|
const (
|
|
chatShellInterruptedReason = "The server stopped while this command was running."
|
|
// chatShellDrain is how long output is still read after the command itself
|
|
// exited, for descendants that keep the pipe open; the group is then killed.
|
|
chatShellDrain = time.Second
|
|
)
|
|
|
|
// chatShellRunner owns the operator's own commands: each runs `$SHELL -lc` in
|
|
// its chat's directory, in its own process group, in a pseudo-terminal where
|
|
// the platform has one (else with stdout and stderr on one pipe), its output
|
|
// kept as one bounded tail that is streamed to the session's watchers and
|
|
// persisted as Pipelines' record of the run.
|
|
type chatShellRunner struct {
|
|
base context.Context
|
|
store chatShellRunStore
|
|
publish func(sessionID string, frame protocol.ChatShellRunEventFrame)
|
|
now func() time.Time
|
|
shell func() string
|
|
// grace separates Stop's SIGINT from its SIGKILL; flush batches live output
|
|
// frames; persist is the interval a running run's output is saved at.
|
|
grace time.Duration
|
|
flush time.Duration
|
|
persist time.Duration
|
|
// undo is how long a dismissed run stays restorable before its purge.
|
|
undo time.Duration
|
|
|
|
mu sync.Mutex
|
|
active map[string]*chatShellProcess
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
type chatShellProcess struct {
|
|
mu sync.Mutex
|
|
record protocol.ChatShellRunRecord
|
|
out *chatShellOutput
|
|
cmd *exec.Cmd
|
|
stopped bool
|
|
done chan struct{}
|
|
// terminal is the pseudo-terminal's controller side (nil in pipe mode);
|
|
// input feeds it in order (chat_shell_terminal.go).
|
|
terminal *os.File
|
|
input chan []byte
|
|
// closed is set once the reader is closed; a read still blocked on a
|
|
// terminal a descendant holds open records nothing after it.
|
|
closed bool
|
|
}
|
|
|
|
func newChatShellRunner(base context.Context, store chatShellRunStore, publish func(string, protocol.ChatShellRunEventFrame)) *chatShellRunner {
|
|
return &chatShellRunner{
|
|
base: base, store: store, publish: publish, now: time.Now, shell: chatShellProgram,
|
|
grace: 3 * time.Second, flush: 100 * time.Millisecond, persist: 2 * time.Second,
|
|
undo: protocol.ChatShellRunUndoWindow * time.Millisecond,
|
|
active: map[string]*chatShellProcess{},
|
|
}
|
|
}
|
|
|
|
// chatShellProgram is the operator's login shell, or /bin/sh without one.
|
|
func chatShellProgram() string {
|
|
if shell := os.Getenv("SHELL"); filepath.IsAbs(shell) {
|
|
if info, err := os.Stat(shell); err == nil && !info.IsDir() {
|
|
return shell
|
|
}
|
|
}
|
|
return "/bin/sh"
|
|
}
|
|
|
|
// recover marks runs a previous server process left running as interrupted:
|
|
// their processes died with it (or are no longer owned by anyone). It also
|
|
// purges dismissals whose undo window passed while no server was running.
|
|
func (r *chatShellRunner) recover(ctx context.Context) {
|
|
if r.store == nil {
|
|
return
|
|
}
|
|
_, _ = r.store.InterruptChatShellRuns(ctx, r.now().UnixMilli(), chatShellInterruptedReason)
|
|
r.purgeDismissed(ctx)
|
|
}
|
|
|
|
// resolveChatShellDirectory requires the chat's recorded directory to still be
|
|
// an existing directory and returns its symlink-free path.
|
|
func resolveChatShellDirectory(directory string) (string, error) {
|
|
directory = strings.TrimSpace(directory)
|
|
if directory == "" || !filepath.IsAbs(directory) {
|
|
return "", errors.New("this chat has no absolute working directory to run commands in")
|
|
}
|
|
real, err := filepath.EvalSymlinks(directory)
|
|
if err != nil {
|
|
return "", fmt.Errorf("the chat's directory %s is not available: %v", directory, err)
|
|
}
|
|
info, err := os.Stat(real)
|
|
if err != nil || !info.IsDir() {
|
|
return "", fmt.Errorf("the chat's directory %s is not a directory", directory)
|
|
}
|
|
return real, nil
|
|
}
|
|
|
|
func newChatShellRunID() (string, error) {
|
|
var b [12]byte
|
|
if _, err := rand.Read(b[:]); err != nil {
|
|
return "", err
|
|
}
|
|
return "run_" + hex.EncodeToString(b[:]), nil
|
|
}
|
|
|
|
// Start launches command for sessionID in directory and returns its running
|
|
// record. Validation and start failures are returned without a record.
|
|
// cols and rows size its terminal (zero picks the defaults).
|
|
func (r *chatShellRunner) Start(ctx context.Context, sessionID, directory, command string, cols, rows int) (protocol.ChatShellRunRecord, error) {
|
|
if r.store == nil {
|
|
return protocol.ChatShellRunRecord{}, errors.New("shell commands need the server's store, which is not configured")
|
|
}
|
|
if strings.TrimSpace(command) == "" {
|
|
return protocol.ChatShellRunRecord{}, errors.New("a nonempty command is required")
|
|
}
|
|
if len(command) > protocol.ChatShellRunCommandLimit {
|
|
return protocol.ChatShellRunRecord{}, fmt.Errorf("the command exceeds %d KiB", protocol.ChatShellRunCommandLimit>>10)
|
|
}
|
|
dir, err := resolveChatShellDirectory(directory)
|
|
if err != nil {
|
|
return protocol.ChatShellRunRecord{}, err
|
|
}
|
|
id, err := newChatShellRunID()
|
|
if err != nil {
|
|
return protocol.ChatShellRunRecord{}, err
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.base.Err() != nil {
|
|
return protocol.ChatShellRunRecord{}, errors.New("the server is stopping")
|
|
}
|
|
running := 0
|
|
for _, p := range r.active {
|
|
if p.record.SessionID == sessionID {
|
|
running++
|
|
}
|
|
}
|
|
if running >= protocol.ChatShellRunConcurrent {
|
|
return protocol.ChatShellRunRecord{}, fmt.Errorf("this chat already runs %d commands; stop one or wait for it to finish", running)
|
|
}
|
|
cmd := exec.Command(r.shell(), "-lc", command)
|
|
cmd.Dir = dir
|
|
cols, rows = chatShellTerminalSize(cols, rows)
|
|
reader, terminal, err := startChatShellCommand(cmd, cols, rows)
|
|
if err != nil {
|
|
return protocol.ChatShellRunRecord{}, fmt.Errorf("could not start the command: %v", err)
|
|
}
|
|
record := protocol.ChatShellRunRecord{ID: id, SessionID: sessionID, Command: command, Directory: dir, Status: protocol.ChatShellRunRunning, StartedAt: r.now().UnixMilli()}
|
|
if terminal != nil {
|
|
record.Terminal, record.Cols, record.Rows = true, cols, rows
|
|
}
|
|
p := &chatShellProcess{record: record, out: newChatShellOutput(protocol.ChatShellRunRetained), cmd: cmd, terminal: terminal, done: make(chan struct{})}
|
|
if err := r.store.SaveChatShellRun(ctx, record, protocol.ChatShellRunHistory); err != nil {
|
|
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
|
reader.Close()
|
|
_ = cmd.Wait()
|
|
return protocol.ChatShellRunRecord{}, fmt.Errorf("could not record the command: %v", err)
|
|
}
|
|
r.active[id] = p
|
|
r.wg.Add(1)
|
|
go r.supervise(p, reader)
|
|
if terminal != nil {
|
|
p.input = make(chan []byte, protocol.ChatShellInputQueue)
|
|
go p.pumpInput()
|
|
}
|
|
r.publishRun(record)
|
|
return record, nil
|
|
}
|
|
|
|
// Stop interrupts a running command's whole process group, then kills it
|
|
// after the grace period. It returns the run's current record.
|
|
func (r *chatShellRunner) Stop(sessionID, runID string) (protocol.ChatShellRunRecord, error) {
|
|
r.mu.Lock()
|
|
p := r.active[runID]
|
|
r.mu.Unlock()
|
|
if p == nil || p.record.SessionID != sessionID {
|
|
return protocol.ChatShellRunRecord{}, errors.New("this command is not running")
|
|
}
|
|
p.mu.Lock()
|
|
p.stopped = true
|
|
record := p.record
|
|
p.mu.Unlock()
|
|
pid := p.cmd.Process.Pid
|
|
_ = syscall.Kill(-pid, syscall.SIGINT)
|
|
go func() {
|
|
select {
|
|
case <-p.done:
|
|
case <-time.After(r.grace):
|
|
_ = syscall.Kill(-pid, syscall.SIGKILL)
|
|
}
|
|
}()
|
|
return record, nil
|
|
}
|
|
|
|
// Wait blocks until every supervised command has finished (tests, shutdown).
|
|
func (r *chatShellRunner) Wait() { r.wg.Wait() }
|
|
|
|
func (r *chatShellRunner) activeProcess(sessionID, runID string) *chatShellProcess {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if p := r.active[runID]; p != nil && p.record.SessionID == sessionID {
|
|
return p
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// snapshot is the process's current record carrying at most max output bytes
|
|
// and their plain-text rendering.
|
|
func (p *chatShellProcess) snapshot(max int) protocol.ChatShellRunRecord {
|
|
p.mu.Lock()
|
|
record := p.record
|
|
record.Output, record.OutputStart = p.out.tail(max)
|
|
record.OutputBytes = p.out.total
|
|
record.Truncated = record.OutputStart > 0
|
|
p.mu.Unlock()
|
|
record.Text = chatShellPlainText(record.Output)
|
|
return record
|
|
}
|
|
|
|
func (r *chatShellRunner) publishRun(record protocol.ChatShellRunRecord) {
|
|
if r.publish == nil {
|
|
return
|
|
}
|
|
record.Output, record.Text = "", ""
|
|
r.publish(record.SessionID, protocol.ChatShellRunEventFrame{SessionID: record.SessionID, RunID: record.ID, Run: &record})
|
|
}
|
|
|
|
func (h *chatHub) shellRunner() *chatShellRunner {
|
|
if h == nil {
|
|
return nil
|
|
}
|
|
return h.shellRuns
|
|
}
|
|
|
|
// startShellRunner gives the hub its operator command runner over store,
|
|
// streaming to the session's room, after interrupting runs a previous server
|
|
// process left recorded as running. A nil store leaves commands refused.
|
|
func (h *chatHub) startShellRunner(ctx context.Context, store chatShellRunStore) {
|
|
if store == nil {
|
|
return
|
|
}
|
|
h.shellRuns = newChatShellRunner(ctx, store, func(sessionID string, frame protocol.ChatShellRunEventFrame) {
|
|
h.fanout(sessionID, protocol.Message{Type: protocol.ChatShellRunEvent, ChatShellRunEvent: &frame}, "")
|
|
})
|
|
h.shellRuns.recover(ctx)
|
|
}
|