f2520a09b8
A dismissed run keeps its command, status, times and output within the usual history bounds. chat_shell_runs lists dismissed runs apart (chat_shell_dismissed_runs), chat_shell_run_output still reads them, and the purge timer and startup purge are gone. Rows an older server purged are listed with purged set and cannot be restored.
268 lines
9.1 KiB
Go
268 lines
9.1 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// chatShellRunStore persists operator shell runs; *pipeline.Store implements it.
|
|
type chatShellRunStore interface {
|
|
SaveChatShellRun(ctx context.Context, run protocol.ChatShellRunRecord, keep int) error
|
|
ChatShellRuns(ctx context.Context, sessionID string) ([]protocol.ChatShellRunRecord, error)
|
|
ChatShellRun(ctx context.Context, sessionID, id string) (protocol.ChatShellRunRecord, bool, error)
|
|
InterruptChatShellRuns(ctx context.Context, endedAt int64, reason string) (int64, error)
|
|
chatShellDismissStore
|
|
}
|
|
|
|
const (
|
|
chatShellInterruptedReason = "The server stopped while this command was running."
|
|
// chatShellDrain is how long output is still read after the command itself
|
|
// exited, for descendants that keep the pipe open; the group is then killed.
|
|
chatShellDrain = time.Second
|
|
)
|
|
|
|
// chatShellRunner owns the operator's own commands: each runs `$SHELL -lc` in
|
|
// its chat's directory, in its own process group, in a pseudo-terminal where
|
|
// the platform has one (else with stdout and stderr on one pipe), its output
|
|
// kept as one bounded tail that is streamed to the session's watchers and
|
|
// persisted as Pipelines' record of the run.
|
|
type chatShellRunner struct {
|
|
base context.Context
|
|
store chatShellRunStore
|
|
publish func(sessionID string, frame protocol.ChatShellRunEventFrame)
|
|
now func() time.Time
|
|
shell func() string
|
|
// grace separates Stop's SIGINT from its SIGKILL; flush batches live output
|
|
// frames; persist is the interval a running run's output is saved at.
|
|
grace time.Duration
|
|
flush time.Duration
|
|
persist time.Duration
|
|
|
|
mu sync.Mutex
|
|
active map[string]*chatShellProcess
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
type chatShellProcess struct {
|
|
mu sync.Mutex
|
|
record protocol.ChatShellRunRecord
|
|
out *chatShellOutput
|
|
cmd *exec.Cmd
|
|
stopped bool
|
|
done chan struct{}
|
|
// terminal is the pseudo-terminal's controller side (nil in pipe mode);
|
|
// input feeds it in order (chat_shell_terminal.go).
|
|
terminal *os.File
|
|
input chan []byte
|
|
// closed is set once the reader is closed; a read still blocked on a
|
|
// terminal a descendant holds open records nothing after it.
|
|
closed bool
|
|
}
|
|
|
|
func newChatShellRunner(base context.Context, store chatShellRunStore, publish func(string, protocol.ChatShellRunEventFrame)) *chatShellRunner {
|
|
return &chatShellRunner{
|
|
base: base, store: store, publish: publish, now: time.Now, shell: chatShellProgram,
|
|
grace: 3 * time.Second, flush: 100 * time.Millisecond, persist: 2 * time.Second,
|
|
active: map[string]*chatShellProcess{},
|
|
}
|
|
}
|
|
|
|
// chatShellProgram is the operator's login shell, or /bin/sh without one.
|
|
func chatShellProgram() string {
|
|
if shell := os.Getenv("SHELL"); filepath.IsAbs(shell) {
|
|
if info, err := os.Stat(shell); err == nil && !info.IsDir() {
|
|
return shell
|
|
}
|
|
}
|
|
return "/bin/sh"
|
|
}
|
|
|
|
// recover marks runs a previous server process left running as interrupted:
|
|
// their processes died with it (or are no longer owned by anyone).
|
|
func (r *chatShellRunner) recover(ctx context.Context) {
|
|
if r.store == nil {
|
|
return
|
|
}
|
|
_, _ = r.store.InterruptChatShellRuns(ctx, r.now().UnixMilli(), chatShellInterruptedReason)
|
|
}
|
|
|
|
// resolveChatShellDirectory requires the chat's recorded directory to still be
|
|
// an existing directory and returns its symlink-free path.
|
|
func resolveChatShellDirectory(directory string) (string, error) {
|
|
directory = strings.TrimSpace(directory)
|
|
if directory == "" || !filepath.IsAbs(directory) {
|
|
return "", errors.New("this chat has no absolute working directory to run commands in")
|
|
}
|
|
real, err := filepath.EvalSymlinks(directory)
|
|
if err != nil {
|
|
return "", fmt.Errorf("the chat's directory %s is not available: %v", directory, err)
|
|
}
|
|
info, err := os.Stat(real)
|
|
if err != nil || !info.IsDir() {
|
|
return "", fmt.Errorf("the chat's directory %s is not a directory", directory)
|
|
}
|
|
return real, nil
|
|
}
|
|
|
|
func newChatShellRunID() (string, error) {
|
|
var b [12]byte
|
|
if _, err := rand.Read(b[:]); err != nil {
|
|
return "", err
|
|
}
|
|
return "run_" + hex.EncodeToString(b[:]), nil
|
|
}
|
|
|
|
// Start launches command for sessionID in directory and returns its running
|
|
// record. Validation and start failures are returned without a record.
|
|
// cols and rows size its terminal (zero picks the defaults).
|
|
func (r *chatShellRunner) Start(ctx context.Context, sessionID, directory, command string, cols, rows int) (protocol.ChatShellRunRecord, error) {
|
|
if r.store == nil {
|
|
return protocol.ChatShellRunRecord{}, errors.New("shell commands need the server's store, which is not configured")
|
|
}
|
|
if strings.TrimSpace(command) == "" {
|
|
return protocol.ChatShellRunRecord{}, errors.New("a nonempty command is required")
|
|
}
|
|
if len(command) > protocol.ChatShellRunCommandLimit {
|
|
return protocol.ChatShellRunRecord{}, fmt.Errorf("the command exceeds %d KiB", protocol.ChatShellRunCommandLimit>>10)
|
|
}
|
|
dir, err := resolveChatShellDirectory(directory)
|
|
if err != nil {
|
|
return protocol.ChatShellRunRecord{}, err
|
|
}
|
|
id, err := newChatShellRunID()
|
|
if err != nil {
|
|
return protocol.ChatShellRunRecord{}, err
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.base.Err() != nil {
|
|
return protocol.ChatShellRunRecord{}, errors.New("the server is stopping")
|
|
}
|
|
running := 0
|
|
for _, p := range r.active {
|
|
if p.record.SessionID == sessionID {
|
|
running++
|
|
}
|
|
}
|
|
if running >= protocol.ChatShellRunConcurrent {
|
|
return protocol.ChatShellRunRecord{}, fmt.Errorf("this chat already runs %d commands; stop one or wait for it to finish", running)
|
|
}
|
|
cmd := exec.Command(r.shell(), "-lc", command)
|
|
cmd.Dir = dir
|
|
cols, rows = chatShellTerminalSize(cols, rows)
|
|
reader, terminal, err := startChatShellCommand(cmd, cols, rows)
|
|
if err != nil {
|
|
return protocol.ChatShellRunRecord{}, fmt.Errorf("could not start the command: %v", err)
|
|
}
|
|
record := protocol.ChatShellRunRecord{ID: id, SessionID: sessionID, Command: command, Directory: dir, Status: protocol.ChatShellRunRunning, StartedAt: r.now().UnixMilli()}
|
|
if terminal != nil {
|
|
record.Terminal, record.Cols, record.Rows = true, cols, rows
|
|
}
|
|
p := &chatShellProcess{record: record, out: newChatShellOutput(protocol.ChatShellRunRetained), cmd: cmd, terminal: terminal, done: make(chan struct{})}
|
|
if err := r.store.SaveChatShellRun(ctx, record, protocol.ChatShellRunHistory); err != nil {
|
|
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
|
reader.Close()
|
|
_ = cmd.Wait()
|
|
return protocol.ChatShellRunRecord{}, fmt.Errorf("could not record the command: %v", err)
|
|
}
|
|
r.active[id] = p
|
|
r.wg.Add(1)
|
|
go r.supervise(p, reader)
|
|
if terminal != nil {
|
|
p.input = make(chan []byte, protocol.ChatShellInputQueue)
|
|
go p.pumpInput()
|
|
}
|
|
r.publishRun(record)
|
|
return record, nil
|
|
}
|
|
|
|
// Stop interrupts a running command's whole process group, then kills it
|
|
// after the grace period. It returns the run's current record.
|
|
func (r *chatShellRunner) Stop(sessionID, runID string) (protocol.ChatShellRunRecord, error) {
|
|
r.mu.Lock()
|
|
p := r.active[runID]
|
|
r.mu.Unlock()
|
|
if p == nil || p.record.SessionID != sessionID {
|
|
return protocol.ChatShellRunRecord{}, errors.New("this command is not running")
|
|
}
|
|
p.mu.Lock()
|
|
p.stopped = true
|
|
record := p.record
|
|
p.mu.Unlock()
|
|
pid := p.cmd.Process.Pid
|
|
_ = syscall.Kill(-pid, syscall.SIGINT)
|
|
go func() {
|
|
select {
|
|
case <-p.done:
|
|
case <-time.After(r.grace):
|
|
_ = syscall.Kill(-pid, syscall.SIGKILL)
|
|
}
|
|
}()
|
|
return record, nil
|
|
}
|
|
|
|
// Wait blocks until every supervised command has finished (tests, shutdown).
|
|
func (r *chatShellRunner) Wait() { r.wg.Wait() }
|
|
|
|
func (r *chatShellRunner) activeProcess(sessionID, runID string) *chatShellProcess {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if p := r.active[runID]; p != nil && p.record.SessionID == sessionID {
|
|
return p
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// snapshot is the process's current record carrying at most max output bytes
|
|
// and their plain-text rendering.
|
|
func (p *chatShellProcess) snapshot(max int) protocol.ChatShellRunRecord {
|
|
p.mu.Lock()
|
|
record := p.record
|
|
record.Output, record.OutputStart = p.out.tail(max)
|
|
record.OutputBytes = p.out.total
|
|
record.Truncated = record.OutputStart > 0
|
|
p.mu.Unlock()
|
|
record.Text = chatShellPlainText(record.Output)
|
|
return record
|
|
}
|
|
|
|
func (r *chatShellRunner) publishRun(record protocol.ChatShellRunRecord) {
|
|
if r.publish == nil {
|
|
return
|
|
}
|
|
record.Output, record.Text = "", ""
|
|
r.publish(record.SessionID, protocol.ChatShellRunEventFrame{SessionID: record.SessionID, RunID: record.ID, Run: &record})
|
|
}
|
|
|
|
func (h *chatHub) shellRunner() *chatShellRunner {
|
|
if h == nil {
|
|
return nil
|
|
}
|
|
return h.shellRuns
|
|
}
|
|
|
|
// startShellRunner gives the hub its operator command runner over store,
|
|
// streaming to the session's room, after interrupting runs a previous server
|
|
// process left recorded as running. A nil store leaves commands refused.
|
|
func (h *chatHub) startShellRunner(ctx context.Context, store chatShellRunStore) {
|
|
if store == nil {
|
|
return
|
|
}
|
|
h.shellRuns = newChatShellRunner(ctx, store, func(sessionID string, frame protocol.ChatShellRunEventFrame) {
|
|
h.fanout(sessionID, protocol.Message{Type: protocol.ChatShellRunEvent, ChatShellRunEvent: &frame}, "")
|
|
})
|
|
h.shellRuns.recover(ctx)
|
|
}
|