Files
pipelines/internal/server/terminal_supervise.go
T

170 lines
4.4 KiB
Go

package server
import (
"context"
"log"
"os"
"os/exec"
"syscall"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// supervise streams one terminal until it ends: its first attachment is
// cmd on terminal. A tmux client that exits while its session lives (it was
// detached or killed) is replaced; a server shutdown leaves a tmux session
// running for the next server process and ends a direct shell.
func (m *terminalManager) supervise(p *terminalProcess, cmd *exec.Cmd, terminal *os.File) {
defer m.wg.Done()
defer close(p.done)
stopFlush, flushDone := make(chan struct{}), make(chan struct{})
go m.flushLoop(p, stopFlush, flushDone)
titled := func(string) {
record := p.snapshot(false, 0)
m.save(record)
m.publishRecord(record)
}
var waitErr error
shutdown := false
for reattached := 0; ; reattached++ {
readDone := make(chan struct{})
go func(terminal *os.File) {
defer close(readDone)
p.read(terminal, titled)
}(terminal)
attachDone := make(chan error, 1)
go func(cmd *exec.Cmd, terminal *os.File) { attachDone <- p.waitAttachment(cmd, terminal, readDone) }(cmd, terminal)
select {
case waitErr = <-attachDone:
case <-m.base.Done():
p.signal(syscall.SIGHUP)
select {
case waitErr = <-attachDone:
case <-time.After(time.Second):
p.signal(syscall.SIGKILL)
waitErr = <-attachDone
}
}
if m.base.Err() != nil {
shutdown = true
break
}
p.mu.Lock()
closing, name := p.closing, p.record.Tmux
p.mu.Unlock()
if closing || name == "" || m.tmux == nil || reattached >= terminalReattachLimit || !m.tmux.alive(name) {
break
}
cmd = m.tmux.attach(name)
next, err := p.startAttachment(cmd)
if err != nil {
log.Printf("terminal reattach %s: %v", p.record.ID, err)
break
}
terminal = next
}
close(stopFlush)
<-flushDone
m.emit(p, true)
p.mu.Lock()
switch {
case shutdown && p.record.Tmux != "":
// Still running in tmux: the next server process reattaches it.
case p.closing:
p.record.Status = protocol.TerminalClosed
default:
p.record.Status = protocol.TerminalExited
p.record.EndedAt = m.now().UnixMilli()
if shutdown {
p.record.Reason = terminalStopReason
} else if p.record.Tmux == "" {
p.record.ExitCode = terminalExitCode(waitErr)
}
}
p.pty, p.cmd = nil, nil
p.mu.Unlock()
record := p.snapshot(false, 0)
switch record.Status {
case protocol.TerminalClosed:
m.forget(record.ID)
default:
m.save(record)
}
if !shutdown {
m.publishRecord(record)
}
}
// flushLoop pushes new output in batches until stop is closed.
func (m *terminalManager) flushLoop(p *terminalProcess, stop <-chan struct{}, done chan<- struct{}) {
defer close(done)
ticker := time.NewTicker(m.flush)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
m.emit(p, false)
}
}
}
// emit pushes the output written since the last frame.
func (m *terminalManager) emit(p *terminalProcess, final bool) {
p.mu.Lock()
text, offset, end, ok := p.out.pending(protocol.TerminalChunkLimit, final)
id := p.record.ID
p.mu.Unlock()
if ok {
m.clients.publish(protocol.TerminalEventFrame{TerminalID: id, Output: text, Offset: offset, End: end})
}
}
// recover lists what a previous server process left running: a tmux-backed
// shell whose session is alive is reattached; any other is recorded as
// ended, saying why.
func (m *terminalManager) recover(ctx context.Context) {
if m.store == nil {
return
}
records, err := m.store.Terminals(ctx)
if err != nil {
log.Printf("terminal recover: %v", err)
return
}
for _, record := range records {
if record.Status != protocol.TerminalRunning {
continue
}
if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) && m.reattach(record) {
continue
}
record.Status, record.EndedAt = protocol.TerminalExited, m.now().UnixMilli()
record.Reason = terminalNoTmuxReason
if record.Tmux != "" {
record.Reason = terminalGoneReason
}
m.save(record)
}
}
// reattach shows a live tmux session in a new client and supervises it.
func (m *terminalManager) reattach(record protocol.TerminalRecord) bool {
cmd := m.tmux.attach(record.Tmux)
p := newTerminalProcess(record)
terminal, err := p.startAttachment(cmd)
if err != nil {
log.Printf("terminal reattach %s: %v", record.ID, err)
return false
}
m.mu.Lock()
m.terms[record.ID] = p
m.mu.Unlock()
m.wg.Add(1)
go m.supervise(p, cmd, terminal)
go p.pumpInput()
return true
}