170 lines
4.4 KiB
Go
170 lines
4.4 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// supervise streams one terminal until it ends: its first attachment is
|
|
// cmd on terminal. A tmux client that exits while its session lives (it was
|
|
// detached or killed) is replaced; a server shutdown leaves a tmux session
|
|
// running for the next server process and ends a direct shell.
|
|
func (m *terminalManager) supervise(p *terminalProcess, cmd *exec.Cmd, terminal *os.File) {
|
|
defer m.wg.Done()
|
|
defer close(p.done)
|
|
stopFlush, flushDone := make(chan struct{}), make(chan struct{})
|
|
go m.flushLoop(p, stopFlush, flushDone)
|
|
titled := func(string) {
|
|
record := p.snapshot(false, 0)
|
|
m.save(record)
|
|
m.publishRecord(record)
|
|
}
|
|
var waitErr error
|
|
shutdown := false
|
|
for reattached := 0; ; reattached++ {
|
|
readDone := make(chan struct{})
|
|
go func(terminal *os.File) {
|
|
defer close(readDone)
|
|
p.read(terminal, titled)
|
|
}(terminal)
|
|
attachDone := make(chan error, 1)
|
|
go func(cmd *exec.Cmd, terminal *os.File) { attachDone <- p.waitAttachment(cmd, terminal, readDone) }(cmd, terminal)
|
|
select {
|
|
case waitErr = <-attachDone:
|
|
case <-m.base.Done():
|
|
p.signal(syscall.SIGHUP)
|
|
select {
|
|
case waitErr = <-attachDone:
|
|
case <-time.After(time.Second):
|
|
p.signal(syscall.SIGKILL)
|
|
waitErr = <-attachDone
|
|
}
|
|
}
|
|
if m.base.Err() != nil {
|
|
shutdown = true
|
|
break
|
|
}
|
|
p.mu.Lock()
|
|
closing, name := p.closing, p.record.Tmux
|
|
p.mu.Unlock()
|
|
if closing || name == "" || m.tmux == nil || reattached >= terminalReattachLimit || !m.tmux.alive(name) {
|
|
break
|
|
}
|
|
cmd = m.tmux.attach(name)
|
|
next, err := p.startAttachment(cmd)
|
|
if err != nil {
|
|
log.Printf("terminal reattach %s: %v", p.record.ID, err)
|
|
break
|
|
}
|
|
terminal = next
|
|
}
|
|
close(stopFlush)
|
|
<-flushDone
|
|
m.emit(p, true)
|
|
p.mu.Lock()
|
|
switch {
|
|
case shutdown && p.record.Tmux != "":
|
|
// Still running in tmux: the next server process reattaches it.
|
|
case p.closing:
|
|
p.record.Status = protocol.TerminalClosed
|
|
default:
|
|
p.record.Status = protocol.TerminalExited
|
|
p.record.EndedAt = m.now().UnixMilli()
|
|
if shutdown {
|
|
p.record.Reason = terminalStopReason
|
|
} else if p.record.Tmux == "" {
|
|
p.record.ExitCode = terminalExitCode(waitErr)
|
|
}
|
|
}
|
|
p.pty, p.cmd = nil, nil
|
|
p.mu.Unlock()
|
|
record := p.snapshot(false, 0)
|
|
switch record.Status {
|
|
case protocol.TerminalClosed:
|
|
m.forget(record.ID)
|
|
default:
|
|
m.save(record)
|
|
}
|
|
if !shutdown {
|
|
m.publishRecord(record)
|
|
}
|
|
}
|
|
|
|
// flushLoop pushes new output in batches until stop is closed.
|
|
func (m *terminalManager) flushLoop(p *terminalProcess, stop <-chan struct{}, done chan<- struct{}) {
|
|
defer close(done)
|
|
ticker := time.NewTicker(m.flush)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-stop:
|
|
return
|
|
case <-ticker.C:
|
|
m.emit(p, false)
|
|
}
|
|
}
|
|
}
|
|
|
|
// emit pushes the output written since the last frame.
|
|
func (m *terminalManager) emit(p *terminalProcess, final bool) {
|
|
p.mu.Lock()
|
|
text, offset, end, ok := p.out.pending(protocol.TerminalChunkLimit, final)
|
|
id := p.record.ID
|
|
p.mu.Unlock()
|
|
if ok {
|
|
m.clients.publish(protocol.TerminalEventFrame{TerminalID: id, Output: text, Offset: offset, End: end})
|
|
}
|
|
}
|
|
|
|
// recover lists what a previous server process left running: a tmux-backed
|
|
// shell whose session is alive is reattached; any other is recorded as
|
|
// ended, saying why.
|
|
func (m *terminalManager) recover(ctx context.Context) {
|
|
if m.store == nil {
|
|
return
|
|
}
|
|
records, err := m.store.Terminals(ctx)
|
|
if err != nil {
|
|
log.Printf("terminal recover: %v", err)
|
|
return
|
|
}
|
|
for _, record := range records {
|
|
if record.Status != protocol.TerminalRunning {
|
|
continue
|
|
}
|
|
if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) && m.reattach(record) {
|
|
continue
|
|
}
|
|
record.Status, record.EndedAt = protocol.TerminalExited, m.now().UnixMilli()
|
|
record.Reason = terminalNoTmuxReason
|
|
if record.Tmux != "" {
|
|
record.Reason = terminalGoneReason
|
|
}
|
|
m.save(record)
|
|
}
|
|
}
|
|
|
|
// reattach shows a live tmux session in a new client and supervises it.
|
|
func (m *terminalManager) reattach(record protocol.TerminalRecord) bool {
|
|
cmd := m.tmux.attach(record.Tmux)
|
|
p := newTerminalProcess(record)
|
|
terminal, err := p.startAttachment(cmd)
|
|
if err != nil {
|
|
log.Printf("terminal reattach %s: %v", record.ID, err)
|
|
return false
|
|
}
|
|
m.mu.Lock()
|
|
m.terms[record.ID] = p
|
|
m.mu.Unlock()
|
|
m.wg.Add(1)
|
|
go m.supervise(p, cmd, terminal)
|
|
go p.pumpInput()
|
|
return true
|
|
}
|