Merge branch 'worktree-agent-a2b0140029ece13b5'

This commit is contained in:
2026-11-10 00:46:16 +00:00
26 changed files with 1213 additions and 611 deletions
@@ -58,8 +58,10 @@ class SettingsTerminalSection extends StatelessWidget {
children: [
const Text(
'Saved on this device. Ctrl+` shows or hides the terminal; on '
'a narrow window it opens full screen. Shells keep running '
'while it is hidden.',
'a narrow window it opens full screen. A terminal starts at '
'the machine\'s login prompt: type that account\'s password '
'once, and it stays logged in while hidden. Terminals end '
'when the server restarts; run tmux inside one to keep work.',
),
if (layout.saveFailed)
Text(
+1 -4
View File
@@ -40,10 +40,7 @@ class TerminalBody extends StatelessWidget {
if (!tab.connected)
'${tab.machineName} is not connected; the terminal resumes when it is.'
else if (!tab.info.running)
tab.info.endedText
else if (!tab.info.persistent)
'This shell ends if the server on ${tab.machineName} restarts '
'(tmux is not installed there).',
tab.info.endedText,
?tab.error,
?hub.error,
];
+14 -3
View File
@@ -29,6 +29,14 @@ class TerminalHub extends ChangeNotifier {
/// The last refusal of an open or close, in the server's words.
String? error;
/// Whether this app run has had its first terminal: the first time the
/// terminal is shown it starts one (`openTerminalIfNone`), later shows
/// only show it. Opening one by hand counts.
bool firstOpenDone = false;
/// The first show's start, while it runs.
Future<void>? firstOpening;
List<TerminalTab> get tabs => List.unmodifiable(_tabs);
List<TerminalMachine> get connected => machines.value;
@@ -79,8 +87,10 @@ class TerminalHub extends ChangeNotifier {
TerminalMachine? machineFor(String? id) =>
connected.where((machine) => machine.id == id).firstOrNull;
/// Opens a shell on [machineId] (the first connected machine when null)
/// in [cwd] (home when empty) and selects its tab. Returns the refusal.
/// Opens a terminal on [machineId] (the first connected machine when
/// null) and selects its tab: it shows that machine's login prompt, and
/// after the password the shell starts in [cwd] (home when empty).
/// Returns the refusal.
Future<String?> open({String? machineId, String cwd = ''}) async {
activate();
final machine = machineId == null
@@ -96,6 +106,7 @@ class TerminalHub extends ChangeNotifier {
final record = reply['terminal'];
if (record is! Map) throw StateError('The server sent no terminal.');
final tab = _adopt(machine, TerminalInfo.fromJson(record));
firstOpenDone = true;
_selected = tab.key;
error = null;
_notify();
@@ -107,7 +118,7 @@ class TerminalHub extends ChangeNotifier {
}
}
/// Ends [tab]'s shell (and its tmux session) and removes the tab. A tab
/// Ends [tab]'s login or shell and removes the tab. A tab
/// whose machine is not connected cannot be closed yet.
Future<String?> close(TerminalTab tab) async {
if (!tab.connected) {
+4 -11
View File
@@ -1,7 +1,8 @@
import '../../core/api_gateway.dart';
/// One machine terminal as the server records it (`terminal_list`,
/// `terminal_event`): a login shell on that machine, running or ended.
/// `terminal_event`): a login on that machine (its login prompt, then the
/// account's shell), running or ended. It ends with the server process.
class TerminalInfo {
const TerminalInfo({
required this.id,
@@ -9,7 +10,6 @@ class TerminalInfo {
required this.status,
this.title = '',
this.shell = '',
this.tmux = '',
this.reason = '',
this.exitCode,
this.createdAt = 0,
@@ -25,7 +25,6 @@ class TerminalInfo {
status: '${json['status'] ?? ''}',
title: '${json['title'] ?? ''}',
shell: '${json['shell'] ?? ''}',
tmux: '${json['tmux'] ?? ''}',
reason: '${json['reason'] ?? ''}',
exitCode: (json['exit_code'] as num?)?.toInt(),
createdAt: (json['created_at'] as num?)?.toInt() ?? 0,
@@ -45,10 +44,6 @@ class TerminalInfo {
final String title;
final String shell;
/// The tmux session keeping the shell alive across server restarts;
/// empty when the shell ends with the server.
final String tmux;
/// Why an ended terminal ended, when it was not its own exit.
final String reason;
final int? exitCode;
@@ -56,8 +51,8 @@ class TerminalInfo {
final int cols;
final int rows;
/// Names the server process's output stream: a restarted server starts
/// a new one, whose byte offsets begin again at zero.
/// Names the server process's output stream; another stream's byte
/// offsets begin again at zero.
final String stream;
final int outputEnd;
@@ -68,7 +63,6 @@ class TerminalInfo {
status: status,
title: title,
shell: shell,
tmux: tmux,
reason: reason,
exitCode: exitCode,
createdAt: createdAt,
@@ -80,7 +74,6 @@ class TerminalInfo {
bool get running => status == 'running';
bool get closed => status == 'closed';
bool get persistent => tmux.isNotEmpty;
/// What an ended terminal shows instead of a prompt.
String get endedText {
@@ -7,12 +7,25 @@ import '../../ui/adaptive/segmented_bar_overflow.dart';
import 'terminal_hub.dart';
import 'terminal_scope.dart';
/// Opening the terminal with none to show starts one, as New terminal
/// does, once every machine's list has arrived.
Future<void> openTerminalIfNone(TerminalHub hub) async {
/// The first time the terminal is shown in an app run it starts one
/// terminal, as New terminal does (it shows the machine's login prompt),
/// unless one is already running once every machine's list has arrived.
/// Every later show only shows it: a hidden terminal stays logged in, and
/// a new one (each asking for the password) is opened by hand.
Future<void> openTerminalIfNone(TerminalHub hub) {
if (hub.firstOpenDone) return Future.value();
return hub.firstOpening ??= _openFirst(
hub,
).whenComplete(() => hub.firstOpening = null);
}
Future<void> _openFirst(TerminalHub hub) async {
hub.activate();
await hub.listed();
if (hub.tabs.isNotEmpty || hub.connected.isEmpty) return;
// With no machine connected yet, a later show tries again.
if (hub.firstOpenDone || hub.connected.isEmpty) return;
hub.firstOpenDone = true;
if (hub.tabs.any((tab) => tab.info.running)) return;
await hub.open();
}
+139 -39
View File
@@ -1,8 +1,11 @@
# Permanent terminal (bottom panel / floating window)
A full terminal available from every destination, independent of chats. It
keeps running when the panel is hidden, the app navigates or reloads, and
(where tmux is installed) across Pipelines server restarts.
A full terminal available from every destination, independent of chats.
Each terminal starts at the machine's login prompt (`su -l <user>` for the
server's own account), so the operator types that account's password before
getting a shell. It keeps running, logged in, when the panel is hidden or the
app navigates or reloads; it ends when the Pipelines server restarts (run
tmux inside it for work that must survive that).
## Behaviour
@@ -10,8 +13,16 @@ keeps running when the panel is hidden, the app navigates or reloads, and
menu): *Bottom panel* docks under the app (the app shrinks above it,
never hidden under it); *Floating window* is a window over the app. Both
are toggled by the bar's Terminal button (every destination's bar ends
with it) and the shortcuts below. Opening it with no terminal yet (once
every machine's list has arrived) starts one, as New terminal does.
with it) and the shortcuts below.
- **First open:** the first time the terminal is shown in an app run (once
every machine's list has arrived) it starts one terminal, as New terminal
does, unless one is already running; it shows the login prompt and the
operator types the password into the terminal (the PTY hides it). From
then on the button, shortcuts and the narrow page only hide and show it:
no toggle opens another terminal, even after every tab is closed (New
terminal does). Opening one by hand first counts as the first
(`TerminalHub.firstOpenDone`, `openTerminalIfNone`). Each new tab is its
own login and asks for the password again.
- **Shortcuts** (`terminal_shortcuts.dart`, handled by the surface as a
`FocusManager` early key handler, so they work wherever focus is, including
inside the terminal and in the chat composer, and neither receives the key):
@@ -67,7 +78,7 @@ keeps running when the panel is hidden, the app navigates or reloads, and
why they ended with a Close tab button. With more than one machine
connected (or tabs from several) each tab names its machine, and New
terminal (+) asks which machine. ⨯ on a tab closes it, which kills the
shell (and its tmux session).
login or shell.
- **Open in terminal:** the chat header's More menu (the chat's folder,
on the chat's machine) and a folder's ⋮ menu in the Files panel open a
new terminal there and show the surface.
@@ -76,7 +87,7 @@ keeps running when the panel is hidden, the app navigates or reloads, and
The layout (mode, shown, height, frame) is saved per device, so a reload
reopens the terminal, lists the machine's terminals and resumes each
from its retained output. A dropped connection resumes after the last
byte held; a restarted server (new stream) is read again from its start.
byte held. A restarted server lists the terminals as ended, with why.
- **Keyboard:** a focused terminal gets every key (Ctrl-C, Esc, Tab, arrows,
Ctrl-letters) except the shortcuts above (Ctrl+T, Alt+T, Cmd+T, Ctrl+`,
and Ctrl+Tab while shown); paste is Cmd-V (Apple) or Ctrl-V/Ctrl-Shift-V. Clicking
@@ -91,8 +102,9 @@ keeps running when the panel is hidden, the app navigates or reloads, and
terminals"). Body `terminal_request {terminal_id?, cwd?, cols?, rows?,
data?, after?}`.
- `terminal_open {cwd, cols, rows}` → `terminal` (subscribes the
connection to its output first).
- `terminal_open {cwd, cols, rows}` → `terminal` once su's password
prompt shows (subscribes the connection to its output first); refused
when the login cannot ask for a password (below).
- `terminal_list` → `terminals` (running and ended, oldest first; live
records carry their current title, size and `stream`).
- `terminal_input {terminal_id, data}` (UTF-8, ≤ 16 KiB, 64-request queue),
@@ -106,54 +118,73 @@ data?, after?}`.
title, exited, closed) to every connection. A slow connection's queue
(256 frames) drops output; the client sees the offset gap and re-reads.
## Persistence
## Login and lifetime (server)
- **tmux present** (looked up on PATH, then /opt/homebrew/bin,
/usr/local/bin, /usr/bin): each shell is `$SHELL -l` (else `/bin/sh`) as
the only pane of session `pipelines-<id>` on a Pipelines-owned tmux
server, `tmux -L pipelines -f /dev/null` (never the operator's own server
or config; `TMUX`/`TMUX_PANE` are stripped). Options: status off, mouse
on (wheel scrolls tmux history), history-limit 10000, escape-time 10,
set-titles on with the pane title. The server shows it through an
ordinary `attach-session` client in a PTY (creack/pty, the chat shell's
`startChatShellTerminal`). On shutdown the client gets SIGHUP and the
session stays; on start `recover` lists the DB's running terminals and
reattaches each whose session is alive (a fresh output stream: tmux
redraws the screen). A client that exits while its session lives is
replaced (up to 5 times).
- **No tmux:** the shell is a direct child; it ends with the server and is
listed as ended with the reason ("tmux is not installed on this
machine"); running tabs say "This shell ends if the server on <machine>
restarts (tmux is not installed there)".
- DB table `terminals` (additive): id, title, cwd, shell, created_at, tmux,
status, ended_at, exit_code, reason, cols, rows. Ended records beyond 16
are pruned. Output stays in memory (1 MiB tail per terminal).
- `internal/server/terminal_login.go`. A terminal runs, in its PTY,
`su -l <user> -c '<script>'` for the server's own account (`user.Current`):
su from PATH, else `/usr/bin/su` (macOS) or `/bin/su`, `/usr/bin/su`
(Linux), with `LC_ALL=C` so the prompt is English. macOS su passes the
arguments after the login name to the shell; util-linux su takes `-c`.
PAM asks for the account's password even though the server already runs
as that user (checked on this Mac: `/etc/pam.d/su` is `pam_rootok`
sufficient, then `pam_opendirectory`). After the login the account's shell
runs the script: it prints the marker OSC `ESC ] 6973;pipelines-login BEL`
(emulators ignore it), cds to the start folder (home when that fails) and
`exec "$SHELL" -l` (su -l's own login shell has read the profile once
already, so the login profile runs twice).
- `terminal_open` replies only once the output shows "password" (within
5 s). It is refused with a reason, and never listed, when su is missing,
the server runs as root (`pam_rootok` would skip the password), the
marker arrives before any prompt (an account without a password), su
exits first (its last line is quoted, e.g. no terminal), or nothing
asks in time. There is no fallback to an unauthenticated shell.
- A wrong password: su prints `su: Authentication failed` (macOS) and exits
1; the tab is ended with "The login did not complete (wrong password, or
the prompt was cancelled)..." and the exit code. Close it and open a new
terminal to try again.
- No tmux: a terminal is a direct child of the server and ends with it. A
stop records "The Pipelines server stopped..."; on start, records still
`running` (a crash, or a stop whose save lost the race with the store
closing) become ended with "The Pipelines server restarted and this
terminal ended with it... run tmux inside a terminal...". The tmux
backing, reattach and the record's `tmux` field are gone; the DB's
`tmux` column stays (additive schema) but is no longer written.
- Leftovers: `cmd/pipelines-server` calls `server.EndLegacyTerminalTmux()`
at start: on `tmux -L pipelines -f /dev/null` only, it kills sessions
named `pipelines-term_*` (what earlier servers created); any other session
there and the operator's own tmux servers are untouched.
- DB table `terminals`: id, title, cwd, shell (`<su> -l <user>`),
created_at, status, ended_at, exit_code, reason, cols, rows. Ended
records beyond 16 are pruned. Output stays in memory (1 MiB tail per
terminal).
## Security
- Only reachable through the authenticated `/connect` websocket, like chat
shell commands; any connection of that server may list, type into or
close its terminals.
shell commands, and each terminal then needs the machine account's
password (su), a second barrier. Any connection of that server may list,
type into or close its terminals, including a logged-in one.
- The start folder must exist inside the operator's home after resolving
symlinks (the Files panel's containment); `~`, `~/sub` and absolute paths
are accepted, `~user` and relative paths refused.
- Input is never logged or stored: the server skips its request log line
for `terminal_input`/`terminal_resize`, nothing is written to the DB, and
the client's input pipe (`ChatShellInputPipe`, shared with chat shells)
keeps nothing once sent.
keeps nothing once sent. The password is typed into the PTY with echo
off, so it is in neither the output nor the retained tail.
## Limits and known gaps
- 8 running terminals per machine; 1 MiB retained output per terminal.
- A tmux shell's exit status is not reported (only direct shells have it).
- Terminals end when the server restarts; persistence is tmux inside the
logged-in shell, by hand.
- Linux su was not run: the arguments follow util-linux su; only this Mac's
`/usr/bin/su` was checked for real.
- The client transport sends one request at a time per connection, so a
keystroke waits behind any slower request on the same machine (as chat
shell input already does).
- Opening a terminal elsewhere (another device) adds its tab here too: the
tabs are the machine's terminals.
- After a server restart the reattached output starts with tmux's redraw
of the screen; older lines are in tmux's history (mouse wheel), not in
the emulator's own scrollback.
- Mobile (narrow page, soft-keyboard field) is verified by widget tests and
a narrow Chrome capture only; no device test ran. Android/Linux builds
were not run.
@@ -165,8 +196,9 @@ data?, after?}`.
- Server: `internal/protocol/terminal.go`, `message.go`;
`internal/pipeline/terminals.go`, `store.go`;
`internal/server/terminal_{manager,supervise,process,tmux,clients,title,cwd,handlers}.go`,
`websocket.go`; tests `terminal_ws_test.go`, `terminal_restart_test.go`,
`internal/server/terminal_{manager,supervise,process,login,legacy_tmux,clients,title,cwd,handlers}.go`,
`websocket.go`, `cmd/pipelines-server/main.go`; tests `terminal_ws_test.go`,
`terminal_login_test.go`, `terminal_lifetime_test.go`, `terminal_legacy_tmux_test.go`,
`internal/pipeline/terminals_test.go`; `docs/cli.md`.
- App: `lib/features/terminal/` (`terminal_{info,stream,tab,hub,listing,machines,layout,scope,surface,stage,focus,shortcuts,panel,float,float_frame,page,body,tab_strip,menu,toggle_button,open_action}.dart`),
`lib/application/app_terminals.dart`, `pipelines_app.dart`,
@@ -291,3 +323,71 @@ Go 1.x / Flutter 3.44.6, macOS host, tmux 3.7c (/opt/homebrew/bin/tmux).
automation permission), so neither real Chrome nor the native macOS
build's Cmd+T was driven by real keystrokes; native behaviour is verified
by widget tests only.
### Login prompt instead of tmux (2026-11-10)
The tmux results above (restart survival, client replacement, the
`/tmp/term-e2e` run) describe the removed tmux backing.
- `go test ./internal/protocol ./internal/pipeline ./internal/server ./cmd/pipelines-server -count=1`:
`ok …/protocol 0.190s`, `ok …/pipeline 8.169s`, `ok …/server 70.840s`,
`ok …/cmd/pipelines-server 0.217s`. `go test -race -run Terminal -count=3
./internal/server`: ok. `go vet`, `gofmt -l`: clean;
`GOOS=linux GOARCH=amd64 go vet` + `go test -c`, `GOOS=freebsd go build
./internal/server`: ok. Terminal tests use a fake `su` on PATH (prints
`Password:`, reads a line with echo off, accepts only `test-password`;
modes for no prompt, silence and no terminal), so no real password is
ever needed: login args per OS and refusals (root, no su, no user,
other systems), refusal without su on PATH or in the usual directories,
the prompt in a real PTY, wrong password ended with its reason and exit
1, right password gives a shell in the start folder with the password
absent from the output, unprompted/silent/failing logins refused and
never listed, ended on stop and on restart with their reasons, no tmux
in the record, leftover-session cleanup on a test socket (only
`pipelines-term_*` killed), and the websocket open/input/resize/resume/
close, exit status and limits tests logged in through the fake.
- `tool/memwatch/full.sh test/features/terminal`: `PASS: 48 passed, 0
failed` (new `terminal_first_open_test.dart`: first show opens exactly
one terminal in the panel and in the window, and six toggles open no
more and close none; closing every tab then toggling opens none; a
running terminal from before is shown, not added to; only ended ones
(server restarted) → one new login; a hand-opened one counts as first
and New terminal still opens another; three quick taps open one; the
narrow page opened three times opens one). `tool/analyze.sh`: clean.
- Real `/usr/bin/su` on this Mac, as the same non-root user, in a PTY
(`/tmp/su-probe/probe.py`): `Password:`; a wrong password →
`su: Authentication failed`, exit 1. The operator's password was never
typed.
- Scratch server live check (`/tmp/term-login-e2e/run.sh`, driver
`drive.mjs`): `env -i`, HOME, XDG_DATA_HOME, DB and `TMUX_TMPDIR` under
`/tmp/term-login-e2e`, OS-chosen loopback port, `PIPELINES_MDNS=off`;
two sessions planted on the scratch `tmux -L pipelines` first:
```
leftover sessions planted on the scratch tmux -L pipelines:
mine
pipelines-term_leftover
server pid 43971 port 50492
scratch tmux sessions after start:
mine
opened term_f04cd8ac8c198a09 status=running shell="/usr/bin/su -l alan" cwd=/tmp/term-login-e2e/home tmux=(none)
output of term_f04cd8ac8c198a09: "Password:"
su processes under the server:
43987 su
server 43971 stopped
su processes left: 0
list after restart: term_f04cd8ac8c198a09 exited reason="The Pipelines server restarted and this terminal ended with it. Terminals end with the server; run tmux inside a terminal to keep work across restarts."
opened term_add0f7930d880b69 status=running shell="/usr/bin/su -l alan" cwd=/tmp/term-login-e2e/home tmux=(none)
output of term_add0f7930d880b69: "Password:"
close: terminal_close status=closed
terminals table: (empty after closing both)
input requests logged: 0
terminal: ended 1 tmux session(s) an earlier Pipelines server left on tmux -L pipelines
```
The stop's own save lost the race with the store closing (`terminal
save …: sql: database is closed`, as before this change), so the
restart recorded the end instead. The scratch server's opencode stage
service outlived it and was killed by hand.
- Not verified: logging in for real (needs the operator's password), Linux
su, and the app against a live server in a browser (widget tests only).
@@ -0,0 +1,133 @@
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/terminal/terminal_hub.dart';
import 'package:pipelines_app/features/terminal/terminal_layout.dart';
import 'terminal_app.dart';
import 'terminal_fixtures.dart';
/// The first show of an app run starts one terminal (the machine's login
/// prompt); toggling afterwards only hides and shows it.
void main() {
late TerminalGateway mac;
late TerminalHub hub;
late TerminalLayout layout;
setUp(() {
mac = TerminalGateway();
hub = TerminalHub(TestMachines([machine('mac', mac, name: 'Mac')]));
layout = TerminalLayout(store: MemoryLayoutStore());
});
tearDown(() async {
hub.dispose();
await mac.close();
});
Future<void> button(WidgetTester tester) async {
await tester.tap(find.byKey(const ValueKey('shell.terminal')));
await tester.pumpAndSettle();
}
for (final mode in TerminalMode.values) {
testWidgets('the first show opens exactly one terminal in the '
'${mode.label.toLowerCase()}; toggling opens no more', (tester) async {
layout.setMode(mode);
await pumpTerminalApp(tester, hub: hub, layout: layout);
expect(mac.bodies('terminal_open'), isEmpty);
await button(tester);
expect(layout.visible, isTrue);
expect(mac.bodies('terminal_open'), hasLength(1));
expect(hub.tabs, hasLength(1));
final first = hub.tabs.single;
for (var i = 0; i < 6; i++) {
await button(tester);
}
expect(layout.visible, isTrue);
expect(mac.bodies('terminal_open'), hasLength(1));
expect(identical(hub.tabs.single, first), isTrue);
// Hiding never closes it: it stays logged in.
expect(mac.bodies('terminal_close'), isEmpty);
expect(first.info.running, isTrue);
});
}
testWidgets('closing every tab and showing again opens none', (tester) async {
await pumpTerminalApp(tester, hub: hub, layout: layout);
await button(tester);
await hub.close(hub.tabs.single);
await tester.pumpAndSettle();
expect(hub.tabs, isEmpty);
await button(tester);
await button(tester);
expect(layout.visible, isTrue);
expect(mac.bodies('terminal_open'), hasLength(1));
expect(find.text('New terminal'), findsWidgets);
});
testWidgets('a terminal still running from before is shown, not added '
'to', (tester) async {
mac.add('term_kept');
await pumpTerminalApp(tester, hub: hub, layout: layout);
await button(tester);
await button(tester);
await button(tester);
expect(mac.bodies('terminal_open'), isEmpty);
expect(hub.tabs.single.info.id, 'term_kept');
});
testWidgets('with only ended terminals (the server restarted) the first '
'show opens one login', (tester) async {
mac
..add('term_old')
..terminals['term_old']!['status'] = 'exited';
await pumpTerminalApp(tester, hub: hub, layout: layout);
await button(tester);
await button(tester);
await button(tester);
expect(mac.bodies('terminal_open'), hasLength(1));
expect(hub.tabs, hasLength(2));
});
testWidgets('a terminal opened by hand first counts as the first', (
tester,
) async {
await pumpTerminalApp(tester, hub: hub, layout: layout);
expect(await hub.open(), isNull);
await button(tester);
await button(tester);
await button(tester);
expect(mac.bodies('terminal_open'), hasLength(1));
// Each New terminal is its own login.
expect(await hub.open(), isNull);
expect(mac.bodies('terminal_open'), hasLength(2));
});
testWidgets('showing twice quickly opens one', (tester) async {
await pumpTerminalApp(tester, hub: hub, layout: layout);
await tester.tap(find.byKey(const ValueKey('shell.terminal')));
await tester.pump();
await tester.tap(find.byKey(const ValueKey('shell.terminal')));
await tester.pump();
await tester.tap(find.byKey(const ValueKey('shell.terminal')));
await tester.pumpAndSettle();
expect(mac.bodies('terminal_open'), hasLength(1));
});
testWidgets('on a narrow window the page opens one and reopening adds '
'none', (tester) async {
await pumpTerminalApp(
tester,
hub: hub,
layout: layout,
size: const Size(390, 800),
);
for (var i = 0; i < 3; i++) {
await button(tester);
expect(find.widgetWithText(AppBar, 'Terminal'), findsOneWidget);
await tester.pageBack();
await tester.pumpAndSettle();
}
expect(mac.bodies('terminal_open'), hasLength(1));
expect(hub.tabs, hasLength(1));
});
}
@@ -36,7 +36,6 @@ class TerminalGateway implements ApiGateway {
'cwd': cwd,
'shell': '/bin/zsh',
'status': 'running',
'tmux': 'pipelines-$id',
'cols': 80,
'rows': 24,
'created_at': 1,
@@ -85,8 +85,8 @@ void main() {
expect(tab.connected, isTrue);
expect(again.bodies('terminal_output').single['after'], 5);
expect(screen(tab), 'one\ntwo');
// The server restarted: the tmux client redrew the screen into a new
// stream starting at zero.
// The machine now serves the terminal from another output stream,
// starting at zero: it is read again from its start.
machines.value = [];
await settle();
final restarted = TerminalGateway(stream: 's2')
+3
View File
@@ -168,6 +168,9 @@ func main() {
// (chatShutdownDetacher) instead of stopping it.
defer chats.Close()
}
// Terminals no longer live in tmux: end the sessions an earlier server
// left on its own tmux socket (never the operator's tmux).
server.EndLegacyTerminalTmux()
srv := &http.Server{Addr: *addr, Handler: server.HandlerWithOptions(ctx, options, store), ReadHeaderTimeout: 5 * time.Second}
// Deferred before WaitRuns so it runs after the runs settle: the owned stage
// service is stopped only when no stage session is still active on it.
+15 -9
View File
@@ -1298,20 +1298,26 @@ Interactive login shells on a server's machine, independent of chats (the
app's terminal panel). Every request carries `terminal_request`
`{terminal_id?, cwd?, cols?, rows?, data?, after?}`:
- `terminal_open {cwd, cols, rows}` starts `$SHELL -l` (else `/bin/sh`) in
`cwd` (default home; it must be an existing folder inside home) and replies
with `terminal`. Where `tmux` is installed the shell runs in its own session
of a Pipelines-owned tmux server (`tmux -L pipelines`, never the operator's
own), so it outlives a server restart and the next server reattaches it.
At most 8 running terminals per machine.
- `terminal_open {cwd, cols, rows}` starts the machine's login prompt for the
server's own account (`su -l <user>` in a PTY; macOS and Linux) and replies
with `terminal` once the password prompt shows. The operator types that
account's password into the terminal (the PTY hides it); the account's login
shell then starts in `cwd` (default home; it must be an existing folder
inside home). A wrong password ends the terminal with a reason. The open is
refused, never replaced by an unauthenticated shell, when su is missing, the
server runs as root, or su opens a shell without asking. Terminals end with
the server process and are then listed as ended; run tmux inside one to
keep work across restarts. A starting server ends the `pipelines-term_*`
sessions an earlier server left on its own tmux socket (`tmux -L
pipelines`), never the operator's tmux. At most 8 running terminals per
machine.
- `terminal_list` replies with `terminals` (running and ended, oldest first).
- `terminal_input {terminal_id, data}` writes UTF-8 input (at most 16 KiB);
`terminal_resize {terminal_id, cols, rows}` resizes. Neither is logged.
- `terminal_output {terminal_id, after}` replies with the retained output (1
MiB tail) from byte `after` of the stream named `stream`; a restarted server
starts a new stream at zero. Opening or reading subscribes the connection to
MiB tail) from byte `after` of the stream named `stream`. Opening or reading subscribes the connection to
the terminal's output frames.
- `terminal_close {terminal_id}` kills the shell and its tmux session.
- `terminal_close {terminal_id}` kills the login or shell.
- Pushed `terminal_event {terminal_id, output, offset, end}` (output) or
`{terminal_id, terminal}` (opened, title, exited, closed).
+8 -7
View File
@@ -8,8 +8,9 @@ import (
)
// initTerminals creates the additive table of machine terminals. It records
// only what a restarted server needs to list and reattach them (id, title,
// cwd, shell, tmux session, status). Output and typed input are never stored.
// only what a restarted server needs to list them (id, title, cwd, shell,
// status). Output and typed input are never stored. The tmux column is left
// from servers that kept shells in tmux; it is no longer written.
func (s *Store) initTerminals() error {
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS terminals (
id TEXT PRIMARY KEY,
@@ -40,11 +41,11 @@ func (s *Store) SaveTerminal(ctx context.Context, t protocol.TerminalRecord, kee
exit = *t.ExitCode
}
if _, err := tx.ExecContext(ctx, `INSERT INTO terminals
(id, title, cwd, shell, created_at, tmux, status, ended_at, exit_code, reason, cols, rows)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
(id, title, cwd, shell, created_at, status, ended_at, exit_code, reason, cols, rows)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET title = excluded.title, status = excluded.status, ended_at = excluded.ended_at,
exit_code = excluded.exit_code, reason = excluded.reason, cols = excluded.cols, rows = excluded.rows`,
t.ID, t.Title, t.Cwd, t.Shell, t.CreatedAt, t.Tmux, t.Status, t.EndedAt, exit, t.Reason, t.Cols, t.Rows); err != nil {
t.ID, t.Title, t.Cwd, t.Shell, t.CreatedAt, t.Status, t.EndedAt, exit, t.Reason, t.Cols, t.Rows); err != nil {
return err
}
if keep > 0 && t.Status != protocol.TerminalRunning {
@@ -65,7 +66,7 @@ func (s *Store) DeleteTerminal(ctx context.Context, id string) error {
// Terminals lists every recorded terminal, oldest first.
func (s *Store) Terminals(ctx context.Context) ([]protocol.TerminalRecord, error) {
rows, err := s.db.QueryContext(ctx, `SELECT id, title, cwd, shell, created_at, tmux, status, ended_at, exit_code, reason, cols, rows
rows, err := s.db.QueryContext(ctx, `SELECT id, title, cwd, shell, created_at, status, ended_at, exit_code, reason, cols, rows
FROM terminals ORDER BY created_at, id`)
if err != nil {
return nil, err
@@ -75,7 +76,7 @@ FROM terminals ORDER BY created_at, id`)
for rows.Next() {
var t protocol.TerminalRecord
var exit sql.NullInt64
if err := rows.Scan(&t.ID, &t.Title, &t.Cwd, &t.Shell, &t.CreatedAt, &t.Tmux, &t.Status, &t.EndedAt, &exit, &t.Reason, &t.Cols, &t.Rows); err != nil {
if err := rows.Scan(&t.ID, &t.Title, &t.Cwd, &t.Shell, &t.CreatedAt, &t.Status, &t.EndedAt, &exit, &t.Reason, &t.Cols, &t.Rows); err != nil {
return nil, err
}
if exit.Valid {
+2 -2
View File
@@ -16,7 +16,7 @@ func TestTerminalsPersistPruneAndDelete(t *testing.T) {
if err != nil {
t.Fatal(err)
}
live := protocol.TerminalRecord{ID: "term_live", Cwd: "/home/a", Shell: "/bin/zsh", CreatedAt: 1, Tmux: "pipelines-term_live", Status: protocol.TerminalRunning, Cols: 80, Rows: 24}
live := protocol.TerminalRecord{ID: "term_live", Cwd: "/home/a", Shell: "/bin/zsh", CreatedAt: 1, Status: protocol.TerminalRunning, Cols: 80, Rows: 24}
if err := store.SaveTerminal(ctx, live, 2); err != nil {
t.Fatal(err)
}
@@ -41,7 +41,7 @@ func TestTerminalsPersistPruneAndDelete(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(list) != 3 || list[0].ID != "term_live" || list[0].Title != "vim" || list[0].Cols != 120 || list[0].Tmux != "pipelines-term_live" ||
if len(list) != 3 || list[0].ID != "term_live" || list[0].Title != "vim" || list[0].Cols != 120 ||
list[1].ID != "term_1" || list[2].ID != "term_2" || list[2].ExitCode == nil || *list[2].ExitCode != 0 {
t.Fatalf("terminals = %+v", list)
}
+19 -24
View File
@@ -1,17 +1,20 @@
package protocol
// Machine terminals: interactive login shells the operator opens on a
// server's machine, independent of any chat. Each runs `$SHELL -l` (else
// /bin/sh) in a pseudo-terminal; where tmux is installed the shell lives in
// a Pipelines-owned tmux server (socket name "pipelines", never the
// operator's own), so it survives a Pipelines server restart and the next
// process reattaches it. Without tmux a shell ends with the server and is
// listed as ended. Typed input is never logged or stored.
// server's machine, independent of any chat. Each starts at the machine's
// login prompt for the server's own account (`su -l <user>` in a
// pseudo-terminal): the operator types that account's password before the
// login shell starts. A terminal ends with the server process and is then
// listed as ended (run tmux inside it to keep work across restarts). Typed
// input is never logged or stored.
const (
// TerminalOpen starts a shell. The body is TerminalRequest (cwd, cols,
// rows); cwd defaults to the operator's home and must be an existing
// directory inside it. The reply carries the new terminal; the
// connection is then subscribed to its terminal_event output frames.
// TerminalOpen starts a terminal at the login prompt. The body is
// TerminalRequest (cwd, cols, rows); cwd defaults to the operator's home
// and must be an existing directory inside it (the shell starts there
// after the login). The reply, sent once the password prompt shows,
// carries the new terminal; the connection is subscribed to its
// terminal_event output frames. A login that cannot ask for a password
// (no su, a root server, an account without one) is refused.
TerminalOpen = "terminal_open"
// TerminalList lists this machine's terminals (running and ended),
// oldest first, without output. The body is optional.
@@ -27,7 +30,7 @@ const (
// tail; OutputStart says where it starts). It also subscribes the
// connection to the terminal's terminal_event output frames.
TerminalOutput = "terminal_output"
// TerminalClose ends a terminal: its shell (and tmux session) is killed
// TerminalClose ends a terminal: its login or shell is killed
// and its record removed. The reply echoes the last record.
TerminalClose = "terminal_close"
// TerminalEvent is pushed: an output chunk to the connections subscribed
@@ -53,8 +56,6 @@ const (
TerminalChunkLimit = 64 << 10
// TerminalEndedKept bounds the ended terminals still listed.
TerminalEndedKept = 16
// TerminalTmuxHistory is the tmux scrollback of one tmux-backed shell.
TerminalTmuxHistory = 10000
// TerminalTitleLimit bounds a recorded title.
TerminalTitleLimit = 200
)
@@ -79,22 +80,16 @@ type TerminalRecord struct {
Cwd string `json:"cwd"`
Shell string `json:"shell"`
CreatedAt int64 `json:"created_at"`
// Tmux is the tmux session name keeping the shell alive across server
// restarts; empty when the shell is a direct child of the server and
// ends with it.
Tmux string `json:"tmux,omitempty"`
Status string `json:"status"`
EndedAt int64 `json:"ended_at,omitempty"`
ExitCode *int `json:"exit_code,omitempty"`
Status string `json:"status"`
EndedAt int64 `json:"ended_at,omitempty"`
ExitCode *int `json:"exit_code,omitempty"`
// Reason says why an ended terminal ended when it was not simply its
// shell exiting (the server restarted without tmux, its tmux session
// was gone).
// shell exiting (the login failed, the server stopped or restarted).
Reason string `json:"reason,omitempty"`
Cols int `json:"cols"`
Rows int `json:"rows"`
// Stream names this server process's output stream of the terminal;
// OutputEnd is the absolute byte count read into it. A restarted server
// that reattaches starts a new stream at zero, so a client holding
// OutputEnd is the absolute byte count read into it. A client holding
// another stream reads the terminal again from its start.
Stream string `json:"stream,omitempty"`
OutputEnd int64 `json:"output_end"`
+69
View File
@@ -0,0 +1,69 @@
package server
import (
"log"
"os"
"os/exec"
"path/filepath"
"strings"
)
// Earlier servers kept each terminal's shell in a session `pipelines-<id>`
// of a Pipelines-owned tmux server (`tmux -L pipelines -f /dev/null`).
// Terminals now end with the server, so a starting server ends the sessions
// such a server left behind. Only that socket and only sessions with that
// name prefix are touched, never the operator's own tmux.
const (
legacyTerminalTmuxSocket = "pipelines"
legacyTerminalTmuxPrefix = "pipelines-term_"
)
// EndLegacyTerminalTmux ends the terminal sessions an earlier Pipelines
// server left in its own tmux server (that tmux server exits with its last
// session). It does nothing where tmux is not installed or not running.
func EndLegacyTerminalTmux() { endLegacyTerminalTmux(legacyTerminalTmuxSocket) }
func endLegacyTerminalTmux(socket string) int {
program := findLegacyTmux()
if program == "" {
return 0
}
tmux := func(args ...string) *exec.Cmd {
cmd := exec.Command(program, append([]string{"-L", socket, "-f", os.DevNull}, args...)...)
cmd.Env = terminalEnv(os.Environ(), "")
return cmd
}
out, err := tmux("list-sessions", "-F", "#{session_name}").Output()
if err != nil {
return 0 // no tmux server on that socket
}
ended := 0
for _, name := range strings.Fields(string(out)) {
if !strings.HasPrefix(name, legacyTerminalTmuxPrefix) {
continue
}
if err := tmux("kill-session", "-t", "="+name).Run(); err != nil {
log.Printf("terminal: ending leftover tmux session %s: %v", name, err)
continue
}
ended++
}
if ended > 0 {
log.Printf("terminal: ended %d tmux session(s) an earlier Pipelines server left on tmux -L %s", ended, socket)
}
return ended
}
// findLegacyTmux is tmux on PATH or in the usual package-manager locations.
func findLegacyTmux() string {
if program, err := exec.LookPath("tmux"); err == nil {
return program
}
for _, dir := range []string{"/opt/homebrew/bin", "/usr/local/bin", "/usr/bin"} {
candidate := filepath.Join(dir, "tmux")
if info, err := os.Stat(candidate); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 {
return candidate
}
}
return ""
}
@@ -0,0 +1,54 @@
//go:build darwin || linux
package server
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
// Startup cleanup ends only the terminal sessions an earlier Pipelines
// server left on its socket; another session there, and every other tmux
// server, is left alone. The test uses a socket of its own.
func TestEndLegacyTerminalTmux(t *testing.T) {
program := findLegacyTmux()
if program == "" {
t.Skip("tmux is not installed")
}
tmpdir, err := os.MkdirTemp("/tmp", "pl-tmux-")
if err != nil {
t.Fatal(err)
}
t.Setenv("TMUX_TMPDIR", tmpdir)
socket := fmt.Sprintf("pipelines-test-%d-%d", os.Getpid(), time.Now().UnixNano())
tmux := func(args ...string) *exec.Cmd {
return exec.Command(program, append([]string{"-L", socket, "-f", os.DevNull}, args...)...)
}
t.Cleanup(func() {
_ = tmux("kill-server").Run()
_ = os.RemoveAll(tmpdir)
})
if ended := endLegacyTerminalTmux(socket); ended != 0 {
t.Fatalf("ended %d with no tmux server", ended)
}
for _, name := range []string{"pipelines-term_a1", "pipelines-term_b2", "mine"} {
if out, err := tmux("new-session", "-d", "-s", name, "sleep", "60").CombinedOutput(); err != nil {
t.Fatalf("new-session %s: %v %s", name, err, out)
}
}
if ended := endLegacyTerminalTmux(socket); ended != 2 {
t.Fatalf("ended %d sessions", ended)
}
out, err := tmux("list-sessions", "-F", "#{session_name}").Output()
if err != nil || strings.TrimSpace(string(out)) != "mine" {
t.Fatalf("sessions left = %q, %v", out, err)
}
if _, err := os.Stat(filepath.Join(tmpdir, fmt.Sprintf("tmux-%d", os.Getuid()), socket)); err != nil {
t.Fatalf("test socket not under TMUX_TMPDIR: %v", err)
}
}
+109
View File
@@ -0,0 +1,109 @@
//go:build darwin || linux
package server
import (
"context"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// A terminal ends with its server: the stop is recorded as ended, and a
// record a crashed (or tmux-era) server left running is listed as ended
// after the restart, saying why.
func TestTerminalEndsWithServer(t *testing.T) {
terminalHome(t)
store := newOperatorStore(t)
first, stop := newTestTerminals(t, store)
term, err := first.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
if err := first.Input(term.ID, fakeSuPassword+"\r"); err != nil {
t.Fatal(err)
}
stop()
first.Wait()
crashed := protocol.TerminalRecord{ID: "term_crashed", Cwd: "/", CreatedAt: 1, Status: protocol.TerminalRunning, Cols: 80, Rows: 24}
if err := store.SaveTerminal(context.Background(), crashed, 0); err != nil {
t.Fatal(err)
}
second, _ := newTestTerminals(t, store)
list, err := second.List(context.Background())
if err != nil {
t.Fatal(err)
}
reasons := map[string]string{}
for _, record := range list {
if record.Status != protocol.TerminalExited || record.EndedAt == 0 {
t.Fatalf("record = %+v", record)
}
reasons[record.ID] = record.Reason
}
if len(reasons) != 2 || reasons[term.ID] != terminalStopReason || reasons["term_crashed"] != terminalRestartReason {
t.Fatalf("reasons = %+v", reasons)
}
if _, err := second.Close(context.Background(), "term_crashed"); err != nil {
t.Fatal(err)
}
if list, _ := second.List(context.Background()); len(list) != 1 {
t.Fatalf("list after closing an ended terminal = %+v", list)
}
}
// The window title the shell sets is followed across reads and recorded.
func TestTerminalTitles(t *testing.T) {
var titles terminalTitles
if _, ok := titles.feed([]byte("plain \x1b[1mtext")); ok {
t.Fatal("title from plain text")
}
if _, ok := titles.feed([]byte("\x1b]0;~/src")); ok {
t.Fatal("title before its end")
}
if title, ok := titles.feed([]byte("/app\x07more")); !ok || title != "~/src/app" {
t.Fatalf("title = %q %v", title, ok)
}
if title, ok := titles.feed([]byte("\x1b]2;vim\x01 x\x1b\\")); !ok || title != "vim x" {
t.Fatalf("ST title = %q %v", title, ok)
}
if _, ok := titles.feed([]byte("\x1b]8;;https://x\x07")); ok {
t.Fatal("a hyperlink is not a title")
}
if _, ok := titles.feed(terminalLoginMarker); ok {
t.Fatal("the login marker is not a title")
}
if title, _ := titles.feed([]byte("\x1b]0;" + strings.Repeat("é", protocol.TerminalTitleLimit) + "\x07")); len(title) > protocol.TerminalTitleLimit {
t.Fatalf("title of %d bytes", len(title))
}
terminalHome(t)
m, _ := newTestTerminals(t, newOperatorStore(t))
term, err := m.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
if err := m.Input(term.ID, fakeSuPassword+"\r"); err != nil {
t.Fatal(err)
}
if err := m.Input(term.ID, "printf '\\033]0;build-%s\\007' ok\r"); err != nil {
t.Fatal(err)
}
waitTerminalTitle(t, m, "build-ok")
}
func waitTerminalTitle(t *testing.T, m *terminalManager, want string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
list, _ := m.List(context.Background())
if len(list) == 1 && list[0].Title == want {
return
}
if time.Now().After(deadline) {
t.Fatalf("list = %+v", list)
}
time.Sleep(50 * time.Millisecond)
}
}
+167
View File
@@ -0,0 +1,167 @@
package server
import (
"bytes"
"errors"
"fmt"
"os"
"os/exec"
"os/user"
"path/filepath"
"runtime"
"strings"
"time"
)
// A machine terminal starts at the operating system's login prompt for the
// server's own account (`su -l <user>`): the operator types that account's
// password into the terminal (the PTY hides it) before getting a shell. A
// terminal is refused rather than opened unauthenticated when su is missing,
// the server runs as root (su would not ask), or su opens a shell without
// asking.
const (
// terminalPromptWait bounds how long Open waits for su's password prompt.
terminalPromptWait = 5 * time.Second
// terminalLoginScan bounds the pre-login output searched for the prompt.
terminalLoginScan = 4 << 10
)
// terminalLoginMarker is printed by the logged-in account's shell before it
// starts the interactive shell: an OSC sequence emulators ignore. Seeing it
// means the login succeeded; seeing it before a password prompt means su
// did not ask for one.
var terminalLoginMarker = []byte("\x1b]6973;pipelines-login\x07")
const (
terminalLoginFailedReason = "The login did not complete (wrong password, or the prompt was cancelled). Close this tab and open a new terminal to try again."
terminalRestartReason = "The Pipelines server restarted and this terminal ended with it. Terminals end with the server; run tmux inside a terminal to keep work across restarts."
terminalStopReason = "The Pipelines server stopped and this terminal ended with it. Run tmux inside a terminal to keep work across restarts."
)
// terminalSuDirs are where su is looked for after PATH, per OS (a service's
// PATH may lack them). Tests replace them.
var terminalSuDirs = map[string][]string{
"darwin": {"/usr/bin"},
"linux": {"/bin", "/usr/bin"},
}
// findTerminalSu is su on PATH or in goos's usual directories; empty when
// none is installed.
func findTerminalSu(goos string) string {
if program, err := exec.LookPath("su"); err == nil {
if abs, absErr := filepath.Abs(program); absErr == nil {
return abs
}
}
for _, dir := range terminalSuDirs[goos] {
candidate := filepath.Join(dir, "su")
if info, err := os.Stat(candidate); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 {
return candidate
}
}
return ""
}
// terminalLoginArgs is the login command of a terminal starting in dir on
// goos: su for username, whose shell then cds to dir and starts the account's
// login shell. It refuses whatever could not ask for the password.
func terminalLoginArgs(goos string, euid int, su, username, dir string) ([]string, error) {
switch goos {
case "darwin", "linux":
default:
return nil, fmt.Errorf("terminals start at the login prompt (su), which Pipelines does not support on %s", goos)
}
switch {
case euid == 0:
return nil, errors.New("the Pipelines server runs as root, so su would not ask for a password; terminals are refused")
case su == "":
return nil, errors.New("su is not installed on this machine, so a terminal cannot ask for the account password; terminals are refused")
case username == "":
return nil, errors.New("the server's account has no user name to log in as; terminals are refused")
}
// Both macOS su and util-linux su take `-l <user> -c <command>` (macOS
// passes the arguments after the login name to the shell).
return []string{su, "-l", username, "-c", terminalLoginScript(dir)}, nil
}
// terminalLoginScript runs in the account's shell after a successful login:
// it marks the login, enters dir (home when that fails) and becomes the
// account's interactive login shell.
func terminalLoginScript(dir string) string {
return `printf '\033]6973;pipelines-login\007'; cd ` + terminalShellQuote(dir) + ` || cd; exec "$SHELL" -l`
}
// terminalShellQuote single-quotes s for sh (and fish).
func terminalShellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}
// terminalLoginCommand is the server's own login command for dir, or why a
// terminal is refused.
func terminalLoginCommand(dir string) (*exec.Cmd, error) {
account, err := user.Current()
name := ""
if err == nil {
name = account.Username
}
args, err := terminalLoginArgs(runtime.GOOS, os.Geteuid(), findTerminalSu(runtime.GOOS), name, dir)
if err != nil {
return nil, err
}
cmd := exec.Command(args[0], args[1:]...)
// su -l discards the environment but TERM; LC_ALL=C keeps its prompt
// English, so it is recognised.
cmd.Env = append(terminalEnv(os.Environ(), dir), "LC_ALL=C")
cmd.Dir = dir
return cmd, nil
}
// terminalLoginWatch follows a terminal's output until its login completes.
type terminalLoginWatch struct {
seen []byte
// prompted: a password prompt appeared; loggedIn: the marker appeared;
// unprompted: the marker appeared before any prompt.
prompted, loggedIn, unprompted bool
}
// feed scans more output and reports whether the state changed.
func (w *terminalLoginWatch) feed(data []byte) bool {
if w.loggedIn {
return false
}
w.seen = append(w.seen, bytes.ToLower(data)...)
if len(w.seen) > terminalLoginScan {
w.seen = w.seen[len(w.seen)-terminalLoginScan:]
}
prompt := bytes.Index(w.seen, []byte("password"))
marker := bytes.Index(w.seen, terminalLoginMarker)
changed := false
if !w.prompted && prompt >= 0 && (marker < 0 || prompt < marker) {
w.prompted, changed = true, true
}
if marker >= 0 {
w.loggedIn, w.unprompted, changed = true, !w.prompted, true
w.seen = nil
}
return changed
}
// terminalEnv is env for a terminal's process: TERM names the emulator the
// clients draw, PWD the start directory (when set), and no TMUX variables
// leak in from a tmux the server runs inside.
func terminalEnv(env []string, cwd string) []string {
out := make([]string, 0, len(env)+2)
for _, entry := range env {
if strings.HasPrefix(entry, "TERM=") || strings.HasPrefix(entry, "TMUX=") || strings.HasPrefix(entry, "TMUX_PANE=") ||
strings.HasPrefix(entry, "LC_ALL=") || (cwd != "" && strings.HasPrefix(entry, "PWD=")) {
continue
}
out = append(out, entry)
}
out = append(out, "TERM=xterm-256color")
if cwd != "" {
out = append(out, "PWD="+cwd)
}
return out
}
+236
View File
@@ -0,0 +1,236 @@
//go:build darwin || linux
package server
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// newTestTerminals is one server process's terminal manager over store.
func newTestTerminals(t *testing.T, store *pipeline.Store) (*terminalManager, context.CancelFunc) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
m := newTerminalManager(ctx, store)
m.recover(ctx)
t.Cleanup(func() { cancel(); m.Wait() })
return m, cancel
}
// waitTerminalOutput polls a terminal's retained output until it contains
// want.
func waitTerminalOutput(t *testing.T, m *terminalManager, id, want string) protocol.TerminalRecord {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
record, err := m.Output(context.Background(), id, 0)
if err != nil {
t.Fatal(err)
}
if strings.Contains(record.Output, want) {
return record
}
if time.Now().After(deadline) {
t.Fatalf("output of %s never contained %q: %q", id, want, record.Output)
}
time.Sleep(50 * time.Millisecond)
}
}
// waitTerminalStatus polls the list until terminal id has status.
func waitTerminalStatus(t *testing.T, m *terminalManager, id, status string) protocol.TerminalRecord {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
list, err := m.List(context.Background())
if err != nil {
t.Fatal(err)
}
for _, record := range list {
if record.ID == id && record.Status == status {
return record
}
}
if time.Now().After(deadline) {
t.Fatalf("terminal %s never became %s: %+v", id, status, list)
}
time.Sleep(50 * time.Millisecond)
}
}
// The login command is su for the server's own account on macOS and Linux
// (su looked up in each one's usual directories); a root server, a missing
// su and other systems are refused.
func TestTerminalLoginCommandPerOS(t *testing.T) {
dir := "/home/a/it's here"
for _, goos := range []string{"darwin", "linux"} {
args, err := terminalLoginArgs(goos, 501, "/usr/bin/su", "alan", dir)
if err != nil {
t.Fatalf("%s: %v", goos, err)
}
want := []string{"/usr/bin/su", "-l", "alan", "-c", `printf '\033]6973;pipelines-login\007'; cd '/home/a/it'\''s here' || cd; exec "$SHELL" -l`}
if strings.Join(args, "\x00") != strings.Join(want, "\x00") {
t.Fatalf("%s args = %q", goos, args)
}
}
if terminalSuDirs["darwin"][0] != "/usr/bin" || strings.Join(terminalSuDirs["linux"], ":") != "/bin:/usr/bin" {
t.Fatalf("su directories = %v", terminalSuDirs)
}
refusals := map[string]struct {
goos string
euid int
su, user string
wantPhrase string
}{
"root": {"linux", 0, "/bin/su", "root", "runs as root"},
"no su": {"darwin", 501, "", "alan", "su is not installed"},
"no user": {"linux", 1000, "/bin/su", "", "no user name"},
"windows": {"windows", 1000, "su", "alan", "does not support on windows"},
"freebsd": {"freebsd", 1000, "/usr/bin/su", "alan", "does not support on freebsd"},
}
for name, c := range refusals {
if _, err := terminalLoginArgs(c.goos, c.euid, c.su, c.user, dir); err == nil || !strings.Contains(err.Error(), c.wantPhrase) {
t.Fatalf("%s: err = %v", name, err)
}
}
}
// Without su on PATH or in the usual directories the server refuses to
// open a terminal (no unauthenticated fallback) and lists nothing.
func TestTerminalRefusedWithoutSu(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("runs as root")
}
terminalHome(t)
empty := t.TempDir()
t.Setenv("PATH", empty)
original := terminalSuDirs
terminalSuDirs = map[string][]string{"darwin": {empty}, "linux": {empty}}
t.Cleanup(func() { terminalSuDirs = original })
if found := findTerminalSu("darwin"); found != "" {
t.Fatalf("found su %q", found)
}
m, _ := newTestTerminals(t, newOperatorStore(t))
if _, err := m.Open(context.Background(), "", "", 0, 0); err == nil || !strings.Contains(err.Error(), "su is not installed") {
t.Fatalf("open without su = %v", err)
}
if list, _ := m.List(context.Background()); len(list) != 0 {
t.Fatalf("list = %+v", list)
}
fakeTerminalSu(t)
if found := findTerminalSu("linux"); !strings.HasSuffix(found, "/su") || filepath.Dir(found) == empty {
t.Fatalf("fake su = %q", found)
}
}
// A new terminal shows the login prompt in a real PTY and runs su, not a
// shell or tmux; a wrong password ends it with a clear reason, the right
// one gives a shell in the start folder.
func TestTerminalShowsLoginPrompt(t *testing.T) {
home := terminalHome(t)
sub := filepath.Join(home, "sub dir")
if err := os.Mkdir(sub, 0o755); err != nil {
t.Fatal(err)
}
m, _ := newTestTerminals(t, newOperatorStore(t))
wrong, err := m.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(strings.Fields(wrong.Shell)[0], "/su") || strings.Contains(wrong.Shell, "tmux") {
t.Fatalf("shell = %q", wrong.Shell)
}
if raw, _ := json.Marshal(wrong); strings.Contains(string(raw), "tmux") {
t.Fatalf("record mentions tmux: %s", raw)
}
if out, _ := m.Output(context.Background(), wrong.ID, 0); !strings.Contains(out.Output, "Password:") {
t.Fatalf("output = %q", out.Output)
}
if err := m.Input(wrong.ID, "not-it\r"); err != nil {
t.Fatal(err)
}
ended := waitTerminalStatus(t, m, wrong.ID, protocol.TerminalExited)
if ended.Reason != terminalLoginFailedReason || ended.ExitCode == nil || *ended.ExitCode != 1 {
t.Fatalf("wrong password = %+v", ended)
}
waitTerminalOutput(t, m, wrong.ID, "Authentication failed")
right, err := m.Open(context.Background(), "", "~/sub dir", 0, 0)
if err != nil {
t.Fatal(err)
}
if err := m.Input(right.ID, fakeSuPassword+"\r"); err != nil {
t.Fatal(err)
}
if err := m.Input(right.ID, "pwd; echo in-$((6*7))\r"); err != nil {
t.Fatal(err)
}
out := waitTerminalOutput(t, m, right.ID, "in-42")
if !strings.Contains(out.Output, sub) || strings.Contains(out.Output, fakeSuPassword) {
t.Fatalf("output = %q", out.Output)
}
if p := m.process(right.ID); p == nil || !loggedIn(p) {
t.Fatal("the login was not recorded")
}
}
func loggedIn(p *terminalProcess) bool {
login, _ := p.loginState()
return login.loggedIn
}
// The watch sees the prompt across reads, the marker after it as a login,
// and the marker before any prompt as an unprompted login.
func TestTerminalLoginWatch(t *testing.T) {
var w terminalLoginWatch
if w.feed([]byte("Pass")) || !w.feed([]byte("word:")) || !w.prompted || w.loggedIn {
t.Fatalf("prompt = %+v", w)
}
if !w.feed(append([]byte("\r\n"), terminalLoginMarker...)) || !w.loggedIn || w.unprompted {
t.Fatalf("login = %+v", w)
}
if w.feed([]byte("password")) {
t.Fatal("output after the login is not watched")
}
var bare terminalLoginWatch
bare.feed(append(append([]byte("motd\r\n"), terminalLoginMarker...), "change your password"...))
if !bare.loggedIn || !bare.unprompted || bare.prompted {
t.Fatalf("unprompted = %+v", bare)
}
}
// A login that never asks for a password is refused and never listed: one
// that opens a shell at once (an account without a password), one that
// stays silent, one that fails without a terminal.
func TestTerminalRefusesUnpromptedLogin(t *testing.T) {
terminalHome(t)
m, _ := newTestTerminals(t, newOperatorStore(t))
m.promptWait = 500 * time.Millisecond
for mode, phrase := range map[string]string{
"noprompt": "without asking for a password",
"silent": "did not ask for a password",
"notty": "su: must be run from a terminal",
} {
t.Setenv("FAKE_SU_MODE", mode)
if record, err := m.Open(context.Background(), "", "", 0, 0); err == nil || !strings.Contains(err.Error(), phrase) {
t.Fatalf("%s: open = %+v, %v", mode, record, err)
}
}
if list, _ := m.List(context.Background()); len(list) != 0 {
t.Fatalf("list = %+v", list)
}
m.mu.Lock()
running := m.runningLocked()
m.mu.Unlock()
if running != 0 {
t.Fatalf("%d refused terminals still running", running)
}
}
+93 -59
View File
@@ -7,8 +7,8 @@ import (
"errors"
"fmt"
"log"
"os"
"os/exec"
"strings"
"sync"
"syscall"
"time"
@@ -25,35 +25,27 @@ type terminalStore interface {
Terminals(ctx context.Context) ([]protocol.TerminalRecord, error)
}
const (
terminalNoTmuxReason = "The Pipelines server restarted and this shell ended with it (tmux is not installed on this machine, so shells do not outlive the server)."
terminalGoneReason = "Its tmux session ended while the Pipelines server was stopped."
terminalStopReason = "The Pipelines server stopped and this shell ended with it (tmux is not installed on this machine)."
// terminalReattachLimit bounds how often a tmux client that exits while
// its session lives is replaced before the terminal is given up.
terminalReattachLimit = 5
)
// terminalManager owns the machine's terminals (terminal.go in protocol).
type terminalManager struct {
base context.Context
store terminalStore
clients *terminalClients
tmux *terminalTmux
shell func() string
home func() (string, error)
now func() time.Time
flush time.Duration
// login is the login command of a terminal starting in a directory.
login func(dir string) (*exec.Cmd, error)
promptWait time.Duration
home func() (string, error)
now func() time.Time
flush time.Duration
mu sync.Mutex
terms map[string]*terminalProcess
wg sync.WaitGroup
}
func newTerminalManager(base context.Context, store terminalStore, tmux *terminalTmux) *terminalManager {
func newTerminalManager(base context.Context, store terminalStore) *terminalManager {
return &terminalManager{
base: base, store: store, clients: &terminalClients{}, tmux: tmux,
shell: chatShellProgram, home: OperatorHomeDirectory, now: time.Now, flush: 50 * time.Millisecond,
base: base, store: store, clients: &terminalClients{},
login: terminalLoginCommand, promptWait: terminalPromptWait, home: OperatorHomeDirectory, now: time.Now, flush: 50 * time.Millisecond,
terms: map[string]*terminalProcess{},
}
}
@@ -66,8 +58,10 @@ func newTerminalID() (string, error) {
return "term_" + hex.EncodeToString(b[:]), nil
}
// Open starts a login shell in cwd at cols x rows and returns its record.
// clientID (when set) is subscribed to its output before the shell writes.
// Open starts a terminal in cwd at cols x rows at the machine's login prompt
// and returns its record once the prompt shows. clientID (when set) is
// subscribed to its output before the login writes. A login that cannot ask
// for the password is refused.
func (m *terminalManager) Open(ctx context.Context, clientID, cwd string, cols, rows int) (protocol.TerminalRecord, error) {
if m.store == nil {
return protocol.TerminalRecord{}, errors.New("terminals need the server's store, which is not configured")
@@ -87,44 +81,46 @@ func (m *terminalManager) Open(ctx context.Context, clientID, cwd string, cols,
if err != nil {
return protocol.TerminalRecord{}, err
}
cmd, err := m.login(dir)
if err != nil {
return protocol.TerminalRecord{}, err
}
p, err := m.start(clientID, id, dir, cmd, cols, rows)
if err != nil {
return protocol.TerminalRecord{}, err
}
if err := m.awaitPrompt(p); err != nil {
m.refuse(p)
return protocol.TerminalRecord{}, err
}
record := p.snapshot(false, 0)
if err := m.store.SaveTerminal(ctx, record, protocol.TerminalEndedKept); err != nil {
m.refuse(p)
return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err)
}
record = p.snapshot(false, 0)
m.publishRecord(record)
return record, nil
}
// start runs cmd in a new terminal's PTY and supervises it; the terminal
// counts against the limit from here on.
func (m *terminalManager) start(clientID, id, dir string, cmd *exec.Cmd, cols, rows int) (*terminalProcess, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.base.Err() != nil {
return protocol.TerminalRecord{}, errors.New("the server is stopping")
return nil, errors.New("the server is stopping")
}
if running := m.runningLocked(); running >= protocol.TerminalOpenLimit {
return protocol.TerminalRecord{}, fmt.Errorf("this machine already has %d open terminals; close one first", running)
return nil, fmt.Errorf("this machine already has %d open terminals; close one first", running)
}
cols, rows = chatShellTerminalSize(cols, rows)
shell := m.shell()
shell := strings.Join(cmd.Args[:min(len(cmd.Args), 3)], " ")
record := protocol.TerminalRecord{ID: id, Cwd: dir, Shell: shell, CreatedAt: m.now().UnixMilli(), Status: protocol.TerminalRunning, Cols: cols, Rows: rows}
var cmd *exec.Cmd
if m.tmux != nil {
record.Tmux = terminalTmuxName(id)
if err := m.tmux.create(record.Tmux, dir, shell, cols, rows); err != nil {
return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err)
}
cmd = m.tmux.attach(record.Tmux)
} else {
cmd = exec.Command(shell, "-l")
cmd.Env = terminalEnv(os.Environ(), dir)
}
cmd.Dir = dir
p := newTerminalProcess(record)
terminal, err := p.startAttachment(cmd)
if err == nil {
err = m.store.SaveTerminal(ctx, record, protocol.TerminalEndedKept)
if err != nil {
p.signal(syscall.SIGKILL)
terminal.Close()
_ = cmd.Wait()
}
}
if err != nil {
if record.Tmux != "" {
_ = m.tmux.kill(record.Tmux)
}
return protocol.TerminalRecord{}, fmt.Errorf("could not start the terminal: %v", err)
return nil, fmt.Errorf("could not start the terminal: %v", err)
}
m.terms[id] = p
if clientID != "" {
@@ -133,8 +129,55 @@ func (m *terminalManager) Open(ctx context.Context, clientID, cwd string, cols,
m.wg.Add(1)
go m.supervise(p, cmd, terminal)
go p.pumpInput()
m.publishRecord(record)
return record, nil
return p, nil
}
// awaitPrompt waits for the login's password prompt: a login that ends,
// opens a shell without asking, or stays silent is refused.
func (m *terminalManager) awaitPrompt(p *terminalProcess) error {
timer := time.NewTimer(m.promptWait)
defer timer.Stop()
for {
login, _ := p.loginState()
switch {
case login.unprompted:
return errors.New("su opened a shell without asking for a password (does the account have none?); terminals are refused")
case login.prompted:
return nil
}
select {
case <-p.loginChanged:
case <-p.done:
login, output := p.loginState()
if login.prompted && !login.unprompted {
return nil
}
return fmt.Errorf("the login prompt (su) ended before asking for a password: %s", terminalLastLine(output))
case <-timer.C:
return fmt.Errorf("the login prompt (su) did not ask for a password within %s; terminals are refused", m.promptWait)
}
}
}
// refuse ends a terminal that never showed its prompt, without listing it.
func (m *terminalManager) refuse(p *terminalProcess) {
p.mu.Lock()
p.closing, p.refused = true, true
p.mu.Unlock()
p.signal(syscall.SIGKILL)
<-p.done
}
// terminalLastLine is the last non-empty line of a terminal's output, for a
// refusal message.
func terminalLastLine(output string) string {
lines := strings.Split(strings.ReplaceAll(output, "\r", ""), "\n")
for i := len(lines) - 1; i >= 0; i-- {
if line := strings.TrimSpace(lines[i]); line != "" {
return line
}
}
return "it printed nothing"
}
func (m *terminalManager) runningLocked() int {
@@ -244,7 +287,7 @@ func (m *terminalManager) Output(ctx context.Context, id string, after int64) (p
return protocol.TerminalRecord{}, errors.New("this machine has no such terminal")
}
// Close ends a terminal (killing its shell and tmux session) and forgets
// Close ends a terminal (killing its login and shell) and forgets
// it. It waits briefly for the shell to go and returns the closed record.
func (m *terminalManager) Close(ctx context.Context, id string) (protocol.TerminalRecord, error) {
if m.store == nil {
@@ -254,13 +297,7 @@ func (m *terminalManager) Close(ctx context.Context, id string) (protocol.Termin
if p != nil && p.running() {
p.mu.Lock()
p.closing = true
name := p.record.Tmux
p.mu.Unlock()
if name != "" && m.tmux != nil {
if err := m.tmux.kill(name); err != nil {
log.Printf("terminal close %s: %v", id, err)
}
}
p.signal(syscall.SIGHUP)
select {
case <-p.done:
@@ -274,9 +311,6 @@ func (m *terminalManager) Close(ctx context.Context, id string) (protocol.Termin
if err != nil {
return protocol.TerminalRecord{}, err
}
if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) {
_ = m.tmux.kill(record.Tmux)
}
m.forget(id)
record.Output, record.OutputStart = "", 0
record.Status = protocol.TerminalClosed
+27 -6
View File
@@ -14,21 +14,27 @@ import (
)
// terminalProcess is one terminal this server process holds: running (its
// attachment is the shell itself, or a tmux client showing the shell's
// session) or ended with its output still retained.
// attachment is the login, then the shell it becomes) or ended with its
// output still retained.
type terminalProcess struct {
mu sync.Mutex
record protocol.TerminalRecord
out *chatShellOutput
titles terminalTitles
// cmd and pty are the current attachment; a tmux client that exits while
// its session lives is replaced.
// login follows the output until the login completes; loginChanged is
// signalled when it changes.
login terminalLoginWatch
loginChanged chan struct{}
// cmd and pty are the running login or shell.
cmd *exec.Cmd
pty *os.File
input chan []byte
done chan struct{}
// closing is set by terminal_close: the end is a close, not an exit.
closing bool
// refused: the login never asked for a password; the terminal is
// forgotten without being listed.
refused bool
}
func newTerminalProcess(record protocol.TerminalRecord) *terminalProcess {
@@ -40,9 +46,19 @@ func newTerminalProcess(record protocol.TerminalRecord) *terminalProcess {
out: newChatShellOutput(protocol.TerminalRetained),
input: make(chan []byte, protocol.ChatShellInputQueue),
done: make(chan struct{}),
loginChanged: make(chan struct{}, 1),
}
}
// loginState is the login's progress and the output retained so far.
func (p *terminalProcess) loginState() (terminalLoginWatch, string) {
p.mu.Lock()
defer p.mu.Unlock()
text, _ := p.out.tail(terminalLoginScan)
return p.login, text
}
func (p *terminalProcess) running() bool {
p.mu.Lock()
defer p.mu.Unlock()
@@ -96,6 +112,12 @@ func (p *terminalProcess) read(terminal *os.File, titled func(string)) {
if n > 0 {
p.mu.Lock()
p.out.write(buffer[:n])
if p.login.feed(buffer[:n]) {
select {
case p.loginChanged <- struct{}{}:
default:
}
}
title, changed := p.titles.feed(buffer[:n])
changed = changed && title != p.record.Title
if changed {
@@ -112,8 +134,7 @@ func (p *terminalProcess) read(terminal *os.File, titled func(string)) {
}
}
// pumpInput writes queued input to the current attachment until the
// terminal ends. Input arriving while a tmux client is replaced is dropped.
// pumpInput writes queued input to the terminal until it ends.
func (p *terminalProcess) pumpInput() {
for {
select {
-223
View File
@@ -1,223 +0,0 @@
//go:build darwin || linux
package server
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// testTmux is a tmux server of this test alone, killed afterwards.
func testTmux(t *testing.T) *terminalTmux {
t.Helper()
tmux := findTerminalTmux(fmt.Sprintf("pipelines-test-%d-%d", os.Getpid(), time.Now().UnixNano()))
if tmux == nil {
t.Skip("tmux is not installed")
}
t.Cleanup(func() {
_ = tmux.command("kill-server").Run()
dir := os.Getenv("TMUX_TMPDIR")
if dir == "" {
dir = "/tmp"
}
_ = os.Remove(filepath.Join(dir, fmt.Sprintf("tmux-%d", os.Getuid()), tmux.socket))
})
return tmux
}
// newTestTerminals is one server process's terminal manager over store.
func newTestTerminals(t *testing.T, store *pipeline.Store, tmux *terminalTmux) (*terminalManager, context.CancelFunc) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
m := newTerminalManager(ctx, store, tmux)
m.shell = func() string { return "/bin/sh" }
m.recover(ctx)
t.Cleanup(func() { cancel(); m.Wait() })
return m, cancel
}
// waitOutput polls a terminal's retained output until it contains want.
func waitTerminalOutput(t *testing.T, m *terminalManager, id, want string) protocol.TerminalRecord {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
record, err := m.Output(context.Background(), id, 0)
if err != nil {
t.Fatal(err)
}
if strings.Contains(record.Output, want) {
return record
}
if time.Now().After(deadline) {
t.Fatalf("output of %s never contained %q: %q", id, want, record.Output)
}
time.Sleep(50 * time.Millisecond)
}
}
// A tmux-backed shell outlives its server process: the next one lists it as
// running, reattaches, redraws its screen and takes input again. Closing it
// ends the tmux session.
func TestTerminalTmuxSurvivesRestart(t *testing.T) {
terminalHome(t)
tmux := testTmux(t)
store := newOperatorStore(t)
first, stop := newTestTerminals(t, store, tmux)
term, err := first.Open(context.Background(), "", "~", 90, 20)
if err != nil {
t.Fatal(err)
}
if term.Tmux != terminalTmuxName(term.ID) {
t.Fatalf("tmux name = %q", term.Tmux)
}
if err := first.Input(term.ID, "export KEEP=still-here; echo marker-$((6*7))\r"); err != nil {
t.Fatal(err)
}
waitTerminalOutput(t, first, term.ID, "marker-42")
stop()
first.Wait()
if !tmux.alive(term.Tmux) {
t.Fatal("the tmux session ended with the server")
}
second, _ := newTestTerminals(t, store, tmux)
list, err := second.List(context.Background())
if err != nil {
t.Fatal(err)
}
if len(list) != 1 || list[0].ID != term.ID || list[0].Status != protocol.TerminalRunning {
t.Fatalf("list after restart = %+v", list)
}
// The reattached client redraws the screen, earlier output included,
// and the shell keeps its state.
waitTerminalOutput(t, second, term.ID, "marker-42")
if err := second.Input(term.ID, "echo $KEEP\r"); err != nil {
t.Fatal(err)
}
waitTerminalOutput(t, second, term.ID, "still-here")
closed, err := second.Close(context.Background(), term.ID)
if err != nil || closed.Status != protocol.TerminalClosed {
t.Fatalf("close = %+v, %v", closed, err)
}
if tmux.alive(term.Tmux) {
t.Fatal("the tmux session outlived close")
}
if list, _ := second.List(context.Background()); len(list) != 0 {
t.Fatalf("list after close = %+v", list)
}
}
// A tmux client killed while its session lives is replaced.
func TestTerminalTmuxClientReplaced(t *testing.T) {
terminalHome(t)
tmux := testTmux(t)
m, _ := newTestTerminals(t, newOperatorStore(t), tmux)
term, err := m.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
waitTerminalOutput(t, m, term.ID, "$")
if err := tmux.command("detach-client", "-s", "="+term.Tmux).Run(); err != nil {
t.Fatal(err)
}
time.Sleep(300 * time.Millisecond)
if err := m.Input(term.ID, "echo back-$((1+1))\r"); err != nil {
t.Fatal(err)
}
waitTerminalOutput(t, m, term.ID, "back-2")
if record, _ := m.Output(context.Background(), term.ID, 0); record.Status != protocol.TerminalRunning {
t.Fatalf("record = %+v", record)
}
}
// Without tmux a shell ends with its server: the stop is recorded as
// ended, and a record a crashed server left running is marked ended too.
func TestTerminalWithoutTmuxEndsWithServer(t *testing.T) {
terminalHome(t)
store := newOperatorStore(t)
first, stop := newTestTerminals(t, store, nil)
term, err := first.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
stop()
first.Wait()
crashed := protocol.TerminalRecord{ID: "term_crashed", Cwd: "/", CreatedAt: 1, Status: protocol.TerminalRunning, Cols: 80, Rows: 24}
gone := protocol.TerminalRecord{ID: "term_gone", Cwd: "/", CreatedAt: 2, Tmux: "pipelines-term_gone", Status: protocol.TerminalRunning}
for _, record := range []protocol.TerminalRecord{crashed, gone} {
if err := store.SaveTerminal(context.Background(), record, 0); err != nil {
t.Fatal(err)
}
}
second, _ := newTestTerminals(t, store, testTmux(t))
list, err := second.List(context.Background())
if err != nil {
t.Fatal(err)
}
reasons := map[string]string{}
for _, record := range list {
if record.Status != protocol.TerminalExited || record.EndedAt == 0 {
t.Fatalf("record = %+v", record)
}
reasons[record.ID] = record.Reason
}
if reasons[term.ID] != terminalStopReason || reasons["term_crashed"] != terminalNoTmuxReason || reasons["term_gone"] != terminalGoneReason {
t.Fatalf("reasons = %+v", reasons)
}
if _, err := second.Close(context.Background(), "term_gone"); err != nil {
t.Fatal(err)
}
if list, _ := second.List(context.Background()); len(list) != 2 {
t.Fatalf("list after closing an ended terminal = %+v", list)
}
}
// The window title the shell sets is followed across reads and recorded.
func TestTerminalTitles(t *testing.T) {
var titles terminalTitles
if _, ok := titles.feed([]byte("plain \x1b[1mtext")); ok {
t.Fatal("title from plain text")
}
if _, ok := titles.feed([]byte("\x1b]0;~/src")); ok {
t.Fatal("title before its end")
}
if title, ok := titles.feed([]byte("/app\x07more")); !ok || title != "~/src/app" {
t.Fatalf("title = %q %v", title, ok)
}
if title, ok := titles.feed([]byte("\x1b]2;vim\x01 x\x1b\\")); !ok || title != "vim x" {
t.Fatalf("ST title = %q %v", title, ok)
}
if _, ok := titles.feed([]byte("\x1b]8;;https://x\x07")); ok {
t.Fatal("a hyperlink is not a title")
}
if title, _ := titles.feed([]byte("\x1b]0;" + strings.Repeat("é", protocol.TerminalTitleLimit) + "\x07")); len(title) > protocol.TerminalTitleLimit {
t.Fatalf("title of %d bytes", len(title))
}
terminalHome(t)
m, _ := newTestTerminals(t, newOperatorStore(t), nil)
term, err := m.Open(context.Background(), "", "", 0, 0)
if err != nil {
t.Fatal(err)
}
if err := m.Input(term.ID, "printf '\\033]0;build-%s\\007' ok\r"); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for {
list, _ := m.List(context.Background())
if len(list) == 1 && list[0].Title == "build-ok" {
break
}
if time.Now().After(deadline) {
t.Fatalf("list = %+v", list)
}
time.Sleep(50 * time.Millisecond)
}
}
+40 -76
View File
@@ -11,87 +11,76 @@ import (
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// supervise streams one terminal until it ends: its first attachment is
// cmd on terminal. A tmux client that exits while its session lives (it was
// detached or killed) is replaced; a server shutdown leaves a tmux session
// running for the next server process and ends a direct shell.
// supervise streams one terminal (cmd on terminal) until it ends: its login
// or shell exits, it is closed, or the server stops, which ends it too.
func (m *terminalManager) supervise(p *terminalProcess, cmd *exec.Cmd, terminal *os.File) {
defer m.wg.Done()
defer close(p.done)
stopFlush, flushDone := make(chan struct{}), make(chan struct{})
go m.flushLoop(p, stopFlush, flushDone)
titled := func(string) {
record := p.snapshot(false, 0)
m.save(record)
m.publishRecord(record)
p.mu.Lock()
listed := !p.refused && p.login.prompted
p.mu.Unlock()
if listed {
record := p.snapshot(false, 0)
m.save(record)
m.publishRecord(record)
}
}
readDone := make(chan struct{})
go func() {
defer close(readDone)
p.read(terminal, titled)
}()
attachDone := make(chan error, 1)
go func() { attachDone <- p.waitAttachment(cmd, terminal, readDone) }()
var waitErr error
shutdown := false
for reattached := 0; ; reattached++ {
readDone := make(chan struct{})
go func(terminal *os.File) {
defer close(readDone)
p.read(terminal, titled)
}(terminal)
attachDone := make(chan error, 1)
go func(cmd *exec.Cmd, terminal *os.File) { attachDone <- p.waitAttachment(cmd, terminal, readDone) }(cmd, terminal)
select {
case waitErr = <-attachDone:
case <-m.base.Done():
p.signal(syscall.SIGHUP)
select {
case waitErr = <-attachDone:
case <-m.base.Done():
p.signal(syscall.SIGHUP)
select {
case waitErr = <-attachDone:
case <-time.After(time.Second):
p.signal(syscall.SIGKILL)
waitErr = <-attachDone
}
case <-time.After(time.Second):
p.signal(syscall.SIGKILL)
waitErr = <-attachDone
}
if m.base.Err() != nil {
shutdown = true
break
}
p.mu.Lock()
closing, name := p.closing, p.record.Tmux
p.mu.Unlock()
if closing || name == "" || m.tmux == nil || reattached >= terminalReattachLimit || !m.tmux.alive(name) {
break
}
cmd = m.tmux.attach(name)
next, err := p.startAttachment(cmd)
if err != nil {
log.Printf("terminal reattach %s: %v", p.record.ID, err)
break
}
terminal = next
}
shutdown := m.base.Err() != nil
close(stopFlush)
<-flushDone
m.emit(p, true)
p.mu.Lock()
// A terminal that never showed its prompt was never listed: Open
// refuses it, so it is forgotten unannounced.
listed := p.login.prompted && !p.refused
switch {
case shutdown && p.record.Tmux != "":
// Still running in tmux: the next server process reattaches it.
case p.closing:
p.record.Status = protocol.TerminalClosed
default:
p.record.Status = protocol.TerminalExited
p.record.EndedAt = m.now().UnixMilli()
if shutdown {
switch {
case shutdown:
p.record.Reason = terminalStopReason
} else if p.record.Tmux == "" {
case !p.login.loggedIn:
p.record.Reason = terminalLoginFailedReason
p.record.ExitCode = terminalExitCode(waitErr)
default:
p.record.ExitCode = terminalExitCode(waitErr)
}
}
p.pty, p.cmd = nil, nil
p.mu.Unlock()
record := p.snapshot(false, 0)
switch record.Status {
case protocol.TerminalClosed:
switch {
case !listed || record.Status == protocol.TerminalClosed:
m.forget(record.ID)
default:
m.save(record)
}
if !shutdown {
if !shutdown && listed {
m.publishRecord(record)
}
}
@@ -122,9 +111,8 @@ func (m *terminalManager) emit(p *terminalProcess, final bool) {
}
}
// recover lists what a previous server process left running: a tmux-backed
// shell whose session is alive is reattached; any other is recorded as
// ended, saying why.
// recover records the terminals a previous server process left running as
// ended: they ended with it.
func (m *terminalManager) recover(ctx context.Context) {
if m.store == nil {
return
@@ -138,32 +126,8 @@ func (m *terminalManager) recover(ctx context.Context) {
if record.Status != protocol.TerminalRunning {
continue
}
if record.Tmux != "" && m.tmux != nil && m.tmux.alive(record.Tmux) && m.reattach(record) {
continue
}
record.Status, record.EndedAt = protocol.TerminalExited, m.now().UnixMilli()
record.Reason = terminalNoTmuxReason
if record.Tmux != "" {
record.Reason = terminalGoneReason
}
record.Reason = terminalRestartReason
m.save(record)
}
}
// reattach shows a live tmux session in a new client and supervises it.
func (m *terminalManager) reattach(record protocol.TerminalRecord) bool {
cmd := m.tmux.attach(record.Tmux)
p := newTerminalProcess(record)
terminal, err := p.startAttachment(cmd)
if err != nil {
log.Printf("terminal reattach %s: %v", record.ID, err)
return false
}
m.mu.Lock()
m.terms[record.ID] = p
m.mu.Unlock()
m.wg.Add(1)
go m.supervise(p, cmd, terminal)
go p.pumpInput()
return true
}
-120
View File
@@ -1,120 +0,0 @@
package server
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// terminalTmuxSocket is the tmux server Pipelines owns (`tmux -L pipelines`):
// its sessions never mix with the operator's own tmux server.
const terminalTmuxSocket = "pipelines"
// terminalTmux runs one Pipelines-owned tmux server. Every terminal shell is
// the only pane of its own session there, so it outlives the Pipelines
// server; the server talks to it through an ordinary attached client in a
// pseudo-terminal.
type terminalTmux struct {
program string
socket string
}
// findTerminalTmux is the tmux on PATH (or the usual package-manager
// locations a service's PATH may lack); nil when none is installed.
func findTerminalTmux(socket string) *terminalTmux {
if socket == "" {
socket = terminalTmuxSocket
}
program, err := exec.LookPath("tmux")
if err != nil {
for _, dir := range []string{"/opt/homebrew/bin", "/usr/local/bin", "/usr/bin"} {
candidate := filepath.Join(dir, "tmux")
if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() && info.Mode()&0o111 != 0 {
program, err = candidate, nil
break
}
}
}
if err != nil {
return nil
}
return &terminalTmux{program: program, socket: socket}
}
// terminalTmuxName is the tmux session of terminal id.
func terminalTmuxName(id string) string { return "pipelines-" + id }
// command is one tmux client invocation against the Pipelines server, with
// no configuration file and outside any tmux the server itself runs in.
func (t *terminalTmux) command(args ...string) *exec.Cmd {
cmd := exec.Command(t.program, append([]string{"-L", t.socket, "-f", os.DevNull}, args...)...)
cmd.Env = terminalEnv(os.Environ(), "")
return cmd
}
// create starts session name running shell -l in cwd at cols x rows,
// detached. The first call also starts the tmux server; the options make
// it invisible (no status line), pass the shell's window title through,
// keep a bounded scrollback reachable by the mouse wheel, and leave Esc
// prompt.
func (t *terminalTmux) create(name, cwd, shell string, cols, rows int) error {
cmd := t.command(
"start-server", ";",
"set-option", "-g", "history-limit", strconv.Itoa(protocol.TerminalTmuxHistory), ";",
"set-option", "-g", "status", "off", ";",
"set-option", "-g", "mouse", "on", ";",
"set-option", "-g", "escape-time", "10", ";",
"set-option", "-g", "set-titles", "on", ";",
"set-option", "-g", "set-titles-string", "#{pane_title}", ";",
"new-session", "-d", "-s", name, "-c", cwd, "-x", strconv.Itoa(cols), "-y", strconv.Itoa(rows), "--", shell, "-l", ";",
"select-pane", "-t", "="+name+":", "-T", "",
)
cmd.Dir = cwd
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("tmux new-session: %v: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
// alive reports whether session name still exists.
func (t *terminalTmux) alive(name string) bool {
return t.command("has-session", "-t", "="+name).Run() == nil
}
// attach is the client command showing session name in the server's
// pseudo-terminal.
func (t *terminalTmux) attach(name string) *exec.Cmd {
return t.command("attach-session", "-t", "="+name)
}
// kill ends session name and its shell.
func (t *terminalTmux) kill(name string) error {
if out, err := t.command("kill-session", "-t", "="+name).CombinedOutput(); err != nil && t.alive(name) {
return fmt.Errorf("tmux kill-session: %v: %s", err, strings.TrimSpace(string(out)))
}
return nil
}
// terminalEnv is env for a terminal's process: TERM names the emulator the
// clients draw, PWD the start directory (when set), and no TMUX variables
// leak in from a tmux the server runs inside.
func terminalEnv(env []string, cwd string) []string {
out := make([]string, 0, len(env)+2)
for _, entry := range env {
if strings.HasPrefix(entry, "TERM=") || strings.HasPrefix(entry, "TMUX=") || strings.HasPrefix(entry, "TMUX_PANE=") ||
(cwd != "" && strings.HasPrefix(entry, "PWD=")) {
continue
}
out = append(out, entry)
}
out = append(out, "TERM=xterm-256color")
if cwd != "" {
out = append(out, "PWD="+cwd)
}
return out
}
+54 -7
View File
@@ -4,6 +4,7 @@ package server
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
@@ -13,8 +14,9 @@ import (
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// terminalHome points the operator's home at a fresh directory and the
// login shell at /bin/sh, so tests never read the operator's own profile.
// terminalHome points the operator's home at a fresh directory, the login
// shell at /bin/sh (so tests never read the operator's own profile) and su
// at the fake below.
func terminalHome(t *testing.T) string {
t.Helper()
home, err := filepath.EvalSymlinks(t.TempDir())
@@ -26,9 +28,51 @@ func terminalHome(t *testing.T) string {
t.Cleanup(func() { operatorHomeDirectory = original })
t.Setenv("SHELL", "/bin/sh")
t.Setenv("ENV", "")
fakeTerminalSu(t)
return home
}
// fakeSuPassword is the only password the fake su accepts.
const fakeSuPassword = "test-password"
// fakeSuScript stands in for su on PATH, so tests never need a real
// password: `su -l <user> -c <script>` prints "Password:", reads a line with
// echo off, and runs the script in /bin/sh when it is fakeSuPassword.
// FAKE_SU_MODE=noprompt runs the script without asking, silent never
// prompts, notty fails as su does without a terminal.
const fakeSuScript = `#!/bin/sh
[ "$1" = -l ] && [ "$3" = -c ] || { echo "fake su: bad arguments: $*"; exit 2; }
case "$FAKE_SU_MODE" in
noprompt) SHELL=/bin/sh exec /bin/sh -c "$4" ;;
silent) exec sleep 30 ;;
notty) echo "su: must be run from a terminal"; exit 1 ;;
esac
stty -echo 2>/dev/null
printf 'Password:'
IFS= read -r password
stty echo 2>/dev/null
printf '\r\n'
[ "$password" = "` + fakeSuPassword + `" ] || { echo "su: Authentication failed"; exit 1; }
SHELL=/bin/sh exec /bin/sh -c "$4"
`
// fakeTerminalSu puts the fake su first on PATH.
func fakeTerminalSu(t *testing.T) {
t.Helper()
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "su"), []byte(fakeSuScript), 0o755); err != nil {
t.Fatal(err)
}
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("FAKE_SU_MODE", "")
}
// terminalLogIn types the fake su's password into terminal id.
func terminalLogIn(t *testing.T, ctx context.Context, conn *websocket.Conn, id string) {
t.Helper()
terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalInput, "password", protocol.TerminalRequest{TerminalID: id, Data: fakeSuPassword + "\r"}))
}
func terminalRequest(kind, id string, body protocol.TerminalRequest) protocol.Message {
return protocol.Message{Type: kind, ID: id, TerminalRequest: &body}
}
@@ -62,12 +106,13 @@ func readTerminalUntil(t *testing.T, ctx context.Context, conn *websocket.Conn,
// lists it, resumes its output by offset and closes it.
func TestTerminalOverWebsocket(t *testing.T) {
home := terminalHome(t)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 2)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t)}, 2)
opened := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{Cols: 80, Rows: 24}))
term := opened.Terminal
if opened.Type != protocol.TerminalOpen || term == nil || term.Cwd != home || term.Status != protocol.TerminalRunning || term.Cols != 80 || term.Tmux != "" || term.Shell != "/bin/sh" {
if opened.Type != protocol.TerminalOpen || term == nil || term.Cwd != home || term.Status != protocol.TerminalRunning || term.Cols != 80 || !strings.HasSuffix(strings.Fields(term.Shell)[0], "/su") {
t.Fatalf("open reply = %+v", opened)
}
terminalLogIn(t, ctx, conns[0], term.ID)
if got := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "in", protocol.TerminalRequest{TerminalID: term.ID, Data: "pwd; echo sum-$((40+2))\r"})); got.Type != protocol.TerminalInput {
t.Fatalf("input reply = %+v", got)
}
@@ -82,7 +127,8 @@ func TestTerminalOverWebsocket(t *testing.T) {
}
// The second client reads everything once, then only what follows.
all := terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalOutput, "all", protocol.TerminalRequest{TerminalID: term.ID})).Terminal
if all.OutputStart != 0 || !strings.Contains(all.Output, home) || !strings.Contains(all.Output, "sum-42") {
if all.OutputStart != 0 || !strings.Contains(all.Output, "Password:") || !strings.Contains(all.Output, home) || !strings.Contains(all.Output, "sum-42") ||
strings.Contains(all.Output, fakeSuPassword) {
t.Fatalf("output = %+v", all)
}
terminalReply(t, ctx, conns[1], terminalRequest(protocol.TerminalInput, "in2", protocol.TerminalRequest{TerminalID: term.ID, Data: "stty size\r"}))
@@ -114,8 +160,9 @@ func TestTerminalOverWebsocket(t *testing.T) {
// keeps its output until it is closed.
func TestTerminalShellExit(t *testing.T) {
terminalHome(t)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 1)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t)}, 1)
term := terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalOpen, "open", protocol.TerminalRequest{})).Terminal
terminalLogIn(t, ctx, conns[0], term.ID)
terminalReply(t, ctx, conns[0], terminalRequest(protocol.TerminalInput, "in", protocol.TerminalRequest{TerminalID: term.ID, Data: "echo bye; exit 3\r"}))
exited := readShellFrame(t, ctx, conns[0], func(m protocol.Message) bool {
return m.Type == protocol.TerminalEvent && m.TerminalEvent.Terminal != nil && m.TerminalEvent.Terminal.Status == protocol.TerminalExited
@@ -136,7 +183,7 @@ func TestTerminalShellExit(t *testing.T) {
// missing, input bounds, unknown terminals, the open limit.
func TestTerminalLimitsAndValidation(t *testing.T) {
home := terminalHome(t)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t), TerminalWithoutTmux: true}, 1)
ctx, _, conns := dialChatConns(t, Options{OperatorRegistry: newOperatorStore(t)}, 1)
conn := conns[0]
for i, cwd := range []string{"/", filepath.Join(home, "missing"), "relative", "~other"} {
if got := terminalReply(t, ctx, conn, terminalRequest(protocol.TerminalOpen, "bad", protocol.TerminalRequest{Cwd: cwd})); got.Type != protocol.Error {
+3 -12
View File
@@ -66,12 +66,6 @@ type Options struct {
// False keeps the previous behaviour (tests, embedded servers): a stopping
// server interrupts its runs and the next start marks them interrupted.
DetachSessions bool
// TerminalTmuxSocket names the Pipelines-owned tmux server machine
// terminals live in (empty: "pipelines"); TerminalWithoutTmux runs
// shells as direct children even where tmux is installed. Tests set
// them so they never share the real server's tmux sessions.
TerminalTmuxSocket string
TerminalWithoutTmux bool
}
// Handler serves the persistent connection with default options. Canceling ctx
@@ -150,18 +144,15 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
hub.startShellRunner(ctx, store)
}
// Machine terminals are recorded beside the operator sessions, in the
// same store; a previous process's tmux-backed shells are reattached.
// same store; the ones a previous process left running are listed as
// ended.
var terminalRecords terminalStore
if len(stores) > 0 {
terminalRecords = stores[0]
} else if store, ok := registry.(terminalStore); ok {
terminalRecords = store
}
var terminalTmuxServer *terminalTmux
if !options.TerminalWithoutTmux {
terminalTmuxServer = findTerminalTmux(options.TerminalTmuxSocket)
}
terminals := newTerminalManager(ctx, terminalRecords, terminalTmuxServer)
terminals := newTerminalManager(ctx, terminalRecords)
terminals.recover(ctx)
acks := newAttentionAckHub()
var ackStore attentionAckStore