web: live run updates over a single app-wide watch (R10)
This commit is contained in:
+354
-30
@@ -7,6 +7,7 @@ import { PROTOCOL_VERSION, Protocol, ChatKeys } from "./protocol.js";
|
||||
import { validateServerUrl, resolveArtifactUrl } from "./serverUrl.js";
|
||||
import {
|
||||
ForegroundAction,
|
||||
RunStream,
|
||||
bannerState,
|
||||
foregroundAction,
|
||||
liveBadgeState,
|
||||
@@ -14,6 +15,12 @@ import {
|
||||
serverBuildLabel,
|
||||
stateBadge,
|
||||
} from "./connection.js";
|
||||
import {
|
||||
foldRunEvent,
|
||||
foldRunEventDetail,
|
||||
isTerminalRunStatus,
|
||||
runDetailNeedsFullSync,
|
||||
} from "./runEvents.js";
|
||||
import {
|
||||
SERVER_URL_HELPER,
|
||||
SERVER_URL_PLACEHOLDER,
|
||||
@@ -400,6 +407,12 @@ const state = {
|
||||
workspace: initialWorkspace,
|
||||
workspaceDraft: initialWorkspace,
|
||||
connection: client.state,
|
||||
// The live run-stream state (R10), independent of any screen: Offline until a
|
||||
// watch is established, Connecting while subscribing/re-subscribing after a
|
||||
// drop, Live once an ack, the first pushed frame or a successful watch read
|
||||
// proves pushes are arriving. The runs list and the run detail render it as the
|
||||
// LiveBadge.kt words.
|
||||
runStream: RunStream.Offline,
|
||||
// The last `pong` reply is remembered so the banner can name a stale server
|
||||
// build and warn about a protocol mismatch (T5) without re-pinging.
|
||||
serverBuild: null,
|
||||
@@ -540,7 +553,28 @@ const state = {
|
||||
chatSubscriptions: new Set(),
|
||||
};
|
||||
|
||||
let runPoll = null;
|
||||
// The app-wide run-event watch (R10), owned by the connection rather than any
|
||||
// screen. `workspace` is the blank string while a watch is established (the
|
||||
// server-wide "any" scope) and null when nothing is watched; `subscriptionId` is
|
||||
// the send-only subscribe frame's id so an ack or refusal can be attributed.
|
||||
// `generation` is bumped on every (re)subscribe and every drop, so an in-flight
|
||||
// read from a superseded watch can neither write its result nor clear the badge.
|
||||
const runWatch = {
|
||||
workspace: null,
|
||||
subscriptionId: null,
|
||||
generation: 0,
|
||||
runsResyncJob: null,
|
||||
runsResyncQueued: false,
|
||||
detailSyncJob: null,
|
||||
detailSyncRequestedRunId: null,
|
||||
detailDeltaCount: 0,
|
||||
};
|
||||
|
||||
// The coalesced resync retry bound, mirroring AppViewModel: at most three
|
||||
// retries (four attempts) at 500 ms x attempt, then the badge drops to
|
||||
// "reconnecting" until a later push or read confirms the stream again.
|
||||
const RUN_SYNC_RETRY_LIMIT = 3;
|
||||
const RUN_SYNC_RETRY_BASE_MS = 500;
|
||||
|
||||
// One shared live clock for the runs list: it ticks only while a shown run is
|
||||
// executing, and each running row reads its elapsed from it rather than starting
|
||||
@@ -586,6 +620,11 @@ function isExecuting(run) {
|
||||
return Boolean(run && run.executing);
|
||||
}
|
||||
|
||||
/** Await one delay; used only by the bounded resync retry backoff. */
|
||||
function delay(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rendering
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -1221,7 +1260,6 @@ function runConnectionAction(label) {
|
||||
const kind = connectionActionKind(label);
|
||||
if (kind === "disconnect") {
|
||||
repo.disconnect();
|
||||
stopRunPoll();
|
||||
return;
|
||||
}
|
||||
if (kind === "reconnect") {
|
||||
@@ -2227,7 +2265,7 @@ function renderRunToolbar() {
|
||||
state.openMenu === "sort" ? renderSortMenu() : null,
|
||||
),
|
||||
h("span", { class: "spacer" }),
|
||||
renderLiveBadge(liveBadgeState({ executing: state.runs.some((run) => run.executing) }, state.connection)),
|
||||
renderLiveBadge(liveBadgeState(state.runStream)),
|
||||
);
|
||||
return toolbar;
|
||||
}
|
||||
@@ -2463,7 +2501,7 @@ function renderRunSummaryTab(d) {
|
||||
chips.append(h("span", { class: `chip chip-${status}`, text: d.status || status }));
|
||||
if (String(d.outcome ?? "").trim()) chips.append(h("span", { class: "muted", text: `outcome ${d.outcome}` }));
|
||||
if (d.executing) chips.append(h("span", { class: "muted", text: "executing" }));
|
||||
chips.append(renderLiveBadge(liveBadgeState(d, state.connection)));
|
||||
chips.append(renderLiveBadge(liveBadgeState(state.runStream)));
|
||||
wrap.append(chips);
|
||||
wrap.append(h("div", { class: "run-identity muted", text: runIdentityLine(d) }));
|
||||
|
||||
@@ -6597,7 +6635,6 @@ function openConfiguredEndpoint({ surfaceError = true } = {}) {
|
||||
function onConnectClicked() {
|
||||
if (state.connection.kind === "connected") {
|
||||
repo.disconnect();
|
||||
stopRunPoll();
|
||||
return;
|
||||
}
|
||||
openConfiguredEndpoint();
|
||||
@@ -6630,6 +6667,13 @@ function onForegroundActive() {
|
||||
}
|
||||
|
||||
async function onConnected() {
|
||||
// Establish the run-event watch immediately once the socket is up (R10): the
|
||||
// unsubscribe/subscribe pair is sent before anything else is read, and its
|
||||
// authoritative `list_runs` resync is the one the view then reuses, so a
|
||||
// connection never stacks two initial history reads. The watch is
|
||||
// send-only/best-effort; a failure leaves the badge "reconnecting" and the
|
||||
// ordinary view load proceeds.
|
||||
const watched = startRunWatch();
|
||||
let reply;
|
||||
try {
|
||||
reply = await repo.ping();
|
||||
@@ -6648,10 +6692,11 @@ async function onConnected() {
|
||||
// the Settings card is the one `loadCurrentView` already performs (Settings is
|
||||
// local, so that path is a single render and nothing is fetched twice).
|
||||
renderBanner();
|
||||
await loadCurrentView();
|
||||
const runsFresh = await watched;
|
||||
await loadCurrentView(false, { runsFresh });
|
||||
}
|
||||
|
||||
async function loadCurrentView(force = false) {
|
||||
async function loadCurrentView(force = false, { runsFresh = false } = {}) {
|
||||
// Settings is local: it has no server-backed list to (re)load, so switching to
|
||||
// or refreshing it must never raise the "connect first" error.
|
||||
if (state.view === "settings") {
|
||||
@@ -6680,12 +6725,15 @@ async function loadCurrentView(force = false) {
|
||||
state.pipelinesLoaded = true;
|
||||
// The last-run line joins list_runs onto the pipelines, so the history is
|
||||
// fetched alongside them; runsLoaded distinguishes a genuinely empty
|
||||
// history from one that has not arrived yet.
|
||||
const { runs, usage } = await repo.listRuns();
|
||||
state.runs = runs;
|
||||
if (usage) state.usage = usage;
|
||||
state.runsLoaded = true;
|
||||
} else {
|
||||
// history from one that has not arrived yet. On connect that history has
|
||||
// just been read by the run watch's own resync, so it is not read twice.
|
||||
if (!runsFresh) {
|
||||
const { runs, usage } = await repo.listRuns();
|
||||
state.runs = runs;
|
||||
if (usage) state.usage = usage;
|
||||
state.runsLoaded = true;
|
||||
}
|
||||
} else if (!runsFresh) {
|
||||
const { runs, usage } = await repo.listRuns();
|
||||
state.runs = runs;
|
||||
state.usage = usage;
|
||||
@@ -6705,6 +6753,269 @@ async function loadCurrentView(force = false) {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The live run stream (R10)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Render whatever a run-state change touched, without re-reading anything. */
|
||||
function renderRunState() {
|
||||
if (state.view === "runs" || state.view === "pipelines") renderContent();
|
||||
refreshAttentionBell();
|
||||
}
|
||||
|
||||
/** Record the explicit stream state and repaint the badge when it changed. */
|
||||
function setRunStream(next) {
|
||||
if (state.runStream === next) return;
|
||||
state.runStream = next;
|
||||
renderRunState();
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the watch bookkeeping and invalidate any in-flight read. Called on every
|
||||
* socket drop and before a new subscribe, so a superseded read can neither write
|
||||
* its result over newer state nor confirm a stream that is gone. The stream's own
|
||||
* state is set by the caller (streamDown / startRunWatch).
|
||||
*/
|
||||
function clearRunWatch() {
|
||||
runWatch.workspace = null;
|
||||
runWatch.subscriptionId = null;
|
||||
runWatch.generation += 1;
|
||||
runWatch.runsResyncQueued = false;
|
||||
}
|
||||
|
||||
/** Nothing is watched: reflect the connection's own state in the live badge. */
|
||||
function streamDown() {
|
||||
clearRunWatch();
|
||||
state.runStream =
|
||||
state.connection.kind === "connecting" || state.connection.kind === "disconnected"
|
||||
? RunStream.Connecting
|
||||
: RunStream.Offline;
|
||||
}
|
||||
|
||||
/** A subscription ack, a push, or a successful watch read proves the stream is live. */
|
||||
function confirmStreamLive() {
|
||||
if (runWatch.workspace === null) return;
|
||||
setRunStream(RunStream.Live);
|
||||
}
|
||||
|
||||
/** A push-driven read failed: stop claiming live until something confirms it again. */
|
||||
function markStreamDegraded() {
|
||||
if (runWatch.workspace !== null) setRunStream(RunStream.Connecting);
|
||||
}
|
||||
|
||||
/**
|
||||
* Establish the one watch over every workspace the client displays (the blank
|
||||
* scope, which the server treats as "any"): the runs list is server-wide, so the
|
||||
* client asks for the runs it shows rather than the one workspace the operator
|
||||
* configured. It best-effort drops the previous watch first (the server caps
|
||||
* subscriptions per connection), marks the watch before sending so a push that
|
||||
* races the send is still attributed to it, then takes one authoritative resync
|
||||
* of the watched state. Resolves true when that resync succeeded, so the caller
|
||||
* knows the runs list is already fresh.
|
||||
*/
|
||||
async function startRunWatch() {
|
||||
const generation = ++runWatch.generation;
|
||||
runWatch.workspace = null;
|
||||
runWatch.subscriptionId = null;
|
||||
state.runStream = RunStream.Connecting;
|
||||
try {
|
||||
repo.unsubscribeRuns();
|
||||
} catch {
|
||||
// The drop failed; the new subscribe is still attempted, but the badge stays
|
||||
// "reconnecting" until an ack, push or successful read confirms the new watch.
|
||||
state.runStream = RunStream.Connecting;
|
||||
}
|
||||
if (generation !== runWatch.generation || state.connection.kind !== "connected") return false;
|
||||
runWatch.workspace = "";
|
||||
try {
|
||||
runWatch.subscriptionId = repo.subscribeRuns("", []);
|
||||
} catch {
|
||||
clearRunWatch();
|
||||
setRunStream(RunStream.Offline);
|
||||
return false;
|
||||
}
|
||||
const resynced = await scheduleRunsResync();
|
||||
if (generation !== runWatch.generation) return false;
|
||||
if (state.runDetail?.id != null) scheduleRunDetailSync(state.runDetail.id);
|
||||
return resynced;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-reads the runs list once for a burst of frames the list could not fold (a
|
||||
* created run whose summary is not in the compact delta, or a terminal
|
||||
* transition that changes the usage aggregate) and once after every
|
||||
* (re)subscribe. Coalesced — at most one in flight with one queued follow-up —
|
||||
* and it loops while a frame arrived, so no push is ever dropped. A failed read
|
||||
* stops claiming live and is retried with a bounded backoff before giving up.
|
||||
* Resolves true when the last read succeeded.
|
||||
*/
|
||||
function scheduleRunsResync() {
|
||||
if (runWatch.runsResyncJob) {
|
||||
runWatch.runsResyncQueued = true;
|
||||
return runWatch.runsResyncJob;
|
||||
}
|
||||
const job = (async () => {
|
||||
let attempt = 0;
|
||||
let ok = true;
|
||||
for (;;) {
|
||||
runWatch.runsResyncQueued = false;
|
||||
const generation = runWatch.generation;
|
||||
try {
|
||||
const { runs, usage } = await repo.listRuns();
|
||||
if (generation !== runWatch.generation) {
|
||||
// The watch was re-established (or dropped) while this read was in
|
||||
// flight; its result is superseded, so re-read under the new watch
|
||||
// instead of writing stale state or clearing the badge.
|
||||
runWatch.runsResyncQueued = true;
|
||||
continue;
|
||||
}
|
||||
state.runs = runs;
|
||||
if (usage) state.usage = usage;
|
||||
state.runsLoaded = true;
|
||||
attempt = 0;
|
||||
ok = true;
|
||||
confirmStreamLive();
|
||||
renderRunState();
|
||||
} catch {
|
||||
if (generation !== runWatch.generation) {
|
||||
runWatch.runsResyncQueued = true;
|
||||
continue;
|
||||
}
|
||||
markStreamDegraded();
|
||||
attempt += 1;
|
||||
if (attempt > RUN_SYNC_RETRY_LIMIT) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
await delay(RUN_SYNC_RETRY_BASE_MS * attempt);
|
||||
if (generation !== runWatch.generation) {
|
||||
runWatch.runsResyncQueued = true;
|
||||
continue;
|
||||
}
|
||||
runWatch.runsResyncQueued = true;
|
||||
ok = false;
|
||||
}
|
||||
if (!runWatch.runsResyncQueued) break;
|
||||
}
|
||||
return ok;
|
||||
})();
|
||||
runWatch.runsResyncJob = job;
|
||||
job.finally(() => {
|
||||
if (runWatch.runsResyncJob === job) runWatch.runsResyncJob = null;
|
||||
});
|
||||
return job;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the full run detail once for a burst of settled-step/terminal pushes,
|
||||
* so the artifacts and output the compact delta omits appear without a timer. A
|
||||
* response is dropped and retried when a newer frame landed while the fetch was
|
||||
* in flight, so the fetch can never regress the fold. The queued request is keyed
|
||||
* by run id, so opening another run mid-fetch fetches that run instead of
|
||||
* discarding the request; a run the operator has left is never written over.
|
||||
*/
|
||||
function scheduleRunDetailSync(runId) {
|
||||
const targetId = Number(runId);
|
||||
if (!(targetId > 0)) return null;
|
||||
if (runWatch.detailSyncJob) {
|
||||
runWatch.detailSyncRequestedRunId = targetId;
|
||||
return runWatch.detailSyncJob;
|
||||
}
|
||||
const job = (async () => {
|
||||
let target = targetId;
|
||||
let attempt = 0;
|
||||
while (target != null) {
|
||||
runWatch.detailSyncRequestedRunId = null;
|
||||
const generation = runWatch.generation;
|
||||
const deltaCount = runWatch.detailDeltaCount;
|
||||
try {
|
||||
const run = await repo.getRun(target);
|
||||
attempt = 0;
|
||||
if (generation !== runWatch.generation) {
|
||||
// A drop/re-subscribe superseded this read; re-read for the new watch
|
||||
// rather than writing a result from the old one.
|
||||
runWatch.detailSyncRequestedRunId = target;
|
||||
continue;
|
||||
}
|
||||
if (state.runDetail?.id === target) {
|
||||
if (runWatch.detailDeltaCount === deltaCount) {
|
||||
state.runDetail = run;
|
||||
renderRunState();
|
||||
} else {
|
||||
// A newer frame overtook the fetch; re-read rather than write the
|
||||
// older run over it.
|
||||
runWatch.detailSyncRequestedRunId = target;
|
||||
}
|
||||
}
|
||||
// else: the operator left this run; its detail is no longer shown.
|
||||
} catch {
|
||||
if (generation !== runWatch.generation) {
|
||||
runWatch.detailSyncRequestedRunId = target;
|
||||
continue;
|
||||
}
|
||||
markStreamDegraded();
|
||||
attempt += 1;
|
||||
if (attempt > RUN_SYNC_RETRY_LIMIT) return;
|
||||
await delay(RUN_SYNC_RETRY_BASE_MS * attempt);
|
||||
if (generation !== runWatch.generation) {
|
||||
runWatch.detailSyncRequestedRunId = target;
|
||||
continue;
|
||||
}
|
||||
// A run opened meanwhile wins over retrying this one.
|
||||
if (runWatch.detailSyncRequestedRunId == null) runWatch.detailSyncRequestedRunId = target;
|
||||
}
|
||||
target = runWatch.detailSyncRequestedRunId;
|
||||
}
|
||||
})();
|
||||
runWatch.detailSyncJob = job;
|
||||
job.finally(() => {
|
||||
if (runWatch.detailSyncJob === job) runWatch.detailSyncJob = null;
|
||||
});
|
||||
return job;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold one pushed `run_event` frame into whatever local state it names: the runs
|
||||
* list (and so the pipeline rows' last-run line and the attention bell) in
|
||||
* place, and the open run detail. A frame for a run the list does not hold
|
||||
* re-syncs the list once; a frame that changed the detail and carries a settled
|
||||
* step, a waiting step/run or a terminal run also fetches the full detail once.
|
||||
*/
|
||||
function applyRunEvent(frame) {
|
||||
const runId = Number(frame?.run_id);
|
||||
if (!(runId > 0)) return;
|
||||
let listChanged = false;
|
||||
if (state.runsLoaded) {
|
||||
const fold = foldRunEvent(state.runs, frame);
|
||||
if (fold.needsResync) {
|
||||
scheduleRunsResync();
|
||||
} else if (fold.runs !== state.runs) {
|
||||
state.runs = fold.runs;
|
||||
listChanged = true;
|
||||
renderRunState();
|
||||
// A terminal transition also settles the usage aggregate (the active
|
||||
// count), which the delta does not carry; one server read refreshes it. A
|
||||
// duplicate terminal frame changes nothing above, so it triggers no read.
|
||||
if (frame.step == null && isTerminalRunStatus(frame.status)) scheduleRunsResync();
|
||||
}
|
||||
}
|
||||
const detail = state.runDetail;
|
||||
if (detail && Number(detail.id) === runId) {
|
||||
runWatch.detailDeltaCount += 1;
|
||||
const folded = foldRunEventDetail(detail, frame);
|
||||
const detailChanged = folded !== detail;
|
||||
if (detailChanged) {
|
||||
state.runDetail = folded;
|
||||
renderRunState();
|
||||
}
|
||||
if (runDetailNeedsFullSync(frame) && (detailChanged || listChanged)) {
|
||||
scheduleRunDetailSync(runId);
|
||||
}
|
||||
}
|
||||
// A list-only change with the detail open is still a fold; if the run list was
|
||||
// not in hand, the resync above covers the new row.
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the template catalogue, then validates each template's source once and
|
||||
* caches the parsed graph (null when it did not parse) so a card's stage count
|
||||
@@ -6786,24 +7097,11 @@ async function selectRun(id, { silent = false } = {}) {
|
||||
try {
|
||||
state.runDetail = await repo.getRun(id);
|
||||
renderContent();
|
||||
updateRunPoll();
|
||||
// The run detail is no longer polled: the watch's pushed `run_event` frames
|
||||
// advance it live (R10), and a settled/terminal frame triggers one full
|
||||
// `get_run` for the heavy fields the compact delta omits.
|
||||
} catch (err) {
|
||||
if (!silent) setError(err);
|
||||
else stopRunPoll();
|
||||
}
|
||||
}
|
||||
|
||||
function updateRunPoll() {
|
||||
stopRunPoll();
|
||||
if (state.view === "runs" && state.selectedRunId && isExecuting(state.runDetail)) {
|
||||
runPoll = setInterval(() => selectRun(state.selectedRunId, { silent: true }), 1000);
|
||||
}
|
||||
}
|
||||
|
||||
function stopRunPoll() {
|
||||
if (runPoll) {
|
||||
clearInterval(runPoll);
|
||||
runPoll = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6922,7 +7220,6 @@ client.addEventListener("state", (event) => {
|
||||
onConnected();
|
||||
}
|
||||
if (next.kind !== "connected" && previousKind === "connected") {
|
||||
stopRunPoll();
|
||||
stopListClock();
|
||||
stopDetailClock();
|
||||
stopChatClock();
|
||||
@@ -6932,6 +7229,11 @@ client.addEventListener("state", (event) => {
|
||||
clearChatSubscriptions();
|
||||
settleChatThreads();
|
||||
}
|
||||
if (next.kind !== "connected") {
|
||||
// Nothing is watched while the socket is down; an in-flight read is
|
||||
// invalidated and the badge falls back to Connecting (or Offline when idle).
|
||||
streamDown();
|
||||
}
|
||||
// A reconnect (not a first connect): re-attach the open chat's stream and
|
||||
// refresh its history wholesale, so a gap that swallowed the end of a turn is
|
||||
// repaired rather than shown stale.
|
||||
@@ -6949,6 +7251,28 @@ client.addEventListener("state", (event) => {
|
||||
client.addEventListener("frame", (event) => {
|
||||
const message = event.detail;
|
||||
if (!message) return;
|
||||
// The live run stream (R10). A pushed `run_event` only reaches a subscriber, so
|
||||
// it confirms the watch even when the server never acks the subscribe. A reply
|
||||
// carrying our subscribe id is the optional ack (live) or a refusal (offline).
|
||||
if (message.type === Protocol.RUN_EVENT) {
|
||||
const frame = message.run_event;
|
||||
if (frame) {
|
||||
confirmStreamLive();
|
||||
applyRunEvent(frame);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (message.id != null && message.id === runWatch.subscriptionId) {
|
||||
if (message.type === Protocol.SUBSCRIBE) {
|
||||
confirmStreamLive();
|
||||
return;
|
||||
}
|
||||
if (message.type === Protocol.ERROR) {
|
||||
clearRunWatch();
|
||||
setRunStream(RunStream.Offline);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (message.type === Protocol.CHAT_PERMISSION) {
|
||||
const permission = message[ChatKeys.PERMISSION];
|
||||
if (permission) onChatPermission(permission);
|
||||
|
||||
Reference in New Issue
Block a user