Merge branch 'main' into large-chat-history
This commit is contained in:
@@ -47,7 +47,7 @@ func main() {
|
||||
// Each new run gets a bounded one-line title from the configured default
|
||||
// harness; the summariser falls back to the prompt's first line, so title
|
||||
// generation never blocks or fails run creation.
|
||||
options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason}
|
||||
options := server.Options{ArtifactsRoot: artifactsRoot, RunTitles: summarise.GenerateTitle, StepVerdicts: summarise.GenerateVerdictReason, VerificationFixtures: server.VerificationFixturesFromEnvironment(os.Getenv)}
|
||||
recorded := ""
|
||||
if value, err := store.OpenCodeEndpoint(ctx); err != nil {
|
||||
log.Printf("OpenCode endpoint setting unreadable: %v", err)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Background shell visibility
|
||||
|
||||
The chat server now opens the broker stream for read-only subscribers and reconciles verified running shell operations from OpenCode's authoritative `/api/shell` readback. The packaged OpenCode skill references and chat documentation should describe that shell visibility survives reconnects only when OpenCode reports explicit `metadata.sessionID` ownership; shared working directories are not ownership evidence.
|
||||
|
||||
## Current source behavior
|
||||
|
||||
Observers share one initial readback per room/stream attachment. The adapter and hub
|
||||
fold bounded lifecycle journals over each snapshot, so a stale read cannot resurrect
|
||||
an exited shell or erase a concurrent create. Shell replay and live delivery enqueue
|
||||
in fold order. Cancellation rejects old-room callbacks and releases local identity
|
||||
state before reconnect. Journals retain at most 256 identities and 1 MiB of id/command
|
||||
text; capacity/read failures surface stable limited-visibility text, never broker
|
||||
response bodies. No shell polling was added. OpenCode's terminal `timeout` and
|
||||
`killed` statuses are excluded from retained running operations as well as `exited`.
|
||||
|
||||
## Verification evidence — 2026-10-18
|
||||
|
||||
`opencode api get /openapi.json` authenticated through supported service discovery.
|
||||
The live schema advertises shell list/detail/output and generic `metadata`; it does
|
||||
not type `metadata.sessionID`. A throwaway session
|
||||
`ses_eb12ab7b3ffe0ZAle7PHpftHMG` and its own `sleep 1` shell
|
||||
`sh_14ed57134001e6fuYqvn62pKit` verified actual create/detail readback preserving
|
||||
explicit `metadata.sessionID`, status `running`, subsequent `exited`/exit 0, and
|
||||
absence from the running list. This verifies explicit API metadata, not automatic
|
||||
agent-tool ownership injection. The CLI event attempt yielded no captured events;
|
||||
live-provider event ordering is **not tested**. Forced ordering evidence comes from
|
||||
channel/barrier regressions, including a real WebSocket handler on a loopback scratch
|
||||
server with fresh SQLite, not from the deployed service.
|
||||
|
||||
## Documentation follow-up / deployment
|
||||
|
||||
`docs/chat-verification.md` and the chat subscribe/replay section of `docs/cli.md`
|
||||
are stale: docs-fold should describe authoritative ownership/readback, coalesced
|
||||
observation, bounded reconciliation and limited-visibility errors, distinguishing
|
||||
fixture ordering evidence from the live readback above. Shared docs and packaged
|
||||
skill references were not edited here. No deployment, installation, service restart
|
||||
or physical-phone/UI verification was performed; this changes no rendered layout.
|
||||
@@ -0,0 +1,47 @@
|
||||
# Delivery stabilization
|
||||
|
||||
Date: 2026-10-18.
|
||||
|
||||
The server now has draft-only `delivery=true` support for OpenCode model stages.
|
||||
At run time it derives the canonical `main` worktree and linked-worktree Git
|
||||
metadata from Git rather than accepting paths from a stage prompt. It verifies
|
||||
the committed item tree in a disposable candidate worktree before fast-forwarding
|
||||
canonical `main`; declared checks are bounded configuration, not proof that any
|
||||
deployed definition has used them. It is denied for Pi (until equivalent
|
||||
capability enforcement exists), foreign repositories, missing main worktrees,
|
||||
and malformed capabilities. This item does not publish or deploy a definition.
|
||||
|
||||
The source also supports an optional controlled-fixture verification declaration
|
||||
(`backend`, `revision`, `capability`, and `fixture`). Ordinary server setup wires
|
||||
an explicit, bounded `reconnect-foreground` HTTP fixture registry when
|
||||
`PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL` and its expected `BACKEND`,
|
||||
`REVISION`, `CAPABILITY`, and `STATE` values are configured. There is no default
|
||||
remote URL. The server, not the model, probes that registered endpoint and
|
||||
collects its returned backend identity, build revision, capability, fixture ID,
|
||||
and state before comparing provenance. Unknown fixtures, incomplete setup,
|
||||
wrong backend/revision/capability/state, non-200 responses, cancellation, and
|
||||
oversized or malformed responses fail closed. A model result's labels are not
|
||||
evidence. This is controlled local/backend fixture infrastructure, not evidence
|
||||
of a rendered layout or a live Android device.
|
||||
Definitions must be scratch-validated against the deployed server only after
|
||||
the server support is deployed; no future delivery or verification definition
|
||||
is activated by this source change.
|
||||
|
||||
`internal/author/skill/references/format.md` and
|
||||
`internal/author/skill/references/queue.md` are stale: a later docs fold should
|
||||
describe `delivery=true`, OpenCode-only availability, candidate verification,
|
||||
and the required activation order (deploy server support before scratch-validating
|
||||
and then publishing a definition).
|
||||
OpenCode V2 resolves external resources at a containing directory, so delivery
|
||||
uses the verified repository metadata directory rather than an impossible
|
||||
single-lock-file scope. It does not grant a parent, home, foreign repository,
|
||||
or global policy. Unrestricted shell permissions are not an OS sandbox.
|
||||
|
||||
Activation remains draft-only: deploy compatible server support, configure the
|
||||
controlled fixture endpoint and matching provenance, scratch-validate the
|
||||
fixture, then publish a definition. No definition is activated by this source
|
||||
change.
|
||||
|
||||
Recovery checklist: retain `connection-reconnect` and unfinished feature
|
||||
branches; merge only complete committed trees that pass mapped checks; do not
|
||||
infer delivery from a dirty worktree or an old deployed-backend capture.
|
||||
+198
-11
@@ -30,7 +30,9 @@ var ErrPermissionUnsupported = errors.New("OpenCode permission approval is not i
|
||||
// call guards. History alone streams/project tool details before decoding; it
|
||||
// does not introduce a total-history cap or a second credential path.
|
||||
type Chat struct {
|
||||
conn *Connection
|
||||
conn *Connection
|
||||
shellMu sync.Mutex
|
||||
shells map[string]*ChatStreamState
|
||||
// mu serializes the adapter's HTTP calls. Connection records its last
|
||||
// operation and status on the shared connection, so two calls must not
|
||||
// overlap; the permission poller runs alongside the read loop's
|
||||
@@ -332,6 +334,114 @@ func decodeChatPermissions(sessionID string, data []byte) ([]protocol.ChatPermis
|
||||
return frames, nil
|
||||
}
|
||||
|
||||
// Shells reads the endpoint's authoritative shell list and returns only running
|
||||
// operations whose metadata explicitly names sessionID. The shell API is global;
|
||||
// filtering locally avoids guessing an unsupported query filter and never assigns
|
||||
// ownership from a shared working directory. A successful read also seeds the
|
||||
// stream state used to attribute later shell.exited events, which carry no session.
|
||||
func (c *Chat) Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error) {
|
||||
if !ValidSessionID(sessionID) {
|
||||
return nil, fmt.Errorf("invalid OpenCode session id")
|
||||
}
|
||||
state, err := c.streamStateContext(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// One read per state; waiting and I/O never hold the event-state mutex.
|
||||
state.mu.Lock()
|
||||
if state.readGate == nil {
|
||||
state.readGate = make(chan struct{}, 1)
|
||||
}
|
||||
gate := state.readGate
|
||||
state.mu.Unlock()
|
||||
select {
|
||||
case gate <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
defer func() { <-gate }()
|
||||
state.mu.Lock()
|
||||
state.readback = &ShellSnapshot{}
|
||||
state.mu.Unlock()
|
||||
defer func() { state.mu.Lock(); state.readback = nil; state.mu.Unlock() }()
|
||||
data, err := c.call(ctx, "get", "/api/shell", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var response struct {
|
||||
Data []chatShellInfo `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &response); err != nil || response.Data == nil {
|
||||
if err := json.Unmarshal(data, &response.Data); err != nil {
|
||||
return nil, fmt.Errorf("invalid OpenCode shell response")
|
||||
}
|
||||
}
|
||||
out := make([]protocol.ChatShell, 0, len(response.Data))
|
||||
for _, info := range response.Data {
|
||||
if info.ID == "" || info.shellSessionID() != sessionID || (info.Status != "" && info.Status != protocol.ChatShellRunning) {
|
||||
continue
|
||||
}
|
||||
status := info.Status
|
||||
if status == "" {
|
||||
status = protocol.ChatShellRunning
|
||||
}
|
||||
out = append(out, protocol.ChatShell{ID: info.ID, Status: status, Command: info.Command})
|
||||
}
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
out, err = state.readback.Merge(out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, 0, len(out))
|
||||
for _, shell := range out {
|
||||
ids = append(ids, shell.ID)
|
||||
}
|
||||
state.replaceShellsLocked(ids)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// streamState returns the session's retained, verified shell identities. It is
|
||||
// deliberately kept across stream replacement so an exit after reconnect remains
|
||||
// attributable; Shells reconciles its contents from authoritative readback on
|
||||
// each attach rather than replacing the pointer a live scanner is using.
|
||||
func (c *Chat) streamState(sessionID string) *ChatStreamState {
|
||||
state, _ := c.streamStateContext(context.Background(), sessionID)
|
||||
return state
|
||||
}
|
||||
|
||||
func (c *Chat) streamStateContext(ctx context.Context, sessionID string) (*ChatStreamState, error) {
|
||||
c.shellMu.Lock()
|
||||
defer c.shellMu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.shells == nil {
|
||||
c.shells = map[string]*ChatStreamState{}
|
||||
}
|
||||
state := c.shells[sessionID]
|
||||
if state == nil {
|
||||
state = &ChatStreamState{}
|
||||
c.shells[sessionID] = state
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// ReleaseShells discards verified shell identities after the last room observer
|
||||
// disconnects. A future attach calls Shells for a fresh authoritative snapshot,
|
||||
// so this bounds state without attributing a foreign shell during the gap.
|
||||
func (c *Chat) ReleaseShells(sessionID string) {
|
||||
if sessionID == "" {
|
||||
return
|
||||
}
|
||||
c.shellMu.Lock()
|
||||
delete(c.shells, sessionID)
|
||||
c.shellMu.Unlock()
|
||||
}
|
||||
|
||||
// Stream subscribes to the OpenCode event firehose and invokes emit for every
|
||||
// frame belonging to sessionID. It reads one bounded SSE frame at a time and
|
||||
// never polls the per-session message log; the global firehose is the only
|
||||
@@ -344,6 +454,12 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
if !sessionID.MatchString(id) {
|
||||
return fmt.Errorf("invalid OpenCode session id")
|
||||
}
|
||||
// Pin state before opening/announcing the stream so a canceled scanner
|
||||
// cannot acquire a replacement room's identity map.
|
||||
state, err := c.streamStateContext(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := c.conn.EventStream(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -352,7 +468,7 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
return scanChatEvents(ctx, body, id, emit)
|
||||
return c.scanChatEventsWithState(ctx, body, id, state, emit)
|
||||
}
|
||||
|
||||
// scanChatEvents turns an SSE body into protocol frames, one bounded frame at a
|
||||
@@ -360,10 +476,13 @@ func (c *Chat) Stream(ctx context.Context, id string, ready func(), emit func(pr
|
||||
// malformed JSON fails the stream instead of producing a malformed frame. The
|
||||
// last assistant message id is retained so a later tool/step event that omits it
|
||||
// still attributes every frame to a stable message.
|
||||
func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error {
|
||||
func (c *Chat) scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit func(protocol.ChatEventFrame) error) error {
|
||||
return c.scanChatEventsWithState(ctx, body, sessionID, c.streamState(sessionID), emit)
|
||||
}
|
||||
|
||||
func (c *Chat) scanChatEventsWithState(ctx context.Context, body io.Reader, sessionID string, state *ChatStreamState, emit func(protocol.ChatEventFrame) error) error {
|
||||
scanner := bufio.NewScanner(body)
|
||||
scanner.Buffer(make([]byte, 0, 64<<10), MaxResponse)
|
||||
state := &ChatStreamState{}
|
||||
for scanner.Scan() {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
@@ -384,7 +503,7 @@ func scanChatEvents(ctx context.Context, body io.Reader, sessionID string, emit
|
||||
continue
|
||||
}
|
||||
if frame.MessageID != "" {
|
||||
state.LastMessageID = frame.MessageID
|
||||
state.setLastMessageID(frame.MessageID)
|
||||
}
|
||||
if err := emit(frame); err != nil {
|
||||
return err
|
||||
@@ -487,8 +606,29 @@ func (i *chatShellInfo) shellSessionID() string {
|
||||
// what attributes the exit to its session: another session's shell is an
|
||||
// unknown id and is ignored.
|
||||
type ChatStreamState struct {
|
||||
mu sync.Mutex
|
||||
LastMessageID string
|
||||
Shells map[string]struct{}
|
||||
readGate chan struct{}
|
||||
readback *ShellSnapshot
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) lastMessageID() string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.LastMessageID
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) setLastMessageID(id string) {
|
||||
if s == nil || id == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.LastMessageID = id
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// knowsShell reports whether id is a shell this stream has already surfaced for
|
||||
@@ -497,6 +637,8 @@ func (s *ChatStreamState) knowsShell(id string) bool {
|
||||
if s == nil || id == "" {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, ok := s.Shells[id]
|
||||
return ok
|
||||
}
|
||||
@@ -506,6 +648,8 @@ func (s *ChatStreamState) markShell(id string) {
|
||||
if s == nil || id == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.Shells == nil {
|
||||
s.Shells = map[string]struct{}{}
|
||||
}
|
||||
@@ -514,12 +658,37 @@ func (s *ChatStreamState) markShell(id string) {
|
||||
|
||||
// forgetShell drops id once its exit has been surfaced.
|
||||
func (s *ChatStreamState) forgetShell(id string) {
|
||||
if s == nil || s.Shells == nil {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.Shells == nil {
|
||||
return
|
||||
}
|
||||
delete(s.Shells, id)
|
||||
}
|
||||
|
||||
// replaceShells applies one authoritative running-shell snapshot without
|
||||
// replacing the state object an already attached event scanner owns.
|
||||
func (s *ChatStreamState) replaceShells(ids []string) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.replaceShellsLocked(ids)
|
||||
}
|
||||
|
||||
func (s *ChatStreamState) replaceShellsLocked(ids []string) {
|
||||
s.Shells = make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
if id != "" {
|
||||
s.Shells[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeChatEvent maps one OpenCode firehose `data:` payload to a
|
||||
// protocol.ChatEventFrame, folding the small per-session context ([stream]) it
|
||||
// needs across frames. ok is false for an event type this adapter does not
|
||||
@@ -566,7 +735,7 @@ func NormalizeChatEvent(payload []byte, sessionID string, stream *ChatStreamStat
|
||||
messageID = data.MessageIDAlt
|
||||
}
|
||||
if messageID == "" && stream != nil {
|
||||
messageID = stream.LastMessageID
|
||||
messageID = stream.lastMessageID()
|
||||
}
|
||||
// Every pushed frame must carry a stable message id so the client can
|
||||
// attribute it; without one this is not a chat frame.
|
||||
@@ -628,20 +797,38 @@ func normalizeShellEvent(eventType string, data chatEvent, sessionID string, str
|
||||
if status == "" {
|
||||
status = protocol.ChatShellRunning
|
||||
}
|
||||
stream.markShell(data.Info.ID)
|
||||
shell := &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command}
|
||||
if stream != nil {
|
||||
stream.mu.Lock()
|
||||
stream.readback.Record(shell)
|
||||
if stream.Shells == nil {
|
||||
stream.Shells = map[string]struct{}{}
|
||||
}
|
||||
if status == protocol.ChatShellRunning && len(stream.Shells) < chatShellIdentityLimit {
|
||||
stream.Shells[shell.ID] = struct{}{}
|
||||
}
|
||||
stream.mu.Unlock()
|
||||
}
|
||||
return protocol.ChatEventFrame{
|
||||
SessionID: sessionID,
|
||||
Kind: protocol.ChatEventShell,
|
||||
Shell: &protocol.ChatShell{ID: data.Info.ID, Status: status, Command: data.Info.Command},
|
||||
Shell: shell,
|
||||
}, true, nil
|
||||
}
|
||||
if data.ID == "" {
|
||||
return protocol.ChatEventFrame{}, false, fmt.Errorf("malformed OpenCode shell.exited event")
|
||||
}
|
||||
if !stream.knowsShell(data.ID) {
|
||||
if stream == nil {
|
||||
return protocol.ChatEventFrame{}, false, nil
|
||||
}
|
||||
stream.mu.Lock()
|
||||
stream.readback.Record(&protocol.ChatShell{ID: data.ID, Status: protocol.ChatShellExited})
|
||||
_, known := stream.Shells[data.ID]
|
||||
delete(stream.Shells, data.ID)
|
||||
stream.mu.Unlock()
|
||||
if !known {
|
||||
return protocol.ChatEventFrame{}, false, nil
|
||||
}
|
||||
stream.forgetShell(data.ID)
|
||||
status := data.Status
|
||||
if status == "" {
|
||||
status = protocol.ChatShellExited
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestChatShellReadbackBarrier(t *testing.T) {
|
||||
for _, known := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(known), func(t *testing.T) {
|
||||
read, apply := make(chan struct{}), make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
close(read)
|
||||
select {
|
||||
case <-apply:
|
||||
fmt.Fprint(w, `{"data":[{"id":"sh_old","status":"running","metadata":{"sessionID":"ses_1"}},{"id":"sh_foreign","status":"running","metadata":{"sessionID":"ses_other"}}]}`)
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
state := c.streamState("ses_1")
|
||||
if known {
|
||||
state.markShell("sh_old")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
result := make(chan []protocol.ChatShell, 1)
|
||||
errs := make(chan error, 1)
|
||||
go func() { shells, err := c.Shells(ctx, "ses_1"); result <- shells; errs <- err }()
|
||||
select {
|
||||
case <-read:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("read did not reach barrier")
|
||||
}
|
||||
// Both a previously verified exit and an exit first learned while
|
||||
// reading must suppress the stale running snapshot.
|
||||
frame, ok, err := NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_old","status":"exited","exit":0}}`), "ses_1", state)
|
||||
if err != nil || ok != known || (ok && frame.Shell.Status != protocol.ChatShellExited) {
|
||||
t.Fatalf("exit = %+v %v %v", frame, ok, err)
|
||||
}
|
||||
frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.created","data":{"info":{"id":"sh_new","status":"running","command":"sleep 10","metadata":{"sessionID":"ses_1"}}}}`), "ses_1", state)
|
||||
if err != nil || !ok || frame.Shell.ID != "sh_new" {
|
||||
t.Fatalf("create = %+v %v %v", frame, ok, err)
|
||||
}
|
||||
close(apply)
|
||||
got := <-result
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ID != "sh_new" || got[0].Command != "sleep 10" {
|
||||
t.Fatalf("readback lost events/leaked ownership: %+v", got)
|
||||
}
|
||||
if state.knowsShell("sh_old") || state.knowsShell("sh_foreign") || !state.knowsShell("sh_new") {
|
||||
t.Fatal("adapter identity state regressed")
|
||||
}
|
||||
// User-visible subsequent exit remains attributable after reconciliation.
|
||||
frame, ok, err = NormalizeChatEvent([]byte(`{"type":"shell.exited","data":{"id":"sh_new","status":"exited","exit":0}}`), "ses_1", state)
|
||||
if err != nil || !ok || frame.Shell.ID != "sh_new" {
|
||||
t.Fatalf("new shell exit lost: %+v %v %v", frame, ok, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReadbackCancelWaiter(t *testing.T) {
|
||||
read := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(read); <-r.Context().Done() }))
|
||||
defer srv.Close()
|
||||
c := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 2)
|
||||
go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }()
|
||||
select {
|
||||
case <-read:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("read stuck")
|
||||
}
|
||||
go func() { _, err := c.Shells(ctx, "ses_1"); result <- err }()
|
||||
cancel()
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-result:
|
||||
if err == nil {
|
||||
t.Fatal("canceled read succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("read or waiter stuck")
|
||||
}
|
||||
}
|
||||
old := c.streamState("ses_1")
|
||||
c.ReleaseShells("ses_1")
|
||||
if c.streamState("ses_1") == old {
|
||||
t.Fatal("reconnect reused released state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShellSnapshotBounded(t *testing.T) {
|
||||
var snapshot ShellSnapshot
|
||||
for i := 0; i <= chatShellIdentityLimit; i++ {
|
||||
snapshot.Record(&protocol.ChatShell{ID: fmt.Sprintf("sh_%d", i), Status: protocol.ChatShellExited})
|
||||
}
|
||||
if len(snapshot.changes) != chatShellIdentityLimit {
|
||||
t.Fatalf("unbounded journal: %d", len(snapshot.changes))
|
||||
}
|
||||
if _, err := snapshot.Merge([]protocol.ChatShell{{ID: "sh_0", Status: protocol.ChatShellRunning}}); err == nil {
|
||||
t.Fatal("overflow accepted stale snapshot")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package opencode
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// ShellSnapshot records changes while an authoritative read is in flight. Its
|
||||
// owner serializes access. Unknown exits may be recorded, but can only remove an
|
||||
// id: they never establish ownership. Overflow rejects the snapshot, not events.
|
||||
type ShellSnapshot struct {
|
||||
changes map[string]protocol.ChatShell
|
||||
bytes int
|
||||
overflow bool
|
||||
}
|
||||
|
||||
const chatShellIdentityLimit = 256
|
||||
|
||||
func (s *ShellSnapshot) Record(shell *protocol.ChatShell) {
|
||||
if s == nil || shell == nil || shell.ID == "" || s.overflow {
|
||||
return
|
||||
}
|
||||
if s.changes == nil {
|
||||
s.changes = make(map[string]protocol.ChatShell)
|
||||
}
|
||||
old := s.changes[shell.ID]
|
||||
n := s.bytes - len(old.ID) - len(old.Command) + len(shell.ID) + len(shell.Command)
|
||||
if n > MaxResponse || (old.ID == "" && len(s.changes) >= chatShellIdentityLimit) {
|
||||
s.overflow = true
|
||||
return
|
||||
}
|
||||
// A terminal event wins even if a duplicate create is delivered later.
|
||||
if old.ID != "" && old.Status != protocol.ChatShellRunning {
|
||||
return
|
||||
}
|
||||
s.changes[shell.ID] = *shell
|
||||
s.bytes = n
|
||||
}
|
||||
|
||||
func (s *ShellSnapshot) Merge(shells []protocol.ChatShell) ([]protocol.ChatShell, error) {
|
||||
if s.overflow {
|
||||
return nil, fmt.Errorf("shell reconciliation capacity exceeded")
|
||||
}
|
||||
byID := make(map[string]protocol.ChatShell)
|
||||
for _, shell := range shells {
|
||||
if shell.ID != "" && shell.Status == protocol.ChatShellRunning {
|
||||
byID[shell.ID] = shell
|
||||
}
|
||||
}
|
||||
for id, shell := range s.changes {
|
||||
if shell.Status == protocol.ChatShellRunning {
|
||||
byID[id] = shell
|
||||
} else {
|
||||
delete(byID, id)
|
||||
}
|
||||
}
|
||||
if len(byID) > chatShellIdentityLimit {
|
||||
return nil, fmt.Errorf("shell reconciliation capacity exceeded")
|
||||
}
|
||||
ids := make([]string, 0, len(byID))
|
||||
for id := range byID {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
out := make([]protocol.ChatShell, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, byID[id])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -395,3 +395,45 @@ func TestOpenChatRejectsUnrunnableCLI(t *testing.T) {
|
||||
t.Fatal("a missing CLI must fail detection, not open an endpoint")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatShellReadbackSeedsReplacementStream proves an authoritative shell
|
||||
// readback survives a later stream replacement: shell.exited has no session id,
|
||||
// so it must still clear the verified shell after reconnect.
|
||||
func TestChatShellReadbackSeedsReplacementStream(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"sh_1","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_1"}},{"id":"sh_other","status":"running","command":"sleep 30","metadata":{"sessionID":"ses_other"}}]}`))
|
||||
case "/api/event":
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = w.Write([]byte("data: {\"type\":\"shell.exited\",\"data\":{\"id\":\"sh_1\",\"status\":\"exited\",\"exit\":0}}\n\n"))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
chat := NewChat(&Connection{Backend: "v2", URL: srv.URL})
|
||||
shells, err := chat.Shells(context.Background(), "ses_1")
|
||||
if err != nil || len(shells) != 1 || shells[0].ID != "sh_1" {
|
||||
t.Fatalf("Shells = %+v, %v", shells, err)
|
||||
}
|
||||
var got []protocol.ChatEventFrame
|
||||
if err := chat.Stream(context.Background(), "ses_1", nil, func(frame protocol.ChatEventFrame) error { got = append(got, frame); return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Shell == nil || got[0].Shell.ID != "sh_1" || got[0].Shell.Status != protocol.ChatShellExited {
|
||||
t.Fatalf("exit after readback = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatReleaseShellsBoundsDisconnectedState(t *testing.T) {
|
||||
chat := NewChat(&Connection{})
|
||||
state := chat.streamState("ses_1")
|
||||
state.markShell("sh_1")
|
||||
chat.ReleaseShells("ses_1")
|
||||
chat.shellMu.Lock()
|
||||
_, retained := chat.shells["ses_1"]
|
||||
chat.shellMu.Unlock()
|
||||
if retained {
|
||||
t.Fatal("released session shell state was retained")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,9 +32,21 @@ type chatBroker interface {
|
||||
// Permissions lists the session's pending OpenCode approval requests so the
|
||||
// handler can push chat_permission frames.
|
||||
Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error)
|
||||
// Shells returns only shell operations whose ownership is authoritatively
|
||||
// verified for sessionID. It is the reconnect/readback source.
|
||||
Shells(ctx context.Context, sessionID string) ([]protocol.ChatShell, error)
|
||||
Reply(ctx context.Context, request protocol.ChatReplyRequest) error
|
||||
}
|
||||
|
||||
// chatShellReleaser is implemented by brokers that retain verified shell ids for
|
||||
// attributing exit events. The hub calls it only after the last observer leaves
|
||||
// a room; a later attach obtains a fresh authoritative snapshot.
|
||||
// This is a bounded local-cache operation: it must not perform I/O or call the
|
||||
// hub, which serializes it with room replacement under its mutex.
|
||||
type chatShellReleaser interface {
|
||||
ReleaseShells(sessionID string)
|
||||
}
|
||||
|
||||
// operatorRegistry is the server-side operator-session seam. *pipeline.Store
|
||||
// satisfies it. It is a table of its own, separate from the pipeline stage
|
||||
// ownership (SessionOwner/run_session_owners) that settles a run: chat_sessions
|
||||
|
||||
@@ -133,6 +133,13 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.applyLocked(frame)
|
||||
}
|
||||
|
||||
func (h *chatHub) applyLocked(frame protocol.ChatEventFrame) {
|
||||
if frame.SessionID == "" {
|
||||
return
|
||||
}
|
||||
turn := h.turnLocked(frame.SessionID)
|
||||
switch frame.Kind {
|
||||
case protocol.ChatEventStep:
|
||||
@@ -144,6 +151,9 @@ func (h *chatHub) apply(frame protocol.ChatEventFrame) {
|
||||
case protocol.ChatEventShell:
|
||||
// A shell frame carries no message id (it outlives the tool call that
|
||||
// started it), so it is folded before the message-id gate below.
|
||||
if room := h.rooms[frame.SessionID]; room != nil {
|
||||
room.shellReadback.Record(frame.Shell)
|
||||
}
|
||||
turn.applyShell(frame.Shell)
|
||||
return
|
||||
}
|
||||
@@ -240,7 +250,7 @@ func (t *chatTurn) applyShell(shell *protocol.ChatShell) {
|
||||
if shell == nil || shell.ID == "" {
|
||||
return
|
||||
}
|
||||
if shell.Status == protocol.ChatShellExited {
|
||||
if shell.Status != protocol.ChatShellRunning {
|
||||
old, ok := t.shells[shell.ID]
|
||||
if !ok {
|
||||
return
|
||||
|
||||
@@ -82,6 +82,8 @@ func (c *channelChat) Session(context.Context, string) (protocol.ChatSession, bo
|
||||
return protocol.ChatSession{}, false, nil
|
||||
}
|
||||
|
||||
func (c *channelChat) Shells(context.Context, string) ([]protocol.ChatShell, error) { return nil, nil }
|
||||
|
||||
func (c *channelChat) Permissions(context.Context, string) ([]protocol.ChatPermissionFrame, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
@@ -106,6 +106,11 @@ type chatQueuedPrompt struct {
|
||||
acked bool
|
||||
}
|
||||
|
||||
type chatObservation struct {
|
||||
done chan struct{}
|
||||
err error // published by closing done; belongs to this attempt, not the next
|
||||
}
|
||||
|
||||
// chatRoom is one session's shared live channel. Every field is guarded by the
|
||||
// owning chatHub's mutex.
|
||||
type chatRoom struct {
|
||||
@@ -117,6 +122,11 @@ type chatRoom struct {
|
||||
queue []*chatQueuedPrompt
|
||||
running bool
|
||||
streaming bool
|
||||
// observing is non-nil while one caller is attaching the shared broker
|
||||
// stream. It prevents concurrent subscribers from opening duplicate global
|
||||
// event streams before streaming can be published.
|
||||
observing *chatObservation
|
||||
shellReadback *opencode.ShellSnapshot
|
||||
// buffering holds the turn's frames until the prompting client's ack has been
|
||||
// written, so the ack always precedes the frames it acknowledges. It is set
|
||||
// only while the very first turn subscribes to the firehose.
|
||||
@@ -160,6 +170,81 @@ func (h *chatHub) roomLocked(sessionID string) *chatRoom {
|
||||
return room
|
||||
}
|
||||
|
||||
// ensureObserving starts the firehose even for a read-only subscriber, then
|
||||
// reconciles retained shells from the endpoint's authoritative list. A bounded
|
||||
// journal folds concurrent lifecycle events over that snapshot atomically. All
|
||||
// observers share this reconciliation, with no broker I/O under the hub mutex.
|
||||
func (h *chatHub) ensureObserving(room *chatRoom, broker chatBroker) error {
|
||||
if broker == nil {
|
||||
return errors.New("chat is not configured on this server")
|
||||
}
|
||||
h.mu.Lock()
|
||||
if err := room.ctx.Err(); err != nil {
|
||||
h.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
if waiting := room.observing; waiting != nil {
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case <-waiting.done:
|
||||
return waiting.err
|
||||
case <-room.ctx.Done():
|
||||
return room.ctx.Err()
|
||||
}
|
||||
}
|
||||
if room.streaming {
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
room.observing = &chatObservation{done: make(chan struct{})}
|
||||
waiting := room.observing
|
||||
room.shellReadback = &opencode.ShellSnapshot{}
|
||||
h.mu.Unlock()
|
||||
var observeErr error
|
||||
defer func() {
|
||||
h.mu.Lock()
|
||||
room.shellReadback = nil
|
||||
waiting.err = observeErr
|
||||
room.observing = nil
|
||||
close(waiting.done)
|
||||
h.mu.Unlock()
|
||||
}()
|
||||
if err := h.startStream(room, broker); err != nil {
|
||||
observeErr = err
|
||||
return err
|
||||
}
|
||||
shells, err := broker.Shells(room.ctx, room.sessionID)
|
||||
h.mu.Lock()
|
||||
if room.ctx.Err() != nil || h.rooms[room.sessionID] != room {
|
||||
h.mu.Unlock()
|
||||
observeErr = context.Canceled
|
||||
return observeErr
|
||||
}
|
||||
if err == nil {
|
||||
shells, err = room.shellReadback.Merge(shells)
|
||||
}
|
||||
if err != nil {
|
||||
// The firehose remains useful when its authoritative shell readback is
|
||||
// temporarily unavailable. Do not turn that read failure into a chat
|
||||
// failure (or claim the last retained snapshot is current): clear the
|
||||
// replay cache and tell attached clients that shell visibility is limited.
|
||||
// Preserve events already folded during this read, but clear stale replay.
|
||||
live, mergeErr := room.shellReadback.Merge(nil)
|
||||
if mergeErr == nil {
|
||||
h.replaceShellsLocked(room.sessionID, live)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
// Never log broker bodies/URLs: they may contain credentials or private output.
|
||||
log.Printf("chat shell readback session=%s failed (%T)", room.sessionID, err)
|
||||
h.fanout(room.sessionID, protocol.Message{Type: protocol.Error, Error: "background shell visibility is limited"}, "")
|
||||
return nil
|
||||
}
|
||||
h.replaceShellsLocked(room.sessionID, shells)
|
||||
room.shellReadback = nil
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// attach registers write as one client's subscription to sessionID's stream and
|
||||
// returns it, or the existing subscription when the same connection has already
|
||||
// attached. It is idempotent so a client may prompt and subscribe without
|
||||
@@ -209,6 +294,11 @@ func (h *chatHub) detach(sessionID string, sub *chatSub) {
|
||||
if len(room.subs) == 0 {
|
||||
delete(h.rooms, sessionID)
|
||||
room.cancel()
|
||||
// ReleaseShells is a local identity-cache operation (no broker I/O).
|
||||
// Complete it before a reconnect can acquire this session's new room.
|
||||
if releaser, ok := room.broker.(chatShellReleaser); ok {
|
||||
releaser.ReleaseShells(sessionID)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
@@ -265,9 +355,17 @@ func (h *chatHub) broadcastUser(frame protocol.ChatEventFrame, exceptClientID st
|
||||
// once it is written. A step-ended frame finishes the turn, letting the queue
|
||||
// advance.
|
||||
func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
h.apply(frame)
|
||||
h.emitRoom(nil, frame)
|
||||
}
|
||||
|
||||
func (h *chatHub) emitRoom(source *chatRoom, frame protocol.ChatEventFrame) {
|
||||
h.mu.Lock()
|
||||
room := h.rooms[frame.SessionID]
|
||||
if source != nil && (source != room || source.ctx.Err() != nil) {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.applyLocked(frame)
|
||||
buffered := false
|
||||
var targets []*chatSub
|
||||
if room != nil {
|
||||
@@ -283,6 +381,22 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
if room != nil && frame.Kind == protocol.ChatEventStep && frame.Step != nil && frame.Step.Status == protocol.ChatStepEnded {
|
||||
h.finishTurnLocked(room)
|
||||
}
|
||||
// Shell replay and live lifecycle delivery must enqueue in the same order
|
||||
// as folding. Otherwise an exit can overtake an older running replay.
|
||||
if frame.Kind == protocol.ChatEventShell && !buffered {
|
||||
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
|
||||
var slow []*chatSub
|
||||
for _, sub := range targets {
|
||||
if !enqueueChatLocked(sub, msg) {
|
||||
slow = append(slow, sub)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, sub := range slow {
|
||||
sub.close()
|
||||
}
|
||||
return
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if buffered {
|
||||
return
|
||||
@@ -293,6 +407,62 @@ func (h *chatHub) emit(frame protocol.ChatEventFrame) {
|
||||
}
|
||||
}
|
||||
|
||||
// enqueueChatLocked never blocks or calls detach under the hub mutex.
|
||||
func enqueueChatLocked(sub *chatSub, msg protocol.Message) bool {
|
||||
select {
|
||||
case <-sub.done:
|
||||
return true
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case sub.ch <- msg:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (h *chatHub) replayShells(sub *chatSub) {
|
||||
h.mu.Lock()
|
||||
slow := false
|
||||
if h.rooms[sub.room.sessionID] == sub.room && sub.room.subs[sub.clientID] == sub {
|
||||
turn := h.turnLocked(sub.room.sessionID)
|
||||
for _, id := range turn.shellOrder {
|
||||
shell := turn.shells[id]
|
||||
frame := protocol.ChatEventFrame{SessionID: sub.room.sessionID, Kind: protocol.ChatEventShell, Shell: &shell}
|
||||
if !enqueueChatLocked(sub, protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}) {
|
||||
slow = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if slow {
|
||||
sub.close()
|
||||
}
|
||||
}
|
||||
|
||||
// replaceShells reconciles a successful authoritative readback. It drops stale
|
||||
// retained entries (including exits missed while disconnected) and upserts the
|
||||
// verified running set without fabricating a lifecycle event for foreign shells.
|
||||
func (h *chatHub) replaceShells(sessionID string, shells []protocol.ChatShell) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.replaceShellsLocked(sessionID, shells)
|
||||
}
|
||||
|
||||
func (h *chatHub) replaceShellsLocked(sessionID string, shells []protocol.ChatShell) {
|
||||
turn := h.turnLocked(sessionID)
|
||||
for _, shell := range turn.shells {
|
||||
turn.shrink(len(shell.Command))
|
||||
}
|
||||
turn.shells = nil
|
||||
turn.shellOrder = nil
|
||||
for i := range shells {
|
||||
turn.applyShell(&shells[i])
|
||||
}
|
||||
}
|
||||
|
||||
// finishTurnLocked marks the room's turn finished and releases the turn runner's
|
||||
// wait. Callers must hold h.mu.
|
||||
func (h *chatHub) finishTurnLocked(room *chatRoom) {
|
||||
@@ -331,14 +501,20 @@ func (h *chatHub) flushBuffered(sessionID string) {
|
||||
targets = append(targets, sub)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
var slow []*chatSub
|
||||
for i := range pending {
|
||||
frame := pending[i]
|
||||
msg := protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame}
|
||||
for _, sub := range targets {
|
||||
sub.send(msg)
|
||||
if !enqueueChatLocked(sub, msg) {
|
||||
slow = append(slow, sub)
|
||||
}
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, sub := range slow {
|
||||
sub.close()
|
||||
}
|
||||
}
|
||||
|
||||
// submitPrompt accepts one prompt: it starts the turn runner immediately when no
|
||||
@@ -476,7 +652,7 @@ func (h *chatHub) runTurn(room *chatRoom, p *chatQueuedPrompt, done chan struct{
|
||||
return
|
||||
}
|
||||
if first {
|
||||
if err := h.startStream(room, broker); err != nil {
|
||||
if err := h.ensureObserving(room, broker); err != nil {
|
||||
h.failPrompt(room, p, err)
|
||||
h.endRoomTurn(room)
|
||||
return
|
||||
@@ -550,7 +726,7 @@ func (h *chatHub) startStream(room *chatRoom, broker chatBroker) error {
|
||||
result <- broker.Stream(ctx, room.sessionID, func() {
|
||||
once.Do(func() { close(attached) })
|
||||
}, func(frame protocol.ChatEventFrame) error {
|
||||
h.emit(frame)
|
||||
h.emitRoom(room, frame)
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"github.com/coder/websocket"
|
||||
"github.com/coder/websocket/wsjson"
|
||||
)
|
||||
|
||||
type shellBarrierBroker struct {
|
||||
*fakeChat
|
||||
read chan struct{}
|
||||
apply chan struct{}
|
||||
events chan shellBarrierEvent
|
||||
stopped chan struct{}
|
||||
reads atomic.Int32
|
||||
snapshot []protocol.ChatShell
|
||||
}
|
||||
|
||||
// Exercise the real websocket handler on a loopback scratch server and a fresh
|
||||
// SQLite database. A marker on the shared stream bounds the replay assertion
|
||||
// without sleeps, polling or a negative-read timeout.
|
||||
func TestChatShellReconciliationScratchServer(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
srv := httptest.NewServer(HandlerWithOptions(ctx, Options{Chat: b}, newOperatorStore(t)))
|
||||
defer srv.Close()
|
||||
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
subscribe := func(id string) {
|
||||
t.Helper()
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: id, ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
read := func() protocol.Message {
|
||||
t.Helper()
|
||||
var m protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
if m := read(); m.Type != protocol.PipelineList {
|
||||
t.Fatalf("server greeting = %s", m.Type)
|
||||
}
|
||||
subscribe("first")
|
||||
if m := read(); m.ID != "first" || m.Type != protocol.ChatSubscribe {
|
||||
t.Fatalf("ack = %+v", m)
|
||||
}
|
||||
shellBarrierWait(t, b.read)
|
||||
done := make(chan struct{})
|
||||
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_old", Status: protocol.ChatShellExited}, done}
|
||||
shellBarrierWait(t, done)
|
||||
if m := read(); m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellExited {
|
||||
t.Fatalf("exit not visible: %+v", m)
|
||||
}
|
||||
close(b.apply)
|
||||
subscribe("second")
|
||||
for {
|
||||
m := read()
|
||||
if m.ChatEvent != nil && m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
||||
t.Fatalf("stale replay: %+v", m)
|
||||
}
|
||||
if m.ID == "second" {
|
||||
break
|
||||
}
|
||||
}
|
||||
done = make(chan struct{})
|
||||
b.events <- shellBarrierEvent{protocol.ChatShell{ID: "sh_marker", Status: protocol.ChatShellExited}, done}
|
||||
shellBarrierWait(t, done)
|
||||
for {
|
||||
m := read()
|
||||
if m.ChatEvent == nil {
|
||||
continue
|
||||
}
|
||||
if m.ChatEvent.Shell.Status == protocol.ChatShellRunning {
|
||||
t.Fatalf("ghost visible after apply: %+v", m)
|
||||
}
|
||||
if m.ChatEvent.Shell.ID == "sh_marker" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if b.reads.Load() != 1 {
|
||||
t.Fatalf("shared observer read count = %d", b.reads.Load())
|
||||
}
|
||||
conn.CloseNow()
|
||||
shellBarrierWait(t, b.stopped)
|
||||
}
|
||||
|
||||
type shellBarrierEvent struct {
|
||||
shell protocol.ChatShell
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newShellBarrierBroker() *shellBarrierBroker {
|
||||
return &shellBarrierBroker{fakeChat: scriptedChat(), read: make(chan struct{}, 8), apply: make(chan struct{}), events: make(chan shellBarrierEvent), stopped: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (b *shellBarrierBroker) Shells(ctx context.Context, _ string) ([]protocol.ChatShell, error) {
|
||||
b.reads.Add(1)
|
||||
snapshot := append([]protocol.ChatShell(nil), b.snapshot...)
|
||||
b.read <- struct{}{}
|
||||
select {
|
||||
case <-b.apply:
|
||||
return snapshot, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (b *shellBarrierBroker) Stream(ctx context.Context, id string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
defer close(b.stopped)
|
||||
ready()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case event := <-b.events:
|
||||
if err := emit(protocol.ChatEventFrame{SessionID: id, Kind: protocol.ChatEventShell, Shell: &event.shell}); err != nil {
|
||||
return err
|
||||
}
|
||||
close(event.done)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func shellBarrierWait(t *testing.T, ch <-chan struct{}) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("barrier timed out")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReconciliationBarrier(t *testing.T) {
|
||||
for _, status := range []string{protocol.ChatShellExited, protocol.ChatShellRunning, "timeout", "killed"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
b.snapshot = []protocol.ChatShell{{ID: "sh_old", Status: protocol.ChatShellRunning}}
|
||||
h := newChatHub()
|
||||
messages := make(chan protocol.Message, 32)
|
||||
sub := h.attach("ses_1", "one", "", b, func(m protocol.Message) error { messages <- m; return nil })
|
||||
t.Cleanup(sub.close)
|
||||
result := make(chan error, 2)
|
||||
go func() { result <- h.ensureObserving(sub.room, b) }()
|
||||
shellBarrierWait(t, b.read)
|
||||
// A second observer must share the in-flight read, not apply an
|
||||
// independently completing older/newer snapshot out of order.
|
||||
go func() { result <- h.ensureObserving(sub.room, b) }()
|
||||
shell := protocol.ChatShell{ID: "sh_old", Status: status}
|
||||
if status == protocol.ChatShellRunning {
|
||||
shell.ID = "sh_new"
|
||||
shell.Command = "sleep 10"
|
||||
}
|
||||
done := make(chan struct{})
|
||||
select {
|
||||
case b.events <- shellBarrierEvent{shell, done}:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("event blocked behind readback")
|
||||
}
|
||||
shellBarrierWait(t, done)
|
||||
select {
|
||||
case m := <-messages:
|
||||
if m.ChatEvent == nil || m.ChatEvent.Shell == nil || m.ChatEvent.Shell.ID != shell.ID || m.ChatEvent.Shell.Status != status {
|
||||
t.Fatalf("live shell = %+v", m)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("live event not visible during read")
|
||||
}
|
||||
close(b.apply)
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-result:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("observer stuck")
|
||||
}
|
||||
}
|
||||
if b.reads.Load() != 1 {
|
||||
t.Fatalf("redundant/out-of-order reads: %d", b.reads.Load())
|
||||
}
|
||||
snapshot := h.shellSnapshot("ses_1")
|
||||
if status == protocol.ChatShellRunning {
|
||||
if len(snapshot) != 2 {
|
||||
t.Fatalf("new create lost: %+v", snapshot)
|
||||
}
|
||||
} else if len(snapshot) != 0 {
|
||||
t.Fatalf("terminal shell resurrected: %+v", snapshot)
|
||||
}
|
||||
// Assert the same retained state is replayed to the client.
|
||||
h.replayShells(sub)
|
||||
for range len(snapshot) {
|
||||
select {
|
||||
case m := <-messages:
|
||||
if m.ChatEvent == nil || m.ChatEvent.Shell.Status != protocol.ChatShellRunning {
|
||||
t.Fatalf("replay = %+v", m)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("replay missing")
|
||||
}
|
||||
}
|
||||
sub.close()
|
||||
shellBarrierWait(t, b.stopped)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatShellReconciliationCancelReconnectBarrier(t *testing.T) {
|
||||
b := newShellBarrierBroker()
|
||||
h := newChatHub()
|
||||
sub := h.attach("ses_1", "one", "", b, func(protocol.Message) error { return nil })
|
||||
results := make(chan error, 2)
|
||||
go func() { results <- h.ensureObserving(sub.room, b) }()
|
||||
shellBarrierWait(t, b.read)
|
||||
go func() { results <- h.ensureObserving(sub.room, b) }()
|
||||
sub.close()
|
||||
for range 2 {
|
||||
select {
|
||||
case err := <-results:
|
||||
if err == nil {
|
||||
t.Fatal("canceled observer succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("canceled waiter stuck")
|
||||
}
|
||||
}
|
||||
shellBarrierWait(t, b.stopped)
|
||||
next := newShellBarrierBroker()
|
||||
next.snapshot = []protocol.ChatShell{{ID: "sh_new", Status: protocol.ChatShellRunning}}
|
||||
close(next.apply)
|
||||
newSub := h.attach("ses_1", "two", "", next, func(protocol.Message) error { return nil })
|
||||
t.Cleanup(newSub.close)
|
||||
if err := h.ensureObserving(newSub.room, next); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A late callback from the canceled stream cannot poison the new room,
|
||||
// nor can a foreign-session callback from the current stream.
|
||||
h.emitRoom(sub.room, protocol.ChatEventFrame{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_ghost", Status: protocol.ChatShellRunning}})
|
||||
h.emitRoom(newSub.room, protocol.ChatEventFrame{SessionID: "ses_foreign", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_foreign", Status: protocol.ChatShellRunning}})
|
||||
got := h.shellSnapshot("ses_1")
|
||||
if len(got) != 1 || got[0].ID != "sh_new" || len(h.shellSnapshot("ses_foreign")) != 0 {
|
||||
t.Fatalf("reconnect state = %+v", got)
|
||||
}
|
||||
newSub.close()
|
||||
shellBarrierWait(t, next.stopped)
|
||||
}
|
||||
@@ -111,7 +111,14 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not attach to session " + query.SessionID})
|
||||
return
|
||||
}
|
||||
// Acknowledge before the stream is allowed to fan out: otherwise an immediately
|
||||
// available retained shell can overtake the subscribe reply on this connection.
|
||||
_ = write(protocol.Message{Type: protocol.ChatSubscribe, ID: request.ID})
|
||||
if err := hub.ensureObserving(sub.room, broker); err != nil {
|
||||
sub.close()
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "could not observe session " + query.SessionID})
|
||||
return
|
||||
}
|
||||
// Replay every still-unresolved approval request the room holds, not only the
|
||||
// most recent one, so a client that attaches after two requests were raised
|
||||
// still sees both cards.
|
||||
@@ -124,9 +131,5 @@ func handleChatSubscribe(request protocol.Message, broker chatBroker, hub *chatH
|
||||
// shows them rather than nothing until they exit. The message snapshot cannot
|
||||
// carry a shell — a shell outlives the turn and is not a turn — so this is the
|
||||
// shell half of the resume.
|
||||
for _, shell := range hub.shellSnapshot(query.SessionID) {
|
||||
running := shell
|
||||
frame := protocol.ChatEventFrame{SessionID: query.SessionID, Kind: protocol.ChatEventShell, Shell: &running}
|
||||
sub.send(protocol.Message{Type: protocol.ChatEvent, ChatEvent: &frame})
|
||||
}
|
||||
hub.replayShells(sub)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -53,6 +54,8 @@ type fakeChat struct {
|
||||
permissionSessionID string
|
||||
permissions []protocol.ChatPermissionFrame
|
||||
permissionErr error
|
||||
shells []protocol.ChatShell
|
||||
shellErr error
|
||||
sessionCalls int
|
||||
sessions []protocol.ChatSession
|
||||
sessionErr error
|
||||
@@ -63,6 +66,8 @@ type fakeChat struct {
|
||||
started chan struct{}
|
||||
done chan struct{}
|
||||
startOnce sync.Once
|
||||
streamCalls int
|
||||
readyGate <-chan struct{}
|
||||
block bool
|
||||
subscribed bool
|
||||
promptBeforeSubscribe bool
|
||||
@@ -92,13 +97,18 @@ func (f *fakeChat) Prompt(context.Context, string, string) error {
|
||||
func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func(protocol.ChatEventFrame) error) error {
|
||||
f.mu.Lock()
|
||||
f.subscribed = true
|
||||
f.streamCalls++
|
||||
readyGate := f.readyGate
|
||||
f.mu.Unlock()
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
if f.started != nil {
|
||||
f.startOnce.Do(func() { close(f.started) })
|
||||
}
|
||||
if readyGate != nil {
|
||||
<-readyGate
|
||||
}
|
||||
if ready != nil {
|
||||
ready()
|
||||
}
|
||||
if f.block {
|
||||
<-ctx.Done()
|
||||
if f.done != nil {
|
||||
@@ -117,6 +127,42 @@ func (f *fakeChat) Stream(ctx context.Context, _ string, ready func(), emit func
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestChatHubConcurrentObserversShareStream pins the start-up handoff: two
|
||||
// clients that subscribe before the broker says ready still share one firehose.
|
||||
func TestChatHubConcurrentObserversShareStream(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
broker.started = make(chan struct{})
|
||||
gate := make(chan struct{})
|
||||
broker.readyGate = gate
|
||||
hub := newChatHub()
|
||||
hub.mu.Lock()
|
||||
room := hub.roomLocked("ses_1")
|
||||
hub.mu.Unlock()
|
||||
|
||||
errs := make(chan error, 2)
|
||||
go func() { errs <- hub.ensureObserving(room, broker) }()
|
||||
select {
|
||||
case <-broker.started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("first observer did not start the broker stream")
|
||||
}
|
||||
go func() { errs <- hub.ensureObserving(room, broker) }()
|
||||
close(gate)
|
||||
for range 2 {
|
||||
if err := <-errs; err != nil {
|
||||
t.Fatalf("ensureObserving: %v", err)
|
||||
}
|
||||
}
|
||||
broker.mu.Lock()
|
||||
calls := broker.streamCalls
|
||||
broker.mu.Unlock()
|
||||
if calls != 1 {
|
||||
t.Fatalf("broker Stream calls = %d, want 1", calls)
|
||||
}
|
||||
room.cancel()
|
||||
}
|
||||
|
||||
func (f *fakeChat) Interrupt(context.Context, string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
@@ -153,6 +199,12 @@ func (f *fakeChat) Reply(_ context.Context, request protocol.ChatReplyRequest) e
|
||||
// Permissions returns the seeded pending requests, or blocks until the caller's
|
||||
// context is canceled when permBlock is set (used to prove the poller stops on
|
||||
// disconnect). Every call is counted so a test can assert polling happened.
|
||||
func (f *fakeChat) Shells(context.Context, string) ([]protocol.ChatShell, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]protocol.ChatShell(nil), f.shells...), f.shellErr
|
||||
}
|
||||
|
||||
func (f *fakeChat) Permissions(ctx context.Context, sessionID string) ([]protocol.ChatPermissionFrame, error) {
|
||||
f.mu.Lock()
|
||||
f.permissionCalls++
|
||||
@@ -671,6 +723,9 @@ func TestChatInterruptEndsRunningPrompt(t *testing.T) {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
@@ -984,6 +1039,7 @@ func TestChatPromptStreamsShellFrame(t *testing.T) {
|
||||
// shows the background process instead of nothing until it exits.
|
||||
func TestChatSubscribeReplaysRunningShell(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.shells = []protocol.ChatShell{{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}}
|
||||
broker.script = append([]protocol.ChatEventFrame{
|
||||
{SessionID: "ses_1", Kind: protocol.ChatEventShell, Shell: &protocol.ChatShell{ID: "sh_1", Status: protocol.ChatShellRunning, Command: "sleep 120"}},
|
||||
}, broker.script...)
|
||||
@@ -1028,6 +1084,40 @@ func TestChatSubscribeReplaysRunningShell(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatSubscribeKeepsStreamWhenShellReadbackFails ensures a temporary
|
||||
// authoritative readback failure does not make chat unavailable or replay a
|
||||
// stale process as running. The subscriber receives an explicit limitation
|
||||
// while its broker stream remains attached for future lifecycle events.
|
||||
func TestChatSubscribeKeepsStreamWhenShellReadbackFails(t *testing.T) {
|
||||
broker := scriptedChat()
|
||||
broker.block = true
|
||||
broker.shellErr = errors.New("shell endpoint unavailable: https://user:secret@example.invalid private response body")
|
||||
ctx, _, conn := dialChat(t, broker)
|
||||
if err := wsjson.Write(ctx, conn, protocol.Message{Type: protocol.ChatSubscribe, ID: "sub1", ChatSubscribe: &protocol.ChatQuery{SessionID: "ses_1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ack protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &ack); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ack.Type != protocol.ChatSubscribe || ack.ID != "sub1" {
|
||||
t.Fatalf("subscribe ack = %+v", ack)
|
||||
}
|
||||
var limited protocol.Message
|
||||
if err := wsjson.Read(ctx, conn, &limited); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if limited.Type != protocol.Error || limited.Error != "background shell visibility is limited" {
|
||||
t.Fatalf("limited visibility notice = %+v", limited)
|
||||
}
|
||||
broker.mu.Lock()
|
||||
calls := broker.streamCalls
|
||||
broker.mu.Unlock()
|
||||
if calls != 1 {
|
||||
t.Fatalf("broker Stream calls = %d, want 1", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatPromptSubscribesBeforePrompt pins the subscribe-before-prompt order:
|
||||
// the firehose has no replay cursor, so the subscription must be attached before
|
||||
// the prompt is posted or deltas emitted in between are lost.
|
||||
@@ -1068,6 +1158,9 @@ func realAdapterChat(t *testing.T, frames []string) []protocol.ChatEventFrame {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/permission/request":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"location":{"directory":"/work"},"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/shell":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/event":
|
||||
streams.Add(1)
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// verifyDelivery proves that the stage landed a complete, committed tree before
|
||||
// its success can settle the run. It re-derives the capability after the stage:
|
||||
// a stage result and its input cannot select either the landing path or checks.
|
||||
const maxDeliveryCheckOutput = 64 << 10
|
||||
const deliveryCleanupTimeout = 10 * time.Second
|
||||
|
||||
// verifyDelivery verifies the committed item tree in a disposable worktree
|
||||
// before fast-forwarding canonical main. Settlement deliberately performs only
|
||||
// non-mutating repository checks after that mutation: a declared check must
|
||||
// never be the first signal that the landed tree is known-red.
|
||||
func verifyDelivery(ctx context.Context, workspace string, access stage.LandingAccess, checks []string) error {
|
||||
lock, err := acquireDeliveryLock(ctx, access.LockPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Close()
|
||||
current, err := landingAccess(workspace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery access changed: %w", err)
|
||||
}
|
||||
if current != access {
|
||||
return fmt.Errorf("delivery access changed during stage")
|
||||
}
|
||||
for _, directory := range []string{workspace, access.MainWorktree} {
|
||||
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery checkout status: %w", err)
|
||||
}
|
||||
if status != "" {
|
||||
return fmt.Errorf("delivery checkout is not clean: %s", strings.Split(status, "\n")[0])
|
||||
}
|
||||
}
|
||||
itemHead, err := gitOutput(workspace, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery item commit: %w", err)
|
||||
}
|
||||
if err := runDeliveryCandidate(ctx, workspace, itemHead, checks); err != nil {
|
||||
return err
|
||||
}
|
||||
mainHead, err := gitOutput(access.MainWorktree, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery main commit: %w", err)
|
||||
}
|
||||
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", mainHead, itemHead).Run(); err != nil {
|
||||
return fmt.Errorf("delivery item commit cannot fast-forward canonical main")
|
||||
}
|
||||
if output, err := exec.CommandContext(ctx, "git", "-C", access.MainWorktree, "merge", "--ff-only", itemHead).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("delivery fast-forward canonical main: %w: %s", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
for _, directory := range []string{workspace, access.MainWorktree} {
|
||||
status, err := gitOutput(directory, "status", "--porcelain=v1", "--untracked-files=all")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery settlement status: %w", err)
|
||||
}
|
||||
if status != "" {
|
||||
return fmt.Errorf("delivery checkout is not clean after checks: %s", strings.Split(status, "\n")[0])
|
||||
}
|
||||
}
|
||||
// Re-derive at the settlement boundary. A success means exactly the checked
|
||||
// item commit is now reachable from the same canonical main worktree.
|
||||
current, err = landingAccess(workspace)
|
||||
if err != nil || current != access {
|
||||
return fmt.Errorf("delivery access changed during settlement")
|
||||
}
|
||||
mainHead, err = gitOutput(access.MainWorktree, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("delivery main commit: %w", err)
|
||||
}
|
||||
if err := exec.CommandContext(ctx, "git", "-C", workspace, "merge-base", "--is-ancestor", itemHead, mainHead).Run(); err != nil {
|
||||
return fmt.Errorf("delivery item commit is not an ancestor of canonical main")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func acquireDeliveryLock(ctx context.Context, path string) (*os.File, error) {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0600)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delivery lock: %w", err)
|
||||
}
|
||||
for {
|
||||
err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
if err == nil {
|
||||
return file, nil
|
||||
}
|
||||
if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("delivery lock: %w", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("delivery lock: %w", ctx.Err())
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runDeliveryCandidate(ctx context.Context, workspace, itemHead string, checks []string) error {
|
||||
candidate, err := os.MkdirTemp(filepath.Dir(workspace), ".pipelines-delivery-")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create delivery candidate: %w", err)
|
||||
}
|
||||
if err := os.Remove(candidate); err != nil {
|
||||
return fmt.Errorf("prepare delivery candidate: %w", err)
|
||||
}
|
||||
if output, err := boundedCommandOutput(ctx, workspace, "worktree", "add", "--detach", candidate, itemHead); err != nil {
|
||||
cleanupDeliveryCandidate(workspace, candidate)
|
||||
return fmt.Errorf("create delivery candidate: %w: %s", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
defer cleanupDeliveryCandidate(workspace, candidate)
|
||||
for _, check := range checks {
|
||||
if err := runDeliveryCheck(ctx, candidate, check); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupDeliveryCandidate always uses its own bounded context: the stage context
|
||||
// may already be canceled when a check fails or times out.
|
||||
func cleanupDeliveryCandidate(workspace, candidate string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), deliveryCleanupTimeout)
|
||||
defer cancel()
|
||||
_ = exec.CommandContext(ctx, "git", "-C", workspace, "worktree", "remove", "--force", candidate).Run()
|
||||
_ = os.RemoveAll(candidate)
|
||||
}
|
||||
|
||||
func boundedCommandOutput(ctx context.Context, directory string, args ...string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, "git", append([]string{"-C", directory}, args...)...)
|
||||
var output boundedDeliveryOutput
|
||||
cmd.Stdout = &output
|
||||
cmd.Stderr = &output
|
||||
err := cmd.Run()
|
||||
return []byte(output.String()), err
|
||||
}
|
||||
|
||||
func runDeliveryCheck(ctx context.Context, directory, check string) error {
|
||||
cmd := exec.CommandContext(ctx, "sh", "-c", check)
|
||||
cmd.Dir = directory
|
||||
var output boundedDeliveryOutput
|
||||
cmd.Stdout = &output
|
||||
cmd.Stderr = &output
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("delivery check %q failed: %w: %s", check, err, output.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type boundedDeliveryOutput struct{ bytes.Buffer }
|
||||
|
||||
func (b *boundedDeliveryOutput) Write(p []byte) (int, error) {
|
||||
remaining := maxDeliveryCheckOutput - b.Len()
|
||||
if remaining > 0 {
|
||||
_, _ = b.Buffer.Write(p[:min(remaining, len(p))])
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (b *boundedDeliveryOutput) String() string {
|
||||
text := strings.TrimSpace(b.Buffer.String())
|
||||
if b.Len() == maxDeliveryCheckOutput {
|
||||
return text + " (output truncated)"
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
var _ io.Writer = (*boundedDeliveryOutput)(nil)
|
||||
@@ -0,0 +1,130 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// landingAccess derives delivery paths from Git's verified worktree metadata,
|
||||
// never from stage input. A missing canonical main worktree fails closed.
|
||||
func landingAccess(workspace string) (stage.LandingAccess, error) {
|
||||
workspace, err := canonicalDirectory(workspace)
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery workspace: %w", err)
|
||||
}
|
||||
gitDir, err := gitOutput(workspace, "rev-parse", "--git-dir")
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a Git worktree: %w", err)
|
||||
}
|
||||
commonDir, err := gitOutput(workspace, "rev-parse", "--git-common-dir")
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery repository metadata: %w", err)
|
||||
}
|
||||
gitDir, err = canonicalDirectory(resolveGitPath(workspace, gitDir))
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery Git metadata: %w", err)
|
||||
}
|
||||
commonDir, err = canonicalDirectory(resolveGitPath(workspace, commonDir))
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery common Git metadata: %w", err)
|
||||
}
|
||||
main, err := mainWorktree(workspace, commonDir)
|
||||
if err != nil {
|
||||
return stage.LandingAccess{}, err
|
||||
}
|
||||
// Delivery originates in a linked item worktree. Its metadata must be under
|
||||
// the repository's canonical common directory, never a standalone or foreign
|
||||
// checkout. Git's common directory is the bounded capability needed for refs,
|
||||
// worktree state, and the landing lock.
|
||||
if !pathWithin(filepath.Join(commonDir, "worktrees"), gitDir) {
|
||||
return stage.LandingAccess{}, fmt.Errorf("delivery workspace is not a linked Git worktree")
|
||||
}
|
||||
return stage.LandingAccess{MainWorktree: main, GitMetadata: commonDir, LockPath: filepath.Join(commonDir, "pipelines-landing.lock")}, nil
|
||||
}
|
||||
|
||||
func mainWorktree(workspace, commonDir string) (string, error) {
|
||||
list, err := gitOutput(workspace, "worktree", "list", "--porcelain")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("list repository worktrees: %w", err)
|
||||
}
|
||||
for _, record := range strings.Split(strings.TrimSpace(list), "\n\n") {
|
||||
lines := strings.Split(record, "\n")
|
||||
if len(lines) == 0 || !strings.HasPrefix(lines[0], "worktree ") || !worktreeHasBranch(lines, "refs/heads/main") {
|
||||
continue
|
||||
}
|
||||
candidate, err := canonicalDirectory(strings.TrimPrefix(lines[0], "worktree "))
|
||||
if err != nil || candidate == workspace {
|
||||
continue
|
||||
}
|
||||
candidateGitDir, err := gitOutput(candidate, "rev-parse", "--git-dir")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
candidateGitDir, err = canonicalDirectory(resolveGitPath(candidate, candidateGitDir))
|
||||
if err != nil || candidateGitDir != commonDir {
|
||||
continue
|
||||
}
|
||||
candidateCommon, err := gitOutput(candidate, "rev-parse", "--git-common-dir")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
candidateCommon, err = canonicalDirectory(resolveGitPath(candidate, candidateCommon))
|
||||
if err == nil && candidateCommon == commonDir {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no canonical main worktree in the delivery repository")
|
||||
}
|
||||
|
||||
func worktreeHasBranch(lines []string, branch string) bool {
|
||||
for _, line := range lines[1:] {
|
||||
if line == "branch "+branch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func canonicalDirectory(path string) (string, error) {
|
||||
resolved, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Stat(resolved)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("not a directory")
|
||||
}
|
||||
return filepath.Clean(resolved), nil
|
||||
}
|
||||
|
||||
func resolveGitPath(workspace, value string) string {
|
||||
if filepath.IsAbs(value) {
|
||||
return value
|
||||
}
|
||||
return filepath.Join(workspace, value)
|
||||
}
|
||||
|
||||
func pathWithin(parent, child string) bool {
|
||||
rel, err := filepath.Rel(parent, child)
|
||||
return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
func gitOutput(directory string, args ...string) (string, error) {
|
||||
cmd := exec.Command("git", append([]string{"-C", directory}, args...)...)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
data, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return strings.TrimSpace(string(data)), nil
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestLandingAccessAcceptsOnlyVerifiedSameRepositoryMainWorktree(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "add", "README")
|
||||
runGit(t, main, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
canonicalMain, err := canonicalDirectory(main)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if access.MainWorktree != canonicalMain || access.GitMetadata != filepath.Join(canonicalMain, ".git") || access.LockPath != filepath.Join(canonicalMain, ".git", "pipelines-landing.lock") {
|
||||
t.Fatalf("unexpected access: %+v", access)
|
||||
}
|
||||
foreign := filepath.Join(root, "foreign")
|
||||
runGit(t, root, "init", "-b", "main", foreign)
|
||||
if _, err := landingAccess(foreign); err == nil {
|
||||
t.Fatal("foreign repository without an item worktree gained landing access")
|
||||
}
|
||||
if _, err := landingAccess(main); err == nil {
|
||||
t.Fatal("main worktree gained linked-worktree delivery access")
|
||||
}
|
||||
if err := os.RemoveAll(item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "worktree", "prune")
|
||||
if _, err := landingAccess(main); err == nil {
|
||||
t.Fatal("repository without a linked delivery worktree gained landing access")
|
||||
}
|
||||
secondItem := filepath.Join(root, "second-item")
|
||||
runGit(t, main, "worktree", "add", "-b", "second-item", secondItem)
|
||||
runGit(t, main, "branch", "-m", "main", "trunk")
|
||||
if _, err := landingAccess(secondItem); err == nil {
|
||||
t.Fatal("repository without a canonical main worktree gained landing access")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyDeliveryRequiresCleanLandedMainAndPassingChecks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "add", "README")
|
||||
runGit(t, main, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
if err := os.WriteFile(filepath.Join(item, "delivered"), []byte("complete\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, item, "add", "delivered")
|
||||
runGit(t, item, "commit", "-m", "delivery")
|
||||
runGit(t, main, "merge", "--ff-only", "item")
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f delivered"}); err != nil {
|
||||
t.Fatalf("complete landed delivery rejected: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(item, "missing-helper"), []byte("dirty\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"true"}); err == nil {
|
||||
t.Fatal("dirty task-owned helper was accepted")
|
||||
}
|
||||
if err := os.Remove(filepath.Join(item, "missing-helper")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"false"}); err == nil {
|
||||
t.Fatal("failed required check was accepted")
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, item, "add", "unlanded")
|
||||
runGit(t, item, "commit", "-m", "unlanded")
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f unlanded"}); err != nil {
|
||||
t.Fatalf("complete unlanded item was not landed after candidate verification: %v", err)
|
||||
}
|
||||
mainHead := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
itemHead := runGitOutput(t, item, "rev-parse", "HEAD")
|
||||
if mainHead != itemHead {
|
||||
t.Fatal("verified item was not fast-forwarded into canonical main")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyDeliveryFailingCandidateLeavesMainUnchanged(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "add", "README")
|
||||
runGit(t, main, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
if err := os.WriteFile(filepath.Join(item, "candidate"), []byte("candidate\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, item, "add", "candidate")
|
||||
runGit(t, item, "commit", "-m", "candidate")
|
||||
access, err := landingAccess(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := runGitOutput(t, main, "rev-parse", "HEAD")
|
||||
if err := verifyDelivery(context.Background(), item, access, []string{"test -f candidate", "false"}); err == nil {
|
||||
t.Fatal("failing candidate check landed main")
|
||||
}
|
||||
if after := runGitOutput(t, main, "rev-parse", "HEAD"); after != before {
|
||||
t.Fatal("failing candidate check changed canonical main")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerSettlesSuccessfulDeliveryAfterCandidateLanding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
main := filepath.Join(root, "main")
|
||||
runGit(t, root, "init", "-b", "main", main)
|
||||
runGit(t, main, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, main, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(main, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, main, "add", "README")
|
||||
runGit(t, main, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, main, "worktree", "add", "-b", "item", item)
|
||||
if err := os.WriteFile(filepath.Join(item, "unlanded"), []byte("no\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, item, "add", "unlanded")
|
||||
runGit(t, item, "commit", "-m", "unlanded")
|
||||
|
||||
store, err := pipeline.Open(filepath.Join(root, "runner.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"committer"}, true)
|
||||
source = strings.Replace(source, "```configs", "```checks\ntrue\n```\n\n```configs", 1)
|
||||
source = strings.Replace(source, "timeout=1h", "timeout=1h\ndelivery=true", 1)
|
||||
if _, err := store.Submit(ctx, "delivery", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "unlanded", Workspace: item, Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "succeeded" || len(detail.Steps) != 1 {
|
||||
t.Fatalf("complete delivery did not settle successfully: %+v", detail)
|
||||
}
|
||||
if runGitOutput(t, main, "rev-parse", "HEAD") != runGitOutput(t, item, "rev-parse", "HEAD") {
|
||||
t.Fatal("runner did not land the verified item commit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRejectsWrongBackendVerificationEvidence(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1)
|
||||
r, store, id := submitRunner(t, ctx, source)
|
||||
defer store.Close()
|
||||
r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) {
|
||||
return stage.VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
run := startRun(t, r, id, "wrong-backend-evidence")
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "failed" || len(detail.Steps) != 1 || !strings.Contains(detail.Steps[0].Diagnostic, "backend") {
|
||||
t.Fatalf("wrong-backend evidence settled as success: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerIgnoresModelSuppliedVerificationEvidence(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground\n```\n\n```configs", 1)
|
||||
r, store, id := submitRunner(t, ctx, source)
|
||||
defer store.Close()
|
||||
r.verifyTarget = func(context.Context, stage.VerificationTarget) (stage.VerificationEvidence, error) {
|
||||
return stage.VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}, nil
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "invented", "fixture": "other"}}}, nil
|
||||
}
|
||||
run := startRun(t, r, id, "model-supplied-evidence")
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "succeeded" || len(detail.Steps) != 1 {
|
||||
t.Fatalf("controlled verification did not settle: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainWorktreeRejectsNearMatchAndLinkedMain(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
branch string
|
||||
}{
|
||||
{name: "mainline is not main", branch: "mainline"},
|
||||
{name: "linked main is not canonical", branch: "main"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
primary := filepath.Join(root, "primary")
|
||||
runGit(t, root, "init", "-b", "trunk", primary)
|
||||
runGit(t, primary, "config", "user.email", "test@example.invalid")
|
||||
runGit(t, primary, "config", "user.name", "test")
|
||||
if err := os.WriteFile(filepath.Join(primary, "README"), []byte("base\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runGit(t, primary, "add", "README")
|
||||
runGit(t, primary, "commit", "-m", "base")
|
||||
item := filepath.Join(root, "item")
|
||||
runGit(t, primary, "worktree", "add", "-b", "item", item)
|
||||
candidate := filepath.Join(root, test.branch)
|
||||
runGit(t, primary, "worktree", "add", "-b", test.branch, candidate)
|
||||
|
||||
commonDir, err := gitOutput(item, "rev-parse", "--git-common-dir")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
commonDir, err = canonicalDirectory(resolveGitPath(item, commonDir))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := mainWorktree(item, commonDir); err == nil {
|
||||
t.Fatalf("%s gained canonical main access", test.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func runGit(t *testing.T, directory string, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", directory}, args...)...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v: %s", args, err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func runGitOutput(t *testing.T, directory string, args ...string) string {
|
||||
t.Helper()
|
||||
output, err := exec.Command("git", append([]string{"-C", directory}, args...)...).Output()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v", args, err)
|
||||
}
|
||||
return strings.TrimSpace(string(output))
|
||||
}
|
||||
@@ -58,8 +58,11 @@ type runner struct {
|
||||
// directory. It defaults to stage.ExportSession; a failure is best-effort and
|
||||
// never changes the stage's status, outcome, target or result validation.
|
||||
exportSession func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error)
|
||||
wg sync.WaitGroup
|
||||
schedulerWG sync.WaitGroup
|
||||
// verifyTarget is installed only by server wiring for a controlled fixture
|
||||
// workflow. Leaving it nil fails closed for target-aware definitions.
|
||||
verifyTarget stage.VerificationRunner
|
||||
wg sync.WaitGroup
|
||||
schedulerWG sync.WaitGroup
|
||||
// attentionMu guards attentionSent, the per-run set of run-attention events
|
||||
// already delivered — by a stage's own notify policy or by the run-level
|
||||
// default — so neither republishes the same genuine event.
|
||||
@@ -1079,7 +1082,22 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
}
|
||||
operations := &stage.OperationLog{}
|
||||
var landing stage.LandingAccess
|
||||
delivery, deliveryErr := stage.DeliveryEnabled(def)
|
||||
if deliveryErr != nil {
|
||||
finish("failed", "error", deliveryErr.Error())
|
||||
return
|
||||
}
|
||||
if delivery {
|
||||
access, accessErr := landingAccess(workspace)
|
||||
if accessErr != nil {
|
||||
finish("failed", "error", "delivery access denied: "+accessErr.Error())
|
||||
return
|
||||
}
|
||||
landing = access
|
||||
}
|
||||
client := r.agentClient(id, sequence, state.artifactDirs, workspace, artifactsDir, operations, request.WritePaths, tools)
|
||||
client.LandingAccess = landing
|
||||
client.OnSession = func(session string, active bool) error {
|
||||
c, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@@ -1222,6 +1240,45 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
step.Output["salvage"] = result.Output
|
||||
}
|
||||
}
|
||||
if err == nil && result.Outcome == "success" {
|
||||
target, targetErr := stage.VerificationTargetFor(def)
|
||||
if targetErr != nil {
|
||||
err = targetErr
|
||||
} else if target != nil {
|
||||
if r.verifyTarget == nil {
|
||||
err = fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
|
||||
} else {
|
||||
evidence, verificationErr := r.verifyTarget(stageCtx, *target)
|
||||
if verificationErr != nil {
|
||||
err = verificationErr
|
||||
} else {
|
||||
err = stage.VerifyTargetEvidence(*target, evidence)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
step.Status = "failed"
|
||||
step.Diagnostic = err.Error()
|
||||
step.Outcome = "error"
|
||||
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
||||
}
|
||||
}
|
||||
if err == nil && delivery && result.Outcome == "success" {
|
||||
checks, checksErr := stage.DeliveryChecks(def)
|
||||
if checksErr != nil {
|
||||
err = checksErr
|
||||
} else {
|
||||
gateCtx, gateCancel := context.WithTimeout(stageCtx, modelConfig.Timeout)
|
||||
err = verifyDelivery(gateCtx, workspace, landing, checks)
|
||||
gateCancel()
|
||||
}
|
||||
if err != nil {
|
||||
step.Status = "failed"
|
||||
step.Diagnostic = err.Error()
|
||||
step.Outcome = "error"
|
||||
step.Output = stage.Values{"error": err.Error(), "stage": key, "input": input}
|
||||
}
|
||||
}
|
||||
// A failed or timed-out step carries the session pointer even when the
|
||||
// bounded transcript would otherwise be only its stage_limits envelope.
|
||||
if err != nil && stageSession != "" {
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
const (
|
||||
verificationFixtureID = "reconnect-foreground"
|
||||
verificationFixtureResponseLimit int64 = 64 << 10
|
||||
verificationFixtureTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// VerificationFixture is a server-owned, explicitly registered probe. Its
|
||||
// endpoint and expected provenance come from server configuration, never a
|
||||
// stage definition or result.
|
||||
type VerificationFixture struct {
|
||||
ID, Endpoint, Backend, Revision, Capability, State string
|
||||
}
|
||||
|
||||
// VerificationFixturesFromEnvironment returns the single supported fixture
|
||||
// registration when its endpoint is explicitly configured. There is no default
|
||||
// endpoint: an unconfigured server fails target-aware verification closed.
|
||||
func VerificationFixturesFromEnvironment(getenv func(string) string) []VerificationFixture {
|
||||
if getenv == nil {
|
||||
getenv = os.Getenv
|
||||
}
|
||||
endpoint := strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_URL"))
|
||||
if endpoint == "" {
|
||||
return nil
|
||||
}
|
||||
return []VerificationFixture{{
|
||||
ID: verificationFixtureID,
|
||||
Endpoint: endpoint,
|
||||
Backend: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_BACKEND")),
|
||||
Revision: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_REVISION")),
|
||||
Capability: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_CAPABILITY")),
|
||||
State: strings.TrimSpace(getenv("PIPELINES_VERIFICATION_RECONNECT_FOREGROUND_STATE")),
|
||||
}}
|
||||
}
|
||||
|
||||
type verificationFixtureRegistry struct {
|
||||
fixtures map[string]VerificationFixture
|
||||
err error
|
||||
}
|
||||
|
||||
func newVerificationFixtureRegistry(fixtures []VerificationFixture) *verificationFixtureRegistry {
|
||||
r := &verificationFixtureRegistry{fixtures: make(map[string]VerificationFixture, len(fixtures))}
|
||||
for _, fixture := range fixtures {
|
||||
if err := validVerificationFixture(fixture); err != nil {
|
||||
r.err = err
|
||||
return r
|
||||
}
|
||||
if _, exists := r.fixtures[fixture.ID]; exists {
|
||||
r.err = fmt.Errorf("duplicate verification fixture %q", fixture.ID)
|
||||
return r
|
||||
}
|
||||
r.fixtures[fixture.ID] = fixture
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func validVerificationFixture(fixture VerificationFixture) error {
|
||||
if fixture.ID != verificationFixtureID {
|
||||
return fmt.Errorf("unsupported verification fixture %q", fixture.ID)
|
||||
}
|
||||
for _, value := range []string{fixture.Endpoint, fixture.Backend, fixture.Revision, fixture.Capability, fixture.State} {
|
||||
if value == "" || len(value) > 256 {
|
||||
return fmt.Errorf("verification fixture %q has incomplete configuration", fixture.ID)
|
||||
}
|
||||
}
|
||||
u, err := url.Parse(fixture.Endpoint)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return fmt.Errorf("verification fixture %q has invalid endpoint", fixture.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *verificationFixtureRegistry) verify(ctx context.Context, target stage.VerificationTarget) (stage.VerificationEvidence, error) {
|
||||
if r.err != nil {
|
||||
return stage.VerificationEvidence{}, r.err
|
||||
}
|
||||
fixture, exists := r.fixtures[target.Fixture]
|
||||
if !exists {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("unknown verification fixture %q", target.Fixture)
|
||||
}
|
||||
if fixture.Backend != target.Backend || fixture.Revision != target.Revision || fixture.Capability != target.Capability {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification target does not match registered fixture %q", target.Fixture)
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, verificationFixtureTimeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(probeCtx, http.MethodGet, fixture.Endpoint, nil)
|
||||
if err != nil {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q request: %w", fixture.ID, err)
|
||||
}
|
||||
client := &http.Client{Timeout: verificationFixtureTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe: %w", fixture.ID, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q probe returned %s", fixture.ID, response.Status)
|
||||
}
|
||||
var observed struct {
|
||||
Backend string `json:"backend"`
|
||||
Revision string `json:"revision"`
|
||||
Capability string `json:"capability"`
|
||||
Fixture string `json:"fixture"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, verificationFixtureResponseLimit+1))
|
||||
if err != nil {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err)
|
||||
}
|
||||
if int64(len(body)) > verificationFixtureResponseLimit {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response exceeds %d bytes", fixture.ID, verificationFixtureResponseLimit)
|
||||
}
|
||||
if err := json.Unmarshal(body, &observed); err != nil {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q response: %w", fixture.ID, err)
|
||||
}
|
||||
if observed.Backend != fixture.Backend || observed.Revision != fixture.Revision || observed.Capability != fixture.Capability || observed.Fixture != fixture.ID || observed.State != fixture.State {
|
||||
return stage.VerificationEvidence{}, fmt.Errorf("verification fixture %q returned incompatible backend, revision, capability, fixture, or state", fixture.ID)
|
||||
}
|
||||
return stage.VerificationEvidence{Backend: observed.Backend, Revision: observed.Revision, Capability: observed.Capability, Fixture: observed.Fixture}, nil
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestHandlerRunsRegisteredVerificationFixture(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var mu sync.Mutex
|
||||
response := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"}
|
||||
calls := 0
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
calls++
|
||||
_ = json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer fixture.Close()
|
||||
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "fixture.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"worker"}, true)
|
||||
source = strings.Replace(source, "```configs", "```verification\nbackend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture="+verificationFixtureID+"\n```\n\n```configs", 1)
|
||||
if _, err := store.Submit(ctx, "controlled-fixture", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := HandlerWithOptions(ctx, Options{VerificationFixtures: []VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}}}, store).(*runHandler)
|
||||
defer func() {
|
||||
cancel()
|
||||
WaitRuns(handler)
|
||||
}()
|
||||
r := handler.runner
|
||||
if r.verifyTarget == nil {
|
||||
t.Fatal("normal server construction did not wire controlled fixture verification")
|
||||
}
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
// These forged model labels must not influence fixture provenance.
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"verification": map[string]any{"backend": "old-api", "revision": "stale", "capability": "forged", "fixture": "other"}}}, nil
|
||||
}
|
||||
start := func(key string) *protocol.RunDetail {
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "verify"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
detail, err := store.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return detail
|
||||
}
|
||||
if detail := start("compatible"); detail.Status != "succeeded" {
|
||||
t.Fatalf("compatible controlled fixture did not settle: %+v", detail)
|
||||
}
|
||||
mu.Lock()
|
||||
response["revision"] = "old-build"
|
||||
mu.Unlock()
|
||||
if detail := start("stale"); detail.Status != "failed" || !strings.Contains(detail.Steps[0].Diagnostic, "incompatible") {
|
||||
t.Fatalf("stale fixture provenance settled as success: %+v", detail)
|
||||
}
|
||||
mu.Lock()
|
||||
if calls != 2 {
|
||||
t.Fatalf("fixture probes = %d, want one per target-aware stage", calls)
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
func TestVerificationFixtureCancellationClosesProbe(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
canceled := make(chan struct{})
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(started)
|
||||
<-r.Context().Done()
|
||||
close(canceled)
|
||||
}))
|
||||
defer fixture.Close()
|
||||
registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", State: "ready"}})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := registry.verify(ctx, stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: verificationFixtureID})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("fixture probe did not start")
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("canceled fixture probe succeeded")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("canceled fixture probe did not return")
|
||||
}
|
||||
select {
|
||||
case <-canceled:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("canceled fixture request was not cleaned up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerificationFixtureRejectsUnknownAndMismatchedProvenance(t *testing.T) {
|
||||
base := map[string]string{"backend": "branch-api", "revision": "abc123", "capability": "controlled-state", "fixture": verificationFixtureID, "state": "ready"}
|
||||
for field, wrong := range map[string]string{"backend": "old-api", "revision": "old-build", "capability": "other-state", "state": "stale"} {
|
||||
t.Run(field, func(t *testing.T) {
|
||||
response := make(map[string]string, len(base))
|
||||
for key, value := range base {
|
||||
response[key] = value
|
||||
}
|
||||
response[field] = wrong
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(response)
|
||||
}))
|
||||
defer fixture.Close()
|
||||
registry := newVerificationFixtureRegistry([]VerificationFixture{{ID: verificationFixtureID, Endpoint: fixture.URL, Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], State: base["state"]}})
|
||||
_, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: base["backend"], Revision: base["revision"], Capability: base["capability"], Fixture: verificationFixtureID})
|
||||
if err == nil || !strings.Contains(err.Error(), "incompatible") {
|
||||
t.Fatalf("mismatched %s provenance was accepted: %v", field, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
registry := newVerificationFixtureRegistry(nil)
|
||||
if _, err := registry.verify(context.Background(), stage.VerificationTarget{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "unknown"}); err == nil || !strings.Contains(err.Error(), "unknown") {
|
||||
t.Fatalf("unknown fixture was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,10 @@ type Options struct {
|
||||
// read time with no model call, which keeps tests and unconfigured servers
|
||||
// deterministic.
|
||||
StepVerdicts func(context.Context, map[string]any) string
|
||||
// VerificationFixtures are explicit server-owned HTTP fixtures for optional
|
||||
// target-aware verification. Definitions can name only a registered ID; they
|
||||
// cannot select an endpoint or supply provenance.
|
||||
VerificationFixtures []VerificationFixture
|
||||
}
|
||||
|
||||
// Handler serves the persistent connection with default options. Canceling ctx
|
||||
@@ -81,6 +85,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
if runErr == nil && options.StepVerdicts != nil {
|
||||
runs.verdict = options.StepVerdicts
|
||||
}
|
||||
if runErr == nil {
|
||||
fixtures := newVerificationFixtureRegistry(options.VerificationFixtures)
|
||||
runs.verifyTarget = fixtures.verify
|
||||
}
|
||||
if runErr == nil {
|
||||
runs.schedulerWG.Add(1)
|
||||
go runs.scheduleLoop()
|
||||
|
||||
@@ -19,6 +19,7 @@ type AgentClient struct {
|
||||
// ArtifactAccess is supplied only by the runner. Write is this visit's
|
||||
// directory; Read contains this and completed earlier visits in the same run.
|
||||
ArtifactAccess ArtifactAccess
|
||||
LandingAccess LandingAccess
|
||||
OnSession func(string, bool) error
|
||||
SessionID string
|
||||
Transcript string
|
||||
|
||||
+14
-2
@@ -59,7 +59,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
}
|
||||
for block := range def.Blocks {
|
||||
switch block {
|
||||
case "output", "prompt", "edges", "configs", "tools", "permissions":
|
||||
case "output", "prompt", "edges", "configs", "tools", "permissions", "checks", "verification":
|
||||
default:
|
||||
return ModelSpec{}, fmt.Errorf("unexpected block %q for model stage", block)
|
||||
}
|
||||
@@ -70,6 +70,9 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
if _, err := StagePermissions(def); err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
if _, err := VerificationTargetFor(def); err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
for _, key := range []string{"provider", "model"} {
|
||||
if def.Configs[key] == "" {
|
||||
return ModelSpec{}, fmt.Errorf("missing config %s", key)
|
||||
@@ -77,7 +80,7 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
}
|
||||
for key := range def.Configs {
|
||||
switch key {
|
||||
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness":
|
||||
case "provider", "model", "retries", "timeout", "budget_usd", "budget_tokens", "harness", "delivery":
|
||||
default:
|
||||
return ModelSpec{}, fmt.Errorf("unknown config %q", key)
|
||||
}
|
||||
@@ -92,6 +95,15 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
if backend != "opencode" && backend != "pi" {
|
||||
return ModelSpec{}, fmt.Errorf("harness must be opencode or pi")
|
||||
}
|
||||
if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" {
|
||||
return ModelSpec{}, fmt.Errorf("delivery must be true when set")
|
||||
}
|
||||
if def.Configs["delivery"] == "true" && backend != "opencode" {
|
||||
return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness")
|
||||
}
|
||||
if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" {
|
||||
return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages")
|
||||
}
|
||||
if backend == "pi" {
|
||||
tools, _, _ := ToolNames(def)
|
||||
if _, err := piTools(tools); err != nil {
|
||||
|
||||
@@ -14,6 +14,77 @@ type ArtifactAccess struct {
|
||||
Read []string
|
||||
}
|
||||
|
||||
// LandingAccess is the server-derived external capability for a verified
|
||||
// delivery operation. Empty or malformed values grant nothing.
|
||||
type LandingAccess struct {
|
||||
MainWorktree string
|
||||
GitMetadata string
|
||||
LockPath string
|
||||
}
|
||||
|
||||
func (a LandingAccess) valid() bool {
|
||||
for _, path := range []string{a.MainWorktree, a.GitMetadata, a.LockPath} {
|
||||
if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsAny(path, "*?\\") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// The server grants the canonical repository metadata directory as OpenCode
|
||||
// V2's smallest external-directory unit. Keep the lock inside that directory
|
||||
// so a caller cannot turn a valid metadata capability into access elsewhere.
|
||||
return a.GitMetadata == filepath.Join(a.MainWorktree, ".git") &&
|
||||
a.LockPath == filepath.Join(a.GitMetadata, "pipelines-landing.lock")
|
||||
}
|
||||
|
||||
// DeliveryEnabled is the narrow, declarative capability check used before the
|
||||
// runner supplies any landing paths. Only a validated model definition can ask
|
||||
// for it; runtime paths are still derived by the server.
|
||||
func DeliveryEnabled(def Definition) (bool, error) {
|
||||
delivery := def.Configs["delivery"]
|
||||
if delivery == "" {
|
||||
return false, nil
|
||||
}
|
||||
if delivery != "true" {
|
||||
return false, fmt.Errorf("delivery must be true when set")
|
||||
}
|
||||
if EffectiveType(def.Type) != ModelType {
|
||||
return false, fmt.Errorf("delivery is supported only by model stages")
|
||||
}
|
||||
if _, err := (ModelStage{}).Spec(def); err != nil {
|
||||
return false, fmt.Errorf("invalid delivery model stage: %w", err)
|
||||
}
|
||||
if _, err := DeliveryChecks(def); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// DeliveryChecks returns the bounded, definition-declared checks that the
|
||||
// runner executes against a committed delivery candidate before canonical main
|
||||
// is changed. They
|
||||
// are intentionally a block rather than a generic runner setting: each
|
||||
// workflow owns its repository-specific verification commands.
|
||||
func DeliveryChecks(def Definition) ([]string, error) {
|
||||
text := strings.TrimSpace(def.Blocks["checks"])
|
||||
if text == "" {
|
||||
return nil, fmt.Errorf("delivery stages require a nonempty checks block")
|
||||
}
|
||||
var checks []string
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
check := strings.TrimSpace(line)
|
||||
if check == "" {
|
||||
continue
|
||||
}
|
||||
if len(check) > 4096 {
|
||||
return nil, fmt.Errorf("delivery check exceeds 4096 bytes")
|
||||
}
|
||||
checks = append(checks, check)
|
||||
}
|
||||
if len(checks) == 0 || len(checks) > 20 {
|
||||
return nil, fmt.Errorf("delivery stages require 1 to 20 checks")
|
||||
}
|
||||
return checks, nil
|
||||
}
|
||||
|
||||
// ToolNames validates model-stage permissions, not general graph semantics.
|
||||
// A present empty block explicitly denies all tools.
|
||||
func ToolNames(def Definition) ([]string, bool, error) {
|
||||
@@ -155,6 +226,10 @@ func (c *AgentClient) permissionRules() []map[string]string {
|
||||
allow("external_directory", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*")))
|
||||
allow("edit", filepath.ToSlash(filepath.Join(c.ArtifactAccess.Write, "*")))
|
||||
}
|
||||
if c.LandingAccess.valid() {
|
||||
allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.MainWorktree, "*")))
|
||||
allow("external_directory", filepath.ToSlash(filepath.Join(c.LandingAccess.GitMetadata, "*")))
|
||||
}
|
||||
for _, pattern := range []string{"../*", "*/../*"} {
|
||||
rules = append(rules, map[string]string{"action": "external_directory", "resource": pattern, "effect": "deny"})
|
||||
}
|
||||
|
||||
@@ -174,3 +174,107 @@ func TestArtifactPermissionsAreExactAndToolScoped(t *testing.T) {
|
||||
t.Fatal("missing artifact scope enabled an external directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLandingPermissionsAreExactAndServerDerived(t *testing.T) {
|
||||
access := LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/repos/pipelines/.git/pipelines-landing.lock"}
|
||||
client := &AgentClient{Tools: []string{"read", "shell"}, LandingAccess: access}
|
||||
rules := client.permissionRules()
|
||||
for _, path := range []string{"/repos/pipelines/go.mod", "/repos/pipelines/.git/worktrees/item/HEAD", "/repos/pipelines/.git/pipelines-landing.lock"} {
|
||||
if permissionEffect(rules, "external_directory", path) != "allow" {
|
||||
t.Fatalf("verified landing path denied: %q", path)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"/repos/other/go.mod", "/repos/pipelines/../other/go.mod", "/Users/alan/.git/config", "/.git/config"} {
|
||||
if permissionEffect(rules, "external_directory", path) != "deny" {
|
||||
t.Fatalf("unverified landing path allowed: %q", path)
|
||||
}
|
||||
}
|
||||
client.LandingAccess.MainWorktree = "/repos/pipelines/../other"
|
||||
if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/go.mod") != "deny" {
|
||||
t.Fatal("non-canonical landing capability allowed")
|
||||
}
|
||||
client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/pipelines/.git", LockPath: "/tmp/pipelines-landing.lock"}
|
||||
if permissionEffect(client.permissionRules(), "external_directory", "/tmp/pipelines-landing.lock") != "deny" {
|
||||
t.Fatal("landing lock outside verified metadata allowed")
|
||||
}
|
||||
client.LandingAccess = LandingAccess{MainWorktree: "/repos/pipelines", GitMetadata: "/repos/other/.git", LockPath: "/repos/other/.git/pipelines-landing.lock"}
|
||||
if permissionEffect(client.permissionRules(), "external_directory", "/repos/other/.git/config") != "deny" {
|
||||
t.Fatal("unrelated Git metadata allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryEnabledRequiresValidatedModelStage(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["delivery"] = "true"
|
||||
def.Blocks["checks"] = "go test ./..."
|
||||
if enabled, err := DeliveryEnabled(def); err != nil || !enabled {
|
||||
t.Fatalf("valid delivery stage = %v, %v", enabled, err)
|
||||
}
|
||||
for name, change := range map[string]func(*Definition){
|
||||
"malformed value": func(d *Definition) { d.Configs["delivery"] = "yes" },
|
||||
"non-model": func(d *Definition) { d.Type = HumanType },
|
||||
"invalid model": func(d *Definition) { d.Configs["unexpected"] = "value" },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
copy := def
|
||||
copy.Configs = map[string]string{}
|
||||
for key, value := range def.Configs {
|
||||
copy.Configs[key] = value
|
||||
}
|
||||
change(©)
|
||||
if enabled, err := DeliveryEnabled(copy); err == nil || enabled {
|
||||
t.Fatalf("invalid delivery stage = %v, %v", enabled, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryRejectsPiBeforeAgentExecution(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["delivery"] = "true"
|
||||
def.Configs["harness"] = "pi"
|
||||
def.Blocks["checks"] = "true"
|
||||
if err := (ModelStage{}).Validate(def); err == nil || !strings.Contains(err.Error(), "opencode harness") {
|
||||
t.Fatal("Pi delivery was accepted", err)
|
||||
}
|
||||
delete(def.Configs, "delivery")
|
||||
delete(def.Blocks, "checks")
|
||||
if err := (ModelStage{}).Validate(def); err != nil {
|
||||
t.Fatal("Pi non-delivery was changed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryChecksRequireBoundedDeliveryOnlyDeclaration(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Configs["delivery"] = "true"
|
||||
for name, mutate := range map[string]func(*Definition){
|
||||
"missing": func(d *Definition) {},
|
||||
"empty": func(d *Definition) { d.Blocks["checks"] = "\n" },
|
||||
"non delivery": func(d *Definition) { d.Blocks["checks"] = "go test ./..." },
|
||||
"too many checks": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("true\n", 21) },
|
||||
"oversized check": func(d *Definition) { d.Blocks["checks"] = strings.Repeat("x", 4097) },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
copy := def
|
||||
copy.Blocks = map[string]string{}
|
||||
copy.Configs = map[string]string{}
|
||||
for key, value := range def.Blocks {
|
||||
copy.Blocks[key] = value
|
||||
}
|
||||
for key, value := range def.Configs {
|
||||
copy.Configs[key] = value
|
||||
}
|
||||
mutate(©)
|
||||
if name == "non delivery" {
|
||||
delete(copy.Configs, "delivery")
|
||||
if _, err := (ModelStage{}).Spec(copy); err == nil {
|
||||
t.Fatal("non-delivery checks block was accepted")
|
||||
}
|
||||
return
|
||||
}
|
||||
if _, err := DeliveryEnabled(copy); err == nil {
|
||||
t.Fatal("invalid delivery checks were accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// VerificationTarget declares the controlled fixture that can substantiate a
|
||||
// target-specific behavior claim. It is intentionally data-only: the runner
|
||||
// does not execute fixture input supplied by a stage result.
|
||||
type VerificationTarget struct {
|
||||
Backend string
|
||||
Revision string
|
||||
Capability string
|
||||
Fixture string
|
||||
}
|
||||
|
||||
// VerificationEvidence is returned by a server-controlled fixture workflow,
|
||||
// never parsed from a model's result.
|
||||
type VerificationEvidence struct {
|
||||
Backend string
|
||||
Revision string
|
||||
Capability string
|
||||
Fixture string
|
||||
}
|
||||
|
||||
// VerificationRunner is the narrow server seam for a controlled fixture or
|
||||
// workflow. The runner selects it during server setup; model output cannot
|
||||
// select a backend, revision, fixture, or executable.
|
||||
type VerificationRunner func(context.Context, VerificationTarget) (VerificationEvidence, error)
|
||||
|
||||
// VerificationTargetFor parses an optional, bounded verification block. A
|
||||
// declared target requires all provenance fields so an old or other-backend
|
||||
// fixture cannot be presented as evidence for the current target.
|
||||
func VerificationTargetFor(def Definition) (*VerificationTarget, error) {
|
||||
text, declared := def.Blocks["verification"]
|
||||
if !declared {
|
||||
return nil, nil
|
||||
}
|
||||
if len(text) > 4096 {
|
||||
return nil, fmt.Errorf("verification block exceeds 4096 bytes")
|
||||
}
|
||||
values := map[string]string{}
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
key, value = strings.TrimSpace(key), strings.TrimSpace(value)
|
||||
if !ok || value == "" || len(value) > 256 {
|
||||
return nil, fmt.Errorf("invalid verification entry %q", line)
|
||||
}
|
||||
switch key {
|
||||
case "backend", "revision", "capability", "fixture":
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown verification entry %q", key)
|
||||
}
|
||||
if _, exists := values[key]; exists {
|
||||
return nil, fmt.Errorf("duplicate verification entry %q", key)
|
||||
}
|
||||
values[key] = value
|
||||
}
|
||||
target := &VerificationTarget{Backend: values["backend"], Revision: values["revision"], Capability: values["capability"], Fixture: values["fixture"]}
|
||||
if target.Backend == "" || target.Revision == "" || target.Capability == "" || target.Fixture == "" {
|
||||
return nil, fmt.Errorf("verification requires backend, revision, capability, and fixture")
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
// VerifyTargetEvidence compares server-collected controlled-fixture provenance
|
||||
// with the workflow declaration. It deliberately accepts no model result: a
|
||||
// stage cannot substantiate its own target-specific behavior claim.
|
||||
func VerifyTargetEvidence(target VerificationTarget, evidence VerificationEvidence) error {
|
||||
for key, want := range map[string]string{
|
||||
"backend": target.Backend, "revision": target.Revision, "capability": target.Capability, "fixture": target.Fixture,
|
||||
} {
|
||||
got := map[string]string{
|
||||
"backend": evidence.Backend, "revision": evidence.Revision, "capability": evidence.Capability, "fixture": evidence.Fixture,
|
||||
}[key]
|
||||
if got != want {
|
||||
return fmt.Errorf("verification evidence %s does not match declared target", key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// VerifyTargetResult is retained for callers that only need declaration
|
||||
// validation. Result-provided provenance is intentionally not evidence.
|
||||
func VerifyTargetResult(def Definition, _ Values) error {
|
||||
target, err := VerificationTargetFor(def)
|
||||
if err != nil || target == nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("verification target %q requires a server-controlled fixture runner", target.Fixture)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package stage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTargetVerificationRejectsWrongBackendEvidence(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Blocks["verification"] = "backend=branch-api\nrevision=abc123\ncapability=controlled-state\nfixture=reconnect-foreground"
|
||||
if err := (ModelStage{}).Validate(def); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target, err := VerificationTargetFor(def)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
good := VerificationEvidence{Backend: "branch-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}
|
||||
if err := VerifyTargetEvidence(*target, good); err != nil {
|
||||
t.Fatalf("matching fixture evidence rejected: %v", err)
|
||||
}
|
||||
wrongBackend := VerificationEvidence{Backend: "old-api", Revision: "abc123", Capability: "controlled-state", Fixture: "reconnect-foreground"}
|
||||
if err := VerifyTargetEvidence(*target, wrongBackend); err == nil {
|
||||
t.Fatal("wrong-backend fixture evidence accepted")
|
||||
}
|
||||
def.Blocks["verification"] = "backend=branch-api\nrevision=abc123"
|
||||
if err := (ModelStage{}).Validate(def); err == nil {
|
||||
t.Fatal("incomplete target verification declaration accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user