Guard chat bootstrap transitions
Use conditional durable phase transitions and affected-row checks so only the request that reserves each phase can create or activate a manager chat session. Record duplicate ownership conflicts without repeating external work, and cover the reservation and restart state in pipeline and server tests. Checked packaged templates and authoring skill references: the existing task commit already packages the dedicated manager skill and installer inventory; this transition-race fix requires no further template or reference update. Shared documentation remains untouched; the task note is docs/notes/manager-skill-bootstrap.md.
This commit is contained in:
@@ -43,6 +43,10 @@ type ChatCreateBootstrap struct {
|
||||
State string
|
||||
}
|
||||
|
||||
// ErrChatCreateConflict reports that another request owns the durable bootstrap
|
||||
// transition, or that its expected phase no longer matches.
|
||||
var ErrChatCreateConflict = errors.New("chat_create bootstrap ownership/state conflict")
|
||||
|
||||
// initOperatorSessions creates the operator-session registry. It is a table of
|
||||
// its own rather than a kind column on run_steps so a pipeline run's session
|
||||
// ownership (run_session_owners, SessionOwner) is untouched and the two
|
||||
@@ -90,13 +94,35 @@ VALUES(?,?,?,?,?,?) ON CONFLICT(request_id) DO NOTHING`, requestID, title, direc
|
||||
}
|
||||
|
||||
func (s *Store) SetChatCreateSession(ctx context.Context, requestID, sessionID string) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE chat_create_bootstraps SET session_id=?,updated_at=? WHERE request_id=? AND session_id=''`, sessionID, time.Now().UnixMilli(), requestID)
|
||||
return err
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE chat_create_bootstraps SET session_id=?,updated_at=? WHERE request_id=? AND state='creating' AND session_id=''`, sessionID, time.Now().UnixMilli(), requestID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if changed != 1 {
|
||||
return ErrChatCreateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) AdvanceChatCreate(ctx context.Context, requestID, state string) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE chat_create_bootstraps SET state=?,updated_at=? WHERE request_id=?`, state, time.Now().UnixMilli(), requestID)
|
||||
return err
|
||||
// AdvanceChatCreate moves a bootstrap exactly one expected phase. Callers must
|
||||
// reserve external work through this conditional update before performing it.
|
||||
func (s *Store) AdvanceChatCreate(ctx context.Context, requestID, from, to string) error {
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE chat_create_bootstraps SET state=?,updated_at=? WHERE request_id=? AND state=?`, to, time.Now().UnixMilli(), requestID, from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if changed != 1 {
|
||||
return ErrChatCreateConflict
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateOperatorSessions adds the closed_at column to a database created before
|
||||
|
||||
@@ -2,11 +2,52 @@ package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestChatCreateBootstrapTransitionsAreOwnedAndDurable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "operator.db")
|
||||
store, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bootstrap, err := store.BeginChatCreate(ctx, "request", "title", "/work")
|
||||
if err != nil || bootstrap.State != "created" {
|
||||
t.Fatalf("BeginChatCreate = %+v, %v", bootstrap, err)
|
||||
}
|
||||
if err := store.AdvanceChatCreate(ctx, "request", "created", "creating"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.AdvanceChatCreate(ctx, "request", "created", "creating"); !errors.Is(err, ErrChatCreateConflict) {
|
||||
t.Fatalf("second reservation error = %v, want ownership conflict", err)
|
||||
}
|
||||
if err := store.SetChatCreateSession(ctx, "request", "ses_one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.SetChatCreateSession(ctx, "request", "ses_two"); !errors.Is(err, ErrChatCreateConflict) {
|
||||
t.Fatalf("second session owner error = %v, want ownership conflict", err)
|
||||
}
|
||||
if err := store.AdvanceChatCreate(ctx, "request", "creating", "activated"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reopened.Close()
|
||||
bootstrap, err = reopened.BeginChatCreate(ctx, "request", "title", "/work")
|
||||
if err != nil || bootstrap.SessionID != "ses_one" || bootstrap.State != "activated" {
|
||||
t.Fatalf("reopened bootstrap = %+v, %v", bootstrap, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOperatorSessionsRecordListAndUpsert proves the operator-session registry
|
||||
// contract: rows persist the id, directory, title and tag; the list is
|
||||
// most-recently-used first; and a second RecordOperatorSession for the same id
|
||||
|
||||
@@ -56,7 +56,7 @@ type chatShellReleaser interface {
|
||||
type operatorRegistry interface {
|
||||
BeginChatCreate(ctx context.Context, requestID, title, directory string) (pipeline.ChatCreateBootstrap, error)
|
||||
SetChatCreateSession(ctx context.Context, requestID, sessionID string) error
|
||||
AdvanceChatCreate(ctx context.Context, requestID, state string) error
|
||||
AdvanceChatCreate(ctx context.Context, requestID, from, to string) error
|
||||
RecordOperatorSession(ctx context.Context, sessionID, directory, title, tag string) error
|
||||
TouchOperatorSession(ctx context.Context, sessionID string) error
|
||||
OperatorSessions(ctx context.Context) ([]pipeline.OperatorSession, error)
|
||||
|
||||
@@ -78,8 +78,8 @@ func (c *channelChat) Sessions(context.Context) ([]protocol.ChatSession, error)
|
||||
func (c *channelChat) BeginChatCreate(context.Context, string, string, string) (pipeline.ChatCreateBootstrap, error) {
|
||||
return pipeline.ChatCreateBootstrap{State: "created"}, nil
|
||||
}
|
||||
func (c *channelChat) SetChatCreateSession(context.Context, string, string) error { return nil }
|
||||
func (c *channelChat) AdvanceChatCreate(context.Context, string, string) error { return nil }
|
||||
func (c *channelChat) SetChatCreateSession(context.Context, string, string) error { return nil }
|
||||
func (c *channelChat) AdvanceChatCreate(context.Context, string, string, string) error { return nil }
|
||||
|
||||
func (c *channelChat) ActivateSkill(context.Context, string, string) error { return nil }
|
||||
|
||||
|
||||
@@ -37,10 +37,6 @@ func handleChatSessions(ctx context.Context, request protocol.Message, broker ch
|
||||
_ = write(response)
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(request.ID) == "" {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat_create requires a non-empty request id for retry-safe creation"})
|
||||
return
|
||||
}
|
||||
operatorSessions, err := registry.OperatorSessions(ctx)
|
||||
if err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
||||
@@ -104,6 +100,10 @@ func handleChatCreate(ctx context.Context, request protocol.Message, broker chat
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "operator session registry is not configured on this server"})
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(request.ID) == "" {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat_create requires a non-empty request id for retry-safe creation"})
|
||||
return
|
||||
}
|
||||
title := ""
|
||||
directory := ""
|
||||
if request.ChatCreate != nil {
|
||||
@@ -125,6 +125,16 @@ func handleChatCreate(ctx context.Context, request protocol.Message, broker chat
|
||||
}
|
||||
session := protocol.ChatSession{ID: bootstrap.SessionID, Title: title, Directory: resolved}
|
||||
if session.ID == "" {
|
||||
if bootstrap.State != "created" {
|
||||
chatCreateBootstrapError(request, session.ID, bootstrap.State, write)
|
||||
return
|
||||
}
|
||||
// The conditional transition is the durable reservation for the external
|
||||
// Create call. A duplicate request that lost it must not create an orphan.
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "created", "creating"); err != nil {
|
||||
chatCreateConflictError(request, err, write)
|
||||
return
|
||||
}
|
||||
session, err = broker.Create(ctx, title, resolved)
|
||||
if err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: err.Error()})
|
||||
@@ -134,37 +144,50 @@ func handleChatCreate(ctx context.Context, request protocol.Message, broker chat
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + session.ID + " was created but durable bootstrap state could not be saved: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "creating", "created"); err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + session.ID + " was created but durable bootstrap state could not advance: " + err.Error()})
|
||||
return
|
||||
}
|
||||
bootstrap.State = "created"
|
||||
}
|
||||
if bootstrap.State == "created" {
|
||||
// Reserve the activation before calling the external service. If the
|
||||
// process dies after the call but before the success checkpoint, a retry
|
||||
// must report the ambiguous operation rather than append the skill twice.
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "activating"); err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + session.ID + " could not reserve manager bootstrap: " + err.Error()})
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "created", "activating"); err != nil {
|
||||
chatCreateConflictError(request, err, write)
|
||||
return
|
||||
}
|
||||
if err := broker.ActivateSkill(ctx, session.ID, "pipelines-manager"); err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + session.ID + " was created but manager bootstrap failed while activating pipelines-manager: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "activated"); err != nil {
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "activating", "activated"); err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + session.ID + " activated but durable bootstrap state could not be saved: " + err.Error()})
|
||||
return
|
||||
}
|
||||
bootstrap.State = "activated"
|
||||
}
|
||||
if bootstrap.State == "activating" {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + session.ID + " has an unfinished manager bootstrap; activation will not be repeated automatically"})
|
||||
if bootstrap.State == "creating" || bootstrap.State == "activating" || bootstrap.State == "recording" {
|
||||
chatCreateBootstrapError(request, session.ID, bootstrap.State, write)
|
||||
return
|
||||
}
|
||||
if bootstrap.State != "recorded" {
|
||||
if bootstrap.State == "activated" {
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "activated", "recording"); err != nil {
|
||||
chatCreateConflictError(request, err, write)
|
||||
return
|
||||
}
|
||||
if err := registry.RecordOperatorSession(ctx, session.ID, resolved, title, protocol.ChatSessionTagOperator); err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + session.ID + " was created but could not be recorded: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "recorded"); err != nil {
|
||||
if err := registry.AdvanceChatCreate(ctx, request.ID, "recording", "recorded"); err != nil {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + session.ID + " was recorded but durable bootstrap state could not be saved: " + err.Error()})
|
||||
return
|
||||
}
|
||||
} else if bootstrap.State != "recorded" {
|
||||
chatCreateBootstrapError(request, session.ID, bootstrap.State, write)
|
||||
return
|
||||
}
|
||||
session.Directory = resolved
|
||||
session.Tag = protocol.ChatSessionTagOperator
|
||||
@@ -175,6 +198,14 @@ func handleChatCreate(ctx context.Context, request protocol.Message, broker chat
|
||||
_ = write(response)
|
||||
}
|
||||
|
||||
func chatCreateConflictError(request protocol.Message, err error, write func(protocol.Message) error) {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "chat_create bootstrap ownership/state conflict: " + err.Error()})
|
||||
}
|
||||
|
||||
func chatCreateBootstrapError(request protocol.Message, sessionID, state string, write func(protocol.Message) error) {
|
||||
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "session " + sessionID + " has an unfinished manager bootstrap (" + state + "); it will not be repeated automatically"})
|
||||
}
|
||||
|
||||
// handleChatAdopt records a session that already exists on the endpoint in the
|
||||
// operator registry, using the endpoint's own directory and title verbatim, so
|
||||
// it appears in chat_sessions and can be continued. It refuses a nil broker, a
|
||||
|
||||
@@ -76,6 +76,9 @@ type fakeChat struct {
|
||||
createDirectory string
|
||||
created protocol.ChatSession
|
||||
createErr error
|
||||
createGate <-chan struct{}
|
||||
createStarted chan struct{}
|
||||
createOnce sync.Once
|
||||
activateCalls int
|
||||
activateSessionID string
|
||||
activateSkill string
|
||||
@@ -90,8 +93,8 @@ type fakeChat struct {
|
||||
func (f *fakeChat) BeginChatCreate(context.Context, string, string, string) (pipeline.ChatCreateBootstrap, error) {
|
||||
return pipeline.ChatCreateBootstrap{State: "created"}, nil
|
||||
}
|
||||
func (f *fakeChat) SetChatCreateSession(context.Context, string, string) error { return nil }
|
||||
func (f *fakeChat) AdvanceChatCreate(context.Context, string, string) error { return nil }
|
||||
func (f *fakeChat) SetChatCreateSession(context.Context, string, string) error { return nil }
|
||||
func (f *fakeChat) AdvanceChatCreate(context.Context, string, string, string) error { return nil }
|
||||
|
||||
func (f *fakeChat) Prompt(context.Context, string, string) error {
|
||||
f.mu.Lock()
|
||||
@@ -268,10 +271,19 @@ func (f *fakeChat) ActivateSkill(_ context.Context, sessionID, skill string) err
|
||||
|
||||
func (f *fakeChat) Create(_ context.Context, title, directory string) (protocol.ChatSession, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.createCalls++
|
||||
f.createTitle = title
|
||||
f.createDirectory = directory
|
||||
gate, started := f.createGate, f.createStarted
|
||||
f.mu.Unlock()
|
||||
if started != nil {
|
||||
f.createOnce.Do(func() { close(started) })
|
||||
}
|
||||
if gate != nil {
|
||||
<-gate
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.createErr != nil {
|
||||
return protocol.ChatSession{}, f.createErr
|
||||
}
|
||||
@@ -1388,6 +1400,91 @@ func TestChatCreateRetryReusesDurableBootstrap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatCreateDuplicateRequestReservesExternalWork proves the conditional
|
||||
// durable transition, rather than timing or the race detector, elects exactly
|
||||
// one request to call the external Create and ActivateSkill operations.
|
||||
func TestChatCreateDuplicateRequestReservesExternalWork(t *testing.T) {
|
||||
store := newOperatorStore(t)
|
||||
createGate := make(chan struct{})
|
||||
broker := &fakeChat{
|
||||
created: protocol.ChatSession{ID: "ses_once"},
|
||||
createGate: createGate,
|
||||
createStarted: make(chan struct{}),
|
||||
}
|
||||
request := protocol.Message{ID: "same-request", Type: protocol.ChatCreate, ChatCreate: &protocol.ChatCreateRequest{Title: "demo"}}
|
||||
responses := make(chan protocol.Message, 2)
|
||||
run := func() {
|
||||
handleChatCreate(context.Background(), request, broker, store, func(message protocol.Message) error {
|
||||
responses <- message
|
||||
return nil
|
||||
})
|
||||
}
|
||||
go run()
|
||||
select {
|
||||
case <-broker.createStarted:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("first request did not reserve Create")
|
||||
}
|
||||
go run()
|
||||
select {
|
||||
case response := <-responses:
|
||||
if response.Type != protocol.Error || !strings.Contains(response.Error, "unfinished manager bootstrap") {
|
||||
t.Fatalf("losing request response = %+v", response)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("duplicate request did not finish at its durable reservation")
|
||||
}
|
||||
close(createGate)
|
||||
select {
|
||||
case response := <-responses:
|
||||
if response.Type != protocol.ChatCreate || response.ChatSession == nil || response.ChatSession.ID != "ses_once" {
|
||||
t.Fatalf("winning request response = %+v", response)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("winning request did not finish")
|
||||
}
|
||||
broker.mu.Lock()
|
||||
creates, activations := broker.createCalls, broker.activateCalls
|
||||
broker.mu.Unlock()
|
||||
if creates != 1 || activations != 1 {
|
||||
t.Fatalf("duplicate request repeated external work: creates=%d activations=%d", creates, activations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatCreateRequiresRequestID(t *testing.T) {
|
||||
broker := &fakeChat{created: protocol.ChatSession{ID: "ses_new"}}
|
||||
var response protocol.Message
|
||||
handleChatCreate(context.Background(), protocol.Message{Type: protocol.ChatCreate, ChatCreate: &protocol.ChatCreateRequest{Title: "demo"}}, broker, newOperatorStore(t), func(message protocol.Message) error {
|
||||
response = message
|
||||
return nil
|
||||
})
|
||||
if response.Type != protocol.Error || !strings.Contains(response.Error, "non-empty request id") {
|
||||
t.Fatalf("empty chat_create response = %+v", response)
|
||||
}
|
||||
broker.mu.Lock()
|
||||
creates := broker.createCalls
|
||||
broker.mu.Unlock()
|
||||
if creates != 0 {
|
||||
t.Fatalf("empty chat_create called broker.Create %d times", creates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatSessionsAllowsEmptyRequestID(t *testing.T) {
|
||||
store := newOperatorStore(t)
|
||||
if err := store.RecordOperatorSession(context.Background(), "ses_existing", "/work", "Existing", protocol.ChatSessionTagOperator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
broker := &fakeChat{sessions: []protocol.ChatSession{{ID: "ses_existing"}}}
|
||||
var response protocol.Message
|
||||
handleChatSessions(context.Background(), protocol.Message{Type: protocol.ChatSessions}, broker, nil, store, func(message protocol.Message) error {
|
||||
response = message
|
||||
return nil
|
||||
})
|
||||
if response.Type != protocol.ChatSessions || len(response.ChatSessions) != 1 || response.ChatSessions[0].ID != "ses_existing" {
|
||||
t.Fatalf("empty-ID chat_sessions response = %+v", response)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatCreateDirectoryResolution pins directory defaulting and rejection: an
|
||||
// omitted/empty/"~"/"~/..." directory resolves to the operator's home (never the
|
||||
// server's working directory), an explicit existing directory is recorded
|
||||
|
||||
Reference in New Issue
Block a user