test: strengthen background shell output verification
This commit is contained in:
@@ -77,49 +77,81 @@ func main() {
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
if err := verifyShellReadback(ctx, endpoint, *exited); err != nil {
|
||||
shellReadbacks, err := verifyShellReadback(ctx, endpoint, *exited)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
<-ctx.Done()
|
||||
shutdown, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
pipelinesErr := pipelinesServer.Shutdown(shutdown)
|
||||
upstreamErr := upstreamServer.Shutdown(shutdown)
|
||||
stop()
|
||||
if pipelinesErr != nil {
|
||||
fatal(pipelinesErr)
|
||||
}
|
||||
if upstreamErr != nil {
|
||||
fatal(upstreamErr)
|
||||
}
|
||||
info := buildinfo.Current()
|
||||
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{
|
||||
"endpoint": endpoint, "upstream_endpoint": upstreamURL, "backend": attached.Connection.Backend,
|
||||
"revision": info.Commit, "session_id": server.LargeChatFixtureSessionID, "workspace": workspace,
|
||||
"newest_readback": readbacks[0], "older_readback": readbacks[1],
|
||||
"shell_readbacks": shellReadbacks, "clean_shutdown": true, "shutdown_exit_status": 0,
|
||||
})
|
||||
<-ctx.Done()
|
||||
shutdown, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer stop()
|
||||
_ = pipelinesServer.Shutdown(shutdown)
|
||||
_ = upstreamServer.Shutdown(shutdown)
|
||||
}
|
||||
|
||||
func verifyShellReadback(ctx context.Context, endpoint string, exited bool) error {
|
||||
func verifyShellReadback(ctx context.Context, endpoint string, exited bool) (map[string]any, error) {
|
||||
evidence := map[string]any{"test_id": "fixture-shell-owned-and-foreign-refusal", "negative": []string{}}
|
||||
ws, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(endpoint, "http")+"/connect", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return evidence, err
|
||||
}
|
||||
defer ws.Close(websocket.StatusNormalClosure, "fixture complete")
|
||||
if err := wsjson.Write(ctx, ws, protocol.Message{Type: protocol.ChatShells, ID: "fixture-shells", ChatQuery: &protocol.ChatQuery{SessionID: server.LargeChatFixtureSessionID}}); err != nil {
|
||||
return err
|
||||
return evidence, err
|
||||
}
|
||||
var listed protocol.Message
|
||||
if listed, err = readFixtureResponse(ctx, ws, protocol.ChatShells); err != nil {
|
||||
return err
|
||||
return evidence, err
|
||||
}
|
||||
if !exited && (len(listed.ChatShells) != 1 || listed.ChatShells[0].ID != server.LargeChatFixtureShellID) {
|
||||
return fmt.Errorf("fixture shell is not listed")
|
||||
return evidence, fmt.Errorf("fixture shell is not listed")
|
||||
}
|
||||
if err := wsjson.Write(ctx, ws, protocol.Message{Type: protocol.ChatShellOutput, ID: "fixture-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: server.LargeChatFixtureSessionID, ShellID: server.LargeChatFixtureShellID, Limit: 64 << 10}}); err != nil {
|
||||
return err
|
||||
return evidence, err
|
||||
}
|
||||
page, err := readFixtureResponse(ctx, ws, protocol.ChatShellOutput)
|
||||
if err != nil {
|
||||
return err
|
||||
return evidence, err
|
||||
}
|
||||
if page.ChatShellOutput == nil || page.ChatShellOutput.Output != server.LargeChatFixtureShellOutput || page.ChatShellOutput.Shell.Status != map[bool]string{true: "exited", false: "running"}[exited] {
|
||||
return fmt.Errorf("unexpected shell readback: %+v", page.ChatShellOutput)
|
||||
return evidence, fmt.Errorf("unexpected shell readback: %+v", page.ChatShellOutput)
|
||||
}
|
||||
return nil
|
||||
evidence["owned"] = map[string]any{"shell_id": page.ChatShellOutput.Shell.ID, "output": page.ChatShellOutput.Output, "cursor": page.ChatShellOutput.Cursor, "status": page.ChatShellOutput.Shell.Status}
|
||||
negative := evidence["negative"].([]string)
|
||||
negativeRequests := []protocol.Message{
|
||||
{Type: protocol.ChatShells, ID: "foreign-list", ChatQuery: &protocol.ChatQuery{SessionID: server.LargeChatFixtureForeignSessionID}},
|
||||
{Type: protocol.ChatShells, ID: "unknown-list", ChatQuery: &protocol.ChatQuery{SessionID: "ses_unregistered_fixture"}},
|
||||
{Type: protocol.ChatShellOutput, ID: "foreign-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: server.LargeChatFixtureSessionID, ShellID: server.LargeChatFixtureForeignShellID, Limit: 64 << 10}},
|
||||
{Type: protocol.ChatShellOutput, ID: "unknown-output", ChatShellOutputQuery: &protocol.ChatShellOutputQuery{SessionID: "ses_unregistered_fixture", ShellID: server.LargeChatFixtureShellID, Limit: 64 << 10}},
|
||||
}
|
||||
for _, request := range negativeRequests {
|
||||
if err := wsjson.Write(ctx, ws, request); err != nil {
|
||||
return evidence, err
|
||||
}
|
||||
var response protocol.Message
|
||||
if err := wsjson.Read(ctx, ws, &response); err != nil {
|
||||
return evidence, err
|
||||
}
|
||||
if response.Type != protocol.Error || strings.Contains(response.Error, server.LargeChatFixtureForeignShellOutput) {
|
||||
return evidence, fmt.Errorf("negative request %s leaked or succeeded: %+v", request.ID, response)
|
||||
}
|
||||
negative = append(negative, request.ID)
|
||||
}
|
||||
evidence["negative"] = negative
|
||||
evidence["negative_count"] = len(negative)
|
||||
return evidence, nil
|
||||
}
|
||||
|
||||
func verifyBroker(ctx context.Context, endpoint string) ([2]map[string]any, error) {
|
||||
|
||||
Reference in New Issue
Block a user