Files
alan 57f5219628 Artifacts: runner-internal turn files move to a Debugging files group
Each recorded artifact now carries a server group (deliverable or debug).
public/ files are deliverables; input/memory/history/output.md, session
exports and private/ scratch are debugging files. The app's Artifacts tab
lists deliverables and the Debug toggle reveals a separate Debugging files
group (older servers fall back to the filename rule); the TUI lists
deliverables first and marks debug rows; artifact search excludes debug
files unless include_debug / --debug-files is set.
2026-11-04 17:43:47 +00:00

461 lines
17 KiB
Go

package server
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"sync/atomic"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func artifactNamed(files []protocol.StageArtifact, name string) (protocol.StageArtifact, bool) {
for _, file := range files {
if file.Name == name {
return file, true
}
}
return protocol.StageArtifact{}, false
}
func TestSessionEvidenceIsInventoriedBeforeLargeCaptureDirectories(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "1", "stages", "01-test")
if err := os.MkdirAll(filepath.Join(dir, "private"), 0700); err != nil {
t.Fatal(err)
}
for i := 0; i < maxArtifactFiles+10; i++ {
if err := os.WriteFile(filepath.Join(dir, "private", fmt.Sprintf("capture-%03d.txt", i)), []byte("fixture"), 0600); err != nil {
t.Fatal(err)
}
}
for _, name := range []string{"input.md", "memory.md", "output.md", "session.html"} {
if err := os.WriteFile(filepath.Join(dir, name), []byte("runtime evidence"), 0600); err != nil {
t.Fatal(err)
}
}
r := &runner{artifactsRoot: root}
files, skipped, _ := r.scanArtifacts(1, 1, dir)
if len(files) > maxArtifactFiles || skipped == 0 {
t.Fatalf("inventory bound lost: %d %d", len(files), skipped)
}
for _, name := range []string{"input.md", "memory.md", "output.md", "session.html"} {
if _, ok := artifactNamed(files, name); !ok {
t.Fatalf("mandatory runtime file %s hidden by user captures", name)
}
}
// The recorded inventory carries the server's classification: turn files and
// private captures are debugging files.
for _, file := range files {
if file.Group != protocol.ArtifactGroupDebug {
t.Fatalf("%s group %q, want debug", file.Name, file.Group)
}
}
}
func TestArtifactsScanRecordsSizesTypesAndSkips(t *testing.T) {
root := t.TempDir()
stepDir := filepath.Join(root, "4", "2")
if err := os.MkdirAll(filepath.Join(stepDir, "reports"), 0700); err != nil {
t.Fatal(err)
}
png := []byte("\x89PNG\r\n\x1a\n...png bytes...")
if err := os.WriteFile(filepath.Join(stepDir, "shot.png"), png, 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(stepDir, "notes"), []byte("hello artifacts"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(stepDir, "reports", "out.txt"), []byte("report"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(stepDir, "big.bin"), make([]byte, maxArtifactFileBytes+1), 0600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join(stepDir, "shot.png"), filepath.Join(stepDir, "alias.png")); err != nil {
t.Fatal(err)
}
r := &runner{artifactsRoot: root}
files, skipped, truncated := r.scanArtifacts(4, 2, stepDir)
if truncated != 0 {
t.Fatalf("unexpected truncation: %d", truncated)
}
if skipped < 2 {
t.Fatalf("oversized file and symlink must be skipped: %d", skipped)
}
shot, ok := artifactNamed(files, "shot.png")
if !ok || shot.SizeBytes != int64(len(png)) || shot.ContentType != "image/png" || shot.URL != "/artifacts/4/2/shot.png" {
t.Fatalf("png artifact wrong: %+v", shot)
}
notes, ok := artifactNamed(files, "notes")
if !ok || !strings.HasPrefix(notes.ContentType, "text/plain") {
t.Fatalf("extensionless artifact not sniffed: %+v", notes)
}
if nested, ok := artifactNamed(files, "reports/out.txt"); !ok || nested.SizeBytes != int64(len("report")) {
t.Fatalf("nested artifact missing: %+v", files)
}
if _, ok := artifactNamed(files, "alias.png"); ok {
t.Fatal("symlink must not be recorded")
}
if _, ok := artifactNamed(files, "big.bin"); ok {
t.Fatal("oversized artifact must not be recorded")
}
}
func TestArtifactsScanBoundsFileCount(t *testing.T) {
root := t.TempDir()
stepDir := filepath.Join(root, "1", "1")
if err := os.MkdirAll(stepDir, 0700); err != nil {
t.Fatal(err)
}
for i := 0; i < maxArtifactFiles+5; i++ {
name := filepath.Join(stepDir, fmt.Sprintf("f%04d", i))
if err := os.WriteFile(name, []byte("x"), 0600); err != nil {
t.Fatal(err)
}
}
r := &runner{artifactsRoot: root}
files, skipped, _ := r.scanArtifacts(1, 1, stepDir)
if len(files) != maxArtifactFiles {
t.Fatalf("file cap not enforced: %d", len(files))
}
if skipped == 0 {
t.Fatal("over-cap files were not counted as skipped")
}
}
func TestArtifactsScanMissingDirectoryIsEmpty(t *testing.T) {
r := &runner{artifactsRoot: t.TempDir()}
files, skipped, truncated := r.scanArtifacts(1, 1, filepath.Join(t.TempDir(), "absent"))
if len(files) != 0 || skipped != 0 || truncated != 0 {
t.Fatalf("missing directory must be empty: %+v %d %d", files, skipped, truncated)
}
}
func TestArtifactAccessUsesOnlyCurrentAndEarlierVisits(t *testing.T) {
root := t.TempDir()
r := &runner{artifactsRoot: root}
for _, directory := range []string{"7/stages/01-a", "7/stages/02-b/private", "7/stages/03-c", "7/public", "8/stages/01-a"} {
if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil {
t.Fatal(err)
}
}
access := r.artifactAccess(7, 2, []string{"1", "3", "../8/1"}, filepath.Join(root, "7", "stages", "02-b"))
current, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "stages", "02-b"))
private, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "stages", "02-b", "private"))
public, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "public"))
if access.Write != private || access.WriteShared != public || len(access.Read) != 2 || access.Read[0] != current || access.Read[1] != public {
t.Fatalf("artifact scope = %#v", access)
}
if access.Read[1] == filepath.Join(root, "7", "stages", "01-a") {
t.Fatal("a prior turn's private directory was opened to this visit")
}
if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "8", "stages", "01-a")); got.Write != "" || len(got.Read) != 0 {
t.Fatalf("other-run current scope accepted: %#v", got)
}
if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "7", "stages", "03-c")); got.Write != "" || len(got.Read) != 0 {
t.Fatalf("sibling current scope accepted: %#v", got)
}
if err := os.WriteFile(filepath.Join(root, "7", "stages", "04-d"), []byte("not a directory"), 0600); err != nil {
t.Fatal(err)
}
if got := r.artifactAccess(7, 4, nil, filepath.Join(root, "7", "stages", "04-d")); got.Write != "" || len(got.Read) != 0 {
t.Fatalf("file current scope accepted: %#v", got)
}
if err := os.Symlink(filepath.Join(root, "7", "stages", "02-b"), filepath.Join(root, "7", "stages", "05-e")); err != nil {
t.Fatal(err)
}
if got := r.artifactAccess(7, 5, nil, filepath.Join(root, "7", "stages", "05-e")); got.Write != "" || len(got.Read) != 0 {
t.Fatalf("symlinked current scope accepted: %#v", got)
}
}
func TestRunnerPassesDerivedArtifactAccessToAgentClient(t *testing.T) {
root := t.TempDir()
for _, directory := range []string{"7/stages/01-a/private", "7/stages/02-b/private", "7/public"} {
if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil {
t.Fatal(err)
}
}
r := &runner{artifactsRoot: root}
current := filepath.Join(root, "7", "stages", "02-b")
client := r.agentClient(7, 2, []string{"1", "3"}, t.TempDir(), current, &stage.OperationLog{}, nil, []string{"read", "edit"})
if client.ArtifactAccess.Write == "" || client.ArtifactAccess.WriteShared == "" || len(client.ArtifactAccess.Read) != 2 {
t.Fatalf("runner did not pass scoped artifact access: %#v", client.ArtifactAccess)
}
if client.ArtifactAccess.Read[1] == filepath.Join(root, "7", "stages", "01-a") {
t.Fatal("runner passed a prior turn's private directory to the client")
}
piClient := r.piClient(7, 2, []string{"1", "3"}, "pi", t.TempDir(), current, nil, []string{"read", "edit"})
if !reflect.DeepEqual(piClient.ArtifactAccess, client.ArtifactAccess) {
t.Fatalf("runner passed different artifact scope to Pi client: %#v", piClient.ArtifactAccess)
}
}
// runWithArtifact executes one real run whose stage writes a file into the
// runner-provided artifacts directory. It returns the live handler and the
// recorded step so tests can serve the artifact over the real HTTP endpoint.
func runWithArtifact(t *testing.T) (*runHandler, protocol.RunStep) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
store, err := pipeline.Open(filepath.Join(t.TempDir(), "artifacts.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { store.Close() })
if _, err := store.Submit(ctx, "artifacts", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: t.TempDir()}, store).(*runHandler)
t.Cleanup(func() { cancel(); WaitRuns(handler) })
handler.runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Turn.Private == "" || def.Turn.Public == "" {
t.Error("runner did not provide the turn artifact scopes")
return stage.Result{Outcome: "success"}, nil
}
if err := os.WriteFile(filepath.Join(def.Turn.Private, "report.txt"), []byte("artifact bytes"), 0600); err != nil {
return stage.Result{}, err
}
if err := os.WriteFile(filepath.Join(def.Turn.Public, "shared.txt"), []byte("shared bytes"), 0600); err != nil {
return stage.Result{}, err
}
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
}
dir := t.TempDir()
run, err := handler.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "artifact", Workspace: dir, Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
handler.runner.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 1 {
t.Fatalf("unexpected run: %+v", detail)
}
step := detail.Steps[0]
for _, name := range []string{"input.md", "memory.md", "output.md"} {
if _, ok := artifactNamed(step.Artifacts, name); !ok {
t.Fatalf("turn file %s not listed as an artifact: %+v", name, step.Artifacts)
}
}
artifact, ok := artifactNamed(step.Artifacts, "private/report.txt")
if !ok {
t.Fatalf("artifact not retained: %+v", step.Artifacts)
}
shared, ok := artifactNamed(step.Artifacts, "public/shared.txt")
if !ok {
t.Fatalf("shared artifact not retained: %+v", step.Artifacts)
}
step.Artifacts = []protocol.StageArtifact{artifact, shared}
if artifact.SizeBytes != int64(len("artifact bytes")) ||
!strings.HasPrefix(artifact.ContentType, "text/plain") || artifact.URL == "" {
t.Fatalf("recorded artifact wrong: %+v", artifact)
}
if shared.SizeBytes != int64(len("shared bytes")) || shared.URL == "" {
t.Fatalf("recorded shared artifact wrong: %+v", shared)
}
return handler, step
}
func TestArtifactHTTPGetHeadRangeAndRefusals(t *testing.T) {
handler, step := runWithArtifact(t)
srv := httptest.NewServer(handler)
defer srv.Close()
base := srv.URL + step.Artifacts[0].URL
payload := []byte("artifact bytes")
resp, err := http.Get(base)
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusOK || !bytes.Equal(body, payload) {
t.Fatalf("GET wrong: %d %q", resp.StatusCode, body)
}
if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
t.Fatalf("GET content type wrong: %q", ct)
}
if cl := resp.Header.Get("Content-Length"); cl != fmt.Sprint(len(payload)) {
t.Fatalf("GET content length wrong: %q", cl)
}
for _, directive := range []string{"sandbox;", "script-src 'none'", "connect-src 'none'", "form-action 'none'", "frame-ancestors 'self'"} {
if !strings.Contains(resp.Header.Get("Content-Security-Policy"), directive) {
t.Fatalf("artifact lacks response isolation %q: %v", directive, resp.Header)
}
}
if resp.Header.Get("X-Content-Type-Options") != "nosniff" || resp.Header.Get("Cache-Control") != "private, no-store" {
t.Fatalf("artifact content/cache policy = %v", resp.Header)
}
head, err := http.Head(base)
if err != nil {
t.Fatal(err)
}
headBody, _ := io.ReadAll(head.Body)
head.Body.Close()
if head.StatusCode != http.StatusOK || len(headBody) != 0 {
t.Fatalf("HEAD wrong: %d %q", head.StatusCode, headBody)
}
if head.Header.Get("Content-Length") != fmt.Sprint(len(payload)) {
t.Fatalf("HEAD lost content length: %q", head.Header.Get("Content-Length"))
}
rangeReq, _ := http.NewRequest(http.MethodGet, base, nil)
rangeReq.Header.Set("Range", "bytes=0-3")
rangeResp, err := http.DefaultClient.Do(rangeReq)
if err != nil {
t.Fatal(err)
}
rangeBody, _ := io.ReadAll(rangeResp.Body)
rangeResp.Body.Close()
if rangeResp.StatusCode != http.StatusPartialContent || !bytes.Equal(rangeBody, payload[:4]) {
t.Fatalf("Range wrong: %d %q", rangeResp.StatusCode, rangeBody)
}
if got := rangeResp.Header.Get("Content-Range"); got != fmt.Sprintf("bytes 0-3/%d", len(payload)) {
t.Fatalf("Content-Range wrong: %q", got)
}
unsatReq, _ := http.NewRequest(http.MethodGet, base, nil)
unsatReq.Header.Set("Range", "bytes=9999-10000")
unsatResp, err := http.DefaultClient.Do(unsatReq)
if err != nil {
t.Fatal(err)
}
unsatResp.Body.Close()
if unsatResp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
t.Fatalf("unsatisfiable Range wrong: %d", unsatResp.StatusCode)
}
// Unknown run and unknown step must not serve bytes.
for _, path := range []string{"/artifacts/999999/1/report.txt", fmt.Sprintf("/artifacts/%d/99/report.txt", runIDFromURL(step.Artifacts[0].URL))} {
resp, err := http.Get(srv.URL + path)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("%s expected 404, got %d", path, resp.StatusCode)
}
}
}
// Opt-in real browser regression. The server, database, run and malicious file
// are all test fixtures; no model or deployed service is used.
func TestArtifactHTMLBrowserIsolation(t *testing.T) {
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
}
handler, step := runWithArtifact(t)
id := runIDFromURL(step.Artifacts[0].URL)
directory := filepath.Join(handler.runner.artifactsRoot, fmt.Sprint(id), "stages", fmt.Sprintf("%02d-%s", step.Sequence, step.Stage))
attack := `<!doctype html><title>Safe report</title><style>#report {color:rgb(12,34,56)}</style><h1 id="report">Readable report</h1><script>document.title='executed';fetch('/attack')</script>`
if err := os.WriteFile(filepath.Join(directory, "attack.html"), []byte(attack), 0600); err != nil {
t.Fatal(err)
}
var requests atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/attack" {
requests.Add(1)
w.WriteHeader(http.StatusNoContent)
return
}
handler.ServeHTTP(w, r)
}))
defer srv.Close()
cmd := exec.Command("node", "../../web/test/artifact-security-probe.mjs", srv.URL+artifactURL(id, step.Sequence, "attack.html"))
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("browser isolation: %v\n%s", err, output)
} else {
t.Log(string(output))
}
if requests.Load() != 0 {
t.Fatal("artifact script reached the control origin")
}
}
func TestArtifactHTTPGetPublicSharedFile(t *testing.T) {
handler, step := runWithArtifact(t)
var shared protocol.StageArtifact
for _, artifact := range step.Artifacts {
if artifact.Name == "public/shared.txt" {
shared = artifact
}
}
if shared.URL == "" {
t.Fatalf("public artifact missing: %+v", step.Artifacts)
}
if shared.Group != protocol.ArtifactGroupDeliverable {
t.Fatalf("public artifact group %q, want deliverable", shared.Group)
}
srv := httptest.NewServer(handler)
defer srv.Close()
resp, err := http.Get(srv.URL + shared.URL)
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusOK || string(body) != "shared bytes" {
t.Fatalf("public artifact GET = %d %q", resp.StatusCode, body)
}
}
func runIDFromURL(url string) int64 {
var id int64
fmt.Sscanf(url, "/artifacts/%d/", &id)
return id
}
func TestArtifactHTTPRefusesEscapes(t *testing.T) {
handler, step := runWithArtifact(t)
runID := runIDFromURL(step.Artifacts[0].URL)
root := handler.runner.artifactsRoot
// A symlink inside the step directory must never be followed.
stepDir := filepath.Join(root, fmt.Sprint(runID), "stages", "01-one")
secret := filepath.Join(t.TempDir(), "secret.txt")
if err := os.WriteFile(secret, []byte("secret"), 0600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(secret, filepath.Join(stepDir, "escape.txt")); err != nil {
t.Fatal(err)
}
srv := httptest.NewServer(handler)
defer srv.Close()
for _, path := range []string{
fmt.Sprintf("/artifacts/%d/1/escape.txt", runID),
fmt.Sprintf("/artifacts/%d/1/%%2e%%2e/%%2e%%2e/secret.txt", runID),
fmt.Sprintf("/artifacts/%d/1/%%2Fetc%%2Fpasswd", runID),
} {
resp, err := http.Get(srv.URL + path)
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode == http.StatusOK || bytes.Contains(body, []byte("secret")) {
t.Fatalf("%s served forbidden bytes: %d %q", path, resp.StatusCode, body)
}
}
}