452 lines
17 KiB
Go
452 lines
17 KiB
Go
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
func artifactNamed(files []protocol.StageArtifact, name string) (protocol.StageArtifact, bool) {
|
|
for _, file := range files {
|
|
if file.Name == name {
|
|
return file, true
|
|
}
|
|
}
|
|
return protocol.StageArtifact{}, false
|
|
}
|
|
|
|
func TestSessionEvidenceIsInventoriedBeforeLargeCaptureDirectories(t *testing.T) {
|
|
root := t.TempDir()
|
|
dir := filepath.Join(root, "1", "stages", "01-test")
|
|
if err := os.MkdirAll(filepath.Join(dir, "private"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < maxArtifactFiles+10; i++ {
|
|
if err := os.WriteFile(filepath.Join(dir, "private", fmt.Sprintf("capture-%03d.txt", i)), []byte("fixture"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, name := range []string{"input.md", "memory.md", "output.md", "session.html"} {
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte("runtime evidence"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
r := &runner{artifactsRoot: root}
|
|
files, skipped, _ := r.scanArtifacts(1, 1, dir)
|
|
if len(files) > maxArtifactFiles || skipped == 0 {
|
|
t.Fatalf("inventory bound lost: %d %d", len(files), skipped)
|
|
}
|
|
for _, name := range []string{"input.md", "memory.md", "output.md", "session.html"} {
|
|
if _, ok := artifactNamed(files, name); !ok {
|
|
t.Fatalf("mandatory runtime file %s hidden by user captures", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestArtifactsScanRecordsSizesTypesAndSkips(t *testing.T) {
|
|
root := t.TempDir()
|
|
stepDir := filepath.Join(root, "4", "2")
|
|
if err := os.MkdirAll(filepath.Join(stepDir, "reports"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
png := []byte("\x89PNG\r\n\x1a\n...png bytes...")
|
|
if err := os.WriteFile(filepath.Join(stepDir, "shot.png"), png, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(stepDir, "notes"), []byte("hello artifacts"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(stepDir, "reports", "out.txt"), []byte("report"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(stepDir, "big.bin"), make([]byte, maxArtifactFileBytes+1), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(filepath.Join(stepDir, "shot.png"), filepath.Join(stepDir, "alias.png")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := &runner{artifactsRoot: root}
|
|
files, skipped, truncated := r.scanArtifacts(4, 2, stepDir)
|
|
if truncated != 0 {
|
|
t.Fatalf("unexpected truncation: %d", truncated)
|
|
}
|
|
if skipped < 2 {
|
|
t.Fatalf("oversized file and symlink must be skipped: %d", skipped)
|
|
}
|
|
shot, ok := artifactNamed(files, "shot.png")
|
|
if !ok || shot.SizeBytes != int64(len(png)) || shot.ContentType != "image/png" || shot.URL != "/artifacts/4/2/shot.png" {
|
|
t.Fatalf("png artifact wrong: %+v", shot)
|
|
}
|
|
notes, ok := artifactNamed(files, "notes")
|
|
if !ok || !strings.HasPrefix(notes.ContentType, "text/plain") {
|
|
t.Fatalf("extensionless artifact not sniffed: %+v", notes)
|
|
}
|
|
if nested, ok := artifactNamed(files, "reports/out.txt"); !ok || nested.SizeBytes != int64(len("report")) {
|
|
t.Fatalf("nested artifact missing: %+v", files)
|
|
}
|
|
if _, ok := artifactNamed(files, "alias.png"); ok {
|
|
t.Fatal("symlink must not be recorded")
|
|
}
|
|
if _, ok := artifactNamed(files, "big.bin"); ok {
|
|
t.Fatal("oversized artifact must not be recorded")
|
|
}
|
|
}
|
|
|
|
func TestArtifactsScanBoundsFileCount(t *testing.T) {
|
|
root := t.TempDir()
|
|
stepDir := filepath.Join(root, "1", "1")
|
|
if err := os.MkdirAll(stepDir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < maxArtifactFiles+5; i++ {
|
|
name := filepath.Join(stepDir, fmt.Sprintf("f%04d", i))
|
|
if err := os.WriteFile(name, []byte("x"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
r := &runner{artifactsRoot: root}
|
|
files, skipped, _ := r.scanArtifacts(1, 1, stepDir)
|
|
if len(files) != maxArtifactFiles {
|
|
t.Fatalf("file cap not enforced: %d", len(files))
|
|
}
|
|
if skipped == 0 {
|
|
t.Fatal("over-cap files were not counted as skipped")
|
|
}
|
|
}
|
|
|
|
func TestArtifactsScanMissingDirectoryIsEmpty(t *testing.T) {
|
|
r := &runner{artifactsRoot: t.TempDir()}
|
|
files, skipped, truncated := r.scanArtifacts(1, 1, filepath.Join(t.TempDir(), "absent"))
|
|
if len(files) != 0 || skipped != 0 || truncated != 0 {
|
|
t.Fatalf("missing directory must be empty: %+v %d %d", files, skipped, truncated)
|
|
}
|
|
}
|
|
|
|
func TestArtifactAccessUsesOnlyCurrentAndEarlierVisits(t *testing.T) {
|
|
root := t.TempDir()
|
|
r := &runner{artifactsRoot: root}
|
|
for _, directory := range []string{"7/stages/01-a", "7/stages/02-b/private", "7/stages/03-c", "7/public", "8/stages/01-a"} {
|
|
if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
access := r.artifactAccess(7, 2, []string{"1", "3", "../8/1"}, filepath.Join(root, "7", "stages", "02-b"))
|
|
current, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "stages", "02-b"))
|
|
private, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "stages", "02-b", "private"))
|
|
public, _ := filepath.EvalSymlinks(filepath.Join(root, "7", "public"))
|
|
if access.Write != private || access.WriteShared != public || len(access.Read) != 2 || access.Read[0] != current || access.Read[1] != public {
|
|
t.Fatalf("artifact scope = %#v", access)
|
|
}
|
|
if access.Read[1] == filepath.Join(root, "7", "stages", "01-a") {
|
|
t.Fatal("a prior turn's private directory was opened to this visit")
|
|
}
|
|
if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "8", "stages", "01-a")); got.Write != "" || len(got.Read) != 0 {
|
|
t.Fatalf("other-run current scope accepted: %#v", got)
|
|
}
|
|
if got := r.artifactAccess(7, 2, nil, filepath.Join(root, "7", "stages", "03-c")); got.Write != "" || len(got.Read) != 0 {
|
|
t.Fatalf("sibling current scope accepted: %#v", got)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(root, "7", "stages", "04-d"), []byte("not a directory"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := r.artifactAccess(7, 4, nil, filepath.Join(root, "7", "stages", "04-d")); got.Write != "" || len(got.Read) != 0 {
|
|
t.Fatalf("file current scope accepted: %#v", got)
|
|
}
|
|
if err := os.Symlink(filepath.Join(root, "7", "stages", "02-b"), filepath.Join(root, "7", "stages", "05-e")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := r.artifactAccess(7, 5, nil, filepath.Join(root, "7", "stages", "05-e")); got.Write != "" || len(got.Read) != 0 {
|
|
t.Fatalf("symlinked current scope accepted: %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestRunnerPassesDerivedArtifactAccessToAgentClient(t *testing.T) {
|
|
root := t.TempDir()
|
|
for _, directory := range []string{"7/stages/01-a/private", "7/stages/02-b/private", "7/public"} {
|
|
if err := os.MkdirAll(filepath.Join(root, directory), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
r := &runner{artifactsRoot: root}
|
|
current := filepath.Join(root, "7", "stages", "02-b")
|
|
client := r.agentClient(7, 2, []string{"1", "3"}, t.TempDir(), current, &stage.OperationLog{}, nil, []string{"read", "edit"})
|
|
if client.ArtifactAccess.Write == "" || client.ArtifactAccess.WriteShared == "" || len(client.ArtifactAccess.Read) != 2 {
|
|
t.Fatalf("runner did not pass scoped artifact access: %#v", client.ArtifactAccess)
|
|
}
|
|
if client.ArtifactAccess.Read[1] == filepath.Join(root, "7", "stages", "01-a") {
|
|
t.Fatal("runner passed a prior turn's private directory to the client")
|
|
}
|
|
piClient := r.piClient(7, 2, []string{"1", "3"}, "pi", t.TempDir(), current, nil, []string{"read", "edit"})
|
|
if !reflect.DeepEqual(piClient.ArtifactAccess, client.ArtifactAccess) {
|
|
t.Fatalf("runner passed different artifact scope to Pi client: %#v", piClient.ArtifactAccess)
|
|
}
|
|
}
|
|
|
|
// runWithArtifact executes one real run whose stage writes a file into the
|
|
// runner-provided artifacts directory. It returns the live handler and the
|
|
// recorded step so tests can serve the artifact over the real HTTP endpoint.
|
|
func runWithArtifact(t *testing.T) (*runHandler, protocol.RunStep) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "artifacts.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { store.Close() })
|
|
if _, err := store.Submit(ctx, "artifacts", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
handler := Handler(ctx, store).(*runHandler)
|
|
t.Cleanup(func() { cancel(); WaitRuns(handler) })
|
|
handler.runner.artifactsRoot = t.TempDir()
|
|
handler.runner.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
if def.Turn.Private == "" || def.Turn.Public == "" {
|
|
t.Error("runner did not provide the turn artifact scopes")
|
|
return stage.Result{Outcome: "success"}, nil
|
|
}
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Private, "report.txt"), []byte("artifact bytes"), 0600); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
if err := os.WriteFile(filepath.Join(def.Turn.Public, "shared.txt"), []byte("shared bytes"), 0600); err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
|
}
|
|
dir := t.TempDir()
|
|
run, err := handler.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "artifact", Workspace: dir, Input: map[string]any{"prompt": "task"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
handler.runner.wg.Wait()
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Status != "succeeded" || len(detail.Steps) != 1 {
|
|
t.Fatalf("unexpected run: %+v", detail)
|
|
}
|
|
step := detail.Steps[0]
|
|
for _, name := range []string{"input.md", "memory.md", "output.md"} {
|
|
if _, ok := artifactNamed(step.Artifacts, name); !ok {
|
|
t.Fatalf("turn file %s not listed as an artifact: %+v", name, step.Artifacts)
|
|
}
|
|
}
|
|
artifact, ok := artifactNamed(step.Artifacts, "private/report.txt")
|
|
if !ok {
|
|
t.Fatalf("artifact not retained: %+v", step.Artifacts)
|
|
}
|
|
shared, ok := artifactNamed(step.Artifacts, "public/shared.txt")
|
|
if !ok {
|
|
t.Fatalf("shared artifact not retained: %+v", step.Artifacts)
|
|
}
|
|
step.Artifacts = []protocol.StageArtifact{artifact, shared}
|
|
if artifact.SizeBytes != int64(len("artifact bytes")) ||
|
|
!strings.HasPrefix(artifact.ContentType, "text/plain") || artifact.URL == "" {
|
|
t.Fatalf("recorded artifact wrong: %+v", artifact)
|
|
}
|
|
if shared.SizeBytes != int64(len("shared bytes")) || shared.URL == "" {
|
|
t.Fatalf("recorded shared artifact wrong: %+v", shared)
|
|
}
|
|
return handler, step
|
|
}
|
|
|
|
func TestArtifactHTTPGetHeadRangeAndRefusals(t *testing.T) {
|
|
handler, step := runWithArtifact(t)
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
base := srv.URL + step.Artifacts[0].URL
|
|
payload := []byte("artifact bytes")
|
|
|
|
resp, err := http.Get(base)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK || !bytes.Equal(body, payload) {
|
|
t.Fatalf("GET wrong: %d %q", resp.StatusCode, body)
|
|
}
|
|
if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
|
|
t.Fatalf("GET content type wrong: %q", ct)
|
|
}
|
|
if cl := resp.Header.Get("Content-Length"); cl != fmt.Sprint(len(payload)) {
|
|
t.Fatalf("GET content length wrong: %q", cl)
|
|
}
|
|
for _, directive := range []string{"sandbox;", "script-src 'none'", "connect-src 'none'", "form-action 'none'", "frame-ancestors 'self'"} {
|
|
if !strings.Contains(resp.Header.Get("Content-Security-Policy"), directive) {
|
|
t.Fatalf("artifact lacks response isolation %q: %v", directive, resp.Header)
|
|
}
|
|
}
|
|
if resp.Header.Get("X-Content-Type-Options") != "nosniff" || resp.Header.Get("Cache-Control") != "private, no-store" {
|
|
t.Fatalf("artifact content/cache policy = %v", resp.Header)
|
|
}
|
|
|
|
head, err := http.Head(base)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
headBody, _ := io.ReadAll(head.Body)
|
|
head.Body.Close()
|
|
if head.StatusCode != http.StatusOK || len(headBody) != 0 {
|
|
t.Fatalf("HEAD wrong: %d %q", head.StatusCode, headBody)
|
|
}
|
|
if head.Header.Get("Content-Length") != fmt.Sprint(len(payload)) {
|
|
t.Fatalf("HEAD lost content length: %q", head.Header.Get("Content-Length"))
|
|
}
|
|
|
|
rangeReq, _ := http.NewRequest(http.MethodGet, base, nil)
|
|
rangeReq.Header.Set("Range", "bytes=0-3")
|
|
rangeResp, err := http.DefaultClient.Do(rangeReq)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rangeBody, _ := io.ReadAll(rangeResp.Body)
|
|
rangeResp.Body.Close()
|
|
if rangeResp.StatusCode != http.StatusPartialContent || !bytes.Equal(rangeBody, payload[:4]) {
|
|
t.Fatalf("Range wrong: %d %q", rangeResp.StatusCode, rangeBody)
|
|
}
|
|
if got := rangeResp.Header.Get("Content-Range"); got != fmt.Sprintf("bytes 0-3/%d", len(payload)) {
|
|
t.Fatalf("Content-Range wrong: %q", got)
|
|
}
|
|
|
|
unsatReq, _ := http.NewRequest(http.MethodGet, base, nil)
|
|
unsatReq.Header.Set("Range", "bytes=9999-10000")
|
|
unsatResp, err := http.DefaultClient.Do(unsatReq)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
unsatResp.Body.Close()
|
|
if unsatResp.StatusCode != http.StatusRequestedRangeNotSatisfiable {
|
|
t.Fatalf("unsatisfiable Range wrong: %d", unsatResp.StatusCode)
|
|
}
|
|
|
|
// Unknown run and unknown step must not serve bytes.
|
|
for _, path := range []string{"/artifacts/999999/1/report.txt", fmt.Sprintf("/artifacts/%d/99/report.txt", runIDFromURL(step.Artifacts[0].URL))} {
|
|
resp, err := http.Get(srv.URL + path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Fatalf("%s expected 404, got %d", path, resp.StatusCode)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Opt-in real browser regression. The server, database, run and malicious file
|
|
// are all test fixtures; no model or deployed service is used.
|
|
func TestArtifactHTMLBrowserIsolation(t *testing.T) {
|
|
if os.Getenv("PIPELINES_BROWSER_TEST") != "1" {
|
|
t.Skip("set PIPELINES_BROWSER_TEST=1 for the scratch Chrome check")
|
|
}
|
|
handler, step := runWithArtifact(t)
|
|
id := runIDFromURL(step.Artifacts[0].URL)
|
|
directory := filepath.Join(handler.runner.artifactsRoot, fmt.Sprint(id), "stages", fmt.Sprintf("%02d-%s", step.Sequence, step.Stage))
|
|
attack := `<!doctype html><title>Safe report</title><style>#report {color:rgb(12,34,56)}</style><h1 id="report">Readable report</h1><script>document.title='executed';fetch('/attack')</script>`
|
|
if err := os.WriteFile(filepath.Join(directory, "attack.html"), []byte(attack), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var requests atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/attack" {
|
|
requests.Add(1)
|
|
w.WriteHeader(http.StatusNoContent)
|
|
return
|
|
}
|
|
handler.ServeHTTP(w, r)
|
|
}))
|
|
defer srv.Close()
|
|
cmd := exec.Command("node", "../../web/test/artifact-security-probe.mjs", srv.URL+artifactURL(id, step.Sequence, "attack.html"))
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("browser isolation: %v\n%s", err, output)
|
|
} else {
|
|
t.Log(string(output))
|
|
}
|
|
if requests.Load() != 0 {
|
|
t.Fatal("artifact script reached the control origin")
|
|
}
|
|
}
|
|
|
|
func TestArtifactHTTPGetPublicSharedFile(t *testing.T) {
|
|
handler, step := runWithArtifact(t)
|
|
var shared protocol.StageArtifact
|
|
for _, artifact := range step.Artifacts {
|
|
if artifact.Name == "public/shared.txt" {
|
|
shared = artifact
|
|
}
|
|
}
|
|
if shared.URL == "" {
|
|
t.Fatalf("public artifact missing: %+v", step.Artifacts)
|
|
}
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
resp, err := http.Get(srv.URL + shared.URL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK || string(body) != "shared bytes" {
|
|
t.Fatalf("public artifact GET = %d %q", resp.StatusCode, body)
|
|
}
|
|
}
|
|
|
|
func runIDFromURL(url string) int64 {
|
|
var id int64
|
|
fmt.Sscanf(url, "/artifacts/%d/", &id)
|
|
return id
|
|
}
|
|
|
|
func TestArtifactHTTPRefusesEscapes(t *testing.T) {
|
|
handler, step := runWithArtifact(t)
|
|
runID := runIDFromURL(step.Artifacts[0].URL)
|
|
root := handler.runner.artifactsRoot
|
|
|
|
// A symlink inside the step directory must never be followed.
|
|
stepDir := filepath.Join(root, fmt.Sprint(runID), "stages", "01-one")
|
|
secret := filepath.Join(t.TempDir(), "secret.txt")
|
|
if err := os.WriteFile(secret, []byte("secret"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(secret, filepath.Join(stepDir, "escape.txt")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
for _, path := range []string{
|
|
fmt.Sprintf("/artifacts/%d/1/escape.txt", runID),
|
|
fmt.Sprintf("/artifacts/%d/1/%%2e%%2e/%%2e%%2e/secret.txt", runID),
|
|
fmt.Sprintf("/artifacts/%d/1/%%2Fetc%%2Fpasswd", runID),
|
|
} {
|
|
resp, err := http.Get(srv.URL + path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, _ := io.ReadAll(resp.Body)
|
|
resp.Body.Close()
|
|
if resp.StatusCode == http.StatusOK || bytes.Contains(body, []byte("secret")) {
|
|
t.Fatalf("%s served forbidden bytes: %d %q", path, resp.StatusCode, body)
|
|
}
|
|
}
|
|
}
|