538143554c
Every stage used to be told to read memory.md, the full output of every earlier turn (19 KB average, 92 KB max), and paid for it on each model call. memory.md is now essential memory, hard-capped at 4 KB and inlined in the prompt: the task, the previous stage's # handoff section (or its truncated output head), a one-line-per-stage outcome ledger and the server-captured changed files. history.md keeps the old full memory on disk; read-capable stages are pointed at it and input.md for on-demand detail, and it is never inlined. The prompt asks each stage for a short # handoff section.
299 lines
9.2 KiB
Go
299 lines
9.2 KiB
Go
package server
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"log"
|
|
"mime"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// Artifact capture is deliberately small and best-effort. A stage's artifacts
|
|
// directory is evidence, never a gate: every scan error is swallowed and never
|
|
// changes the step's outcome.
|
|
const (
|
|
maxArtifactFiles = 200
|
|
maxArtifactFileBytes = 8 << 20
|
|
maxArtifactTotalBytes = 64 << 20
|
|
artifactSniffBytes = 512
|
|
)
|
|
|
|
// scanArtifacts returns a bounded inventory of one step's artifacts directory.
|
|
// The directory path is supplied by the caller because it was created next to
|
|
// the step; runID and sequence name the served URL. Optional public
|
|
// directories are inventoried with a "public/" name prefix so the shared
|
|
// artifacts are visible from the visit that produced them. A missing,
|
|
// unreadable, symlinked or oversized directory yields no entries and a zero
|
|
// count.
|
|
func (r *runner) scanArtifacts(runID int64, sequence int, directory string, publicDirs ...string) ([]protocol.StageArtifact, int, int) {
|
|
if directory == "" || r.artifactsRoot == "" {
|
|
return nil, 0, 0
|
|
}
|
|
root, err := os.OpenRoot(r.artifactsRoot)
|
|
if err != nil {
|
|
log.Printf("stage artifacts root unavailable: %v", err)
|
|
return nil, 0, 0
|
|
}
|
|
defer root.Close()
|
|
rel, err := artifactRel(root.Name(), directory)
|
|
if err != nil {
|
|
return nil, 0, 0
|
|
}
|
|
if !containedDirectory(root, rel) {
|
|
return nil, 0, 0
|
|
}
|
|
files := []protocol.StageArtifact{}
|
|
seen := map[string]bool{}
|
|
skipped, truncated := 0, 0
|
|
used := int64(0)
|
|
add := func(full, name string, info fs.FileInfo) bool {
|
|
if seen[full] {
|
|
return true
|
|
}
|
|
if len(files) >= maxArtifactFiles {
|
|
skipped++
|
|
return false
|
|
}
|
|
if info.Size() > maxArtifactFileBytes {
|
|
skipped++
|
|
return true
|
|
}
|
|
if used+info.Size() > maxArtifactTotalBytes {
|
|
truncated++
|
|
return false
|
|
}
|
|
contentType := mime.TypeByExtension(filepath.Ext(name))
|
|
if contentType == "" {
|
|
contentType = sniffArtifact(root, full)
|
|
if contentType == "" {
|
|
skipped++
|
|
return true
|
|
}
|
|
}
|
|
used += info.Size()
|
|
seen[full] = true
|
|
files = append(files, protocol.StageArtifact{Name: name, SizeBytes: info.Size(), ContentType: contentType, URL: artifactURL(runID, sequence, name)})
|
|
return true
|
|
}
|
|
// Mandatory runtime evidence must not disappear behind hundreds of private
|
|
// captures/public deliverables in the bounded inventory.
|
|
for _, name := range []string{"input.md", "memory.md", "history.md", "output.md", "session.html"} {
|
|
full := path.Join(rel, name)
|
|
if info, err := root.Lstat(full); err == nil && info.Mode().IsRegular() && info.Mode()&fs.ModeSymlink == 0 && info.Size() <= maxArtifactFileBytes {
|
|
add(full, name, info)
|
|
}
|
|
}
|
|
scan := func(relative, prefix string) {
|
|
_ = fs.WalkDir(root.FS(), relative, func(name string, entry fs.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
// A single unreadable entry must not abort the whole inventory.
|
|
return nil
|
|
}
|
|
if name == relative || entry.IsDir() {
|
|
return nil
|
|
}
|
|
if seen[name] {
|
|
return nil
|
|
}
|
|
if entry.Type()&fs.ModeSymlink != 0 || !entry.Type().IsRegular() {
|
|
skipped++
|
|
return nil
|
|
}
|
|
info, infoErr := entry.Info()
|
|
if infoErr != nil || !info.Mode().IsRegular() {
|
|
skipped++
|
|
return nil
|
|
}
|
|
artifactName := strings.TrimPrefix(name, relative+"/")
|
|
artifactName = prefix + artifactName
|
|
full := path.Join(relative, strings.TrimPrefix(artifactName, prefix))
|
|
if !add(full, artifactName, info) {
|
|
return fs.SkipAll
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
scan(rel, "")
|
|
for _, public := range publicDirs {
|
|
if public == "" {
|
|
continue
|
|
}
|
|
publicRel, relErr := artifactRel(root.Name(), public)
|
|
if relErr != nil || !containedDirectory(root, publicRel) {
|
|
continue
|
|
}
|
|
scan(publicRel, "public/")
|
|
}
|
|
if len(files) == 0 {
|
|
files = nil
|
|
}
|
|
return files, skipped, truncated
|
|
}
|
|
|
|
// artifactRel returns the slash path of directory below the artifacts root. A
|
|
// directory outside the root is refused.
|
|
func artifactRel(root, directory string) (string, error) {
|
|
relative, err := filepath.Rel(root, directory)
|
|
if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) || relative == ".." || filepath.IsAbs(relative) {
|
|
return "", fmt.Errorf("artifact directory is outside the artifacts root")
|
|
}
|
|
return filepath.ToSlash(relative), nil
|
|
}
|
|
|
|
// containedDirectory reports whether rel names a real directory below root with
|
|
// no symlink in any component.
|
|
func containedDirectory(root *os.Root, rel string) bool {
|
|
if rel == "" || !fs.ValidPath(rel) || rel == "." {
|
|
return false
|
|
}
|
|
parts := strings.Split(rel, "/")
|
|
for i := range parts {
|
|
info, err := root.Lstat(strings.Join(parts[:i+1], "/"))
|
|
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
|
return false
|
|
}
|
|
if i == len(parts)-1 {
|
|
return info.IsDir()
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// sniffArtifact opens one regular file without following a final symlink and
|
|
// returns the detected content type from at most the first 512 bytes.
|
|
func sniffArtifact(root *os.Root, rel string) string {
|
|
file, err := root.OpenFile(rel, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer file.Close()
|
|
buffer := make([]byte, artifactSniffBytes)
|
|
n, err := io.ReadFull(file, buffer)
|
|
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
|
|
return ""
|
|
}
|
|
return http.DetectContentType(buffer[:n])
|
|
}
|
|
|
|
// artifactURL builds the server-relative URL for one artifact, escaping each
|
|
// path segment so a name cannot forge path structure.
|
|
func artifactURL(runID int64, sequence int, name string) string {
|
|
parts := strings.Split(name, "/")
|
|
for i, part := range parts {
|
|
parts[i] = url.PathEscape(part)
|
|
}
|
|
return "/artifacts/" + strconv.FormatInt(runID, 10) + "/" + strconv.Itoa(sequence) + "/" + strings.Join(parts, "/")
|
|
}
|
|
|
|
// serveArtifact serves one recorded step artifact. It refuses unknown runs or
|
|
// steps and any name that is absolute, non-local, symlinked in any component or
|
|
// not a regular file, so nothing outside <artifactsRoot>/<run>/<step> is served.
|
|
// http.ServeContent supplies Content-Type (extension then sniff), Content-Length,
|
|
// HEAD and Range handling from the opened *os.File.
|
|
func (r *runner) serveArtifact(w http.ResponseWriter, req *http.Request) {
|
|
name := req.PathValue("name")
|
|
runID, runErr := strconv.ParseInt(req.PathValue("run"), 10, 64)
|
|
sequence, stepErr := strconv.Atoi(req.PathValue("step"))
|
|
if runErr != nil || stepErr != nil || runID <= 0 || sequence <= 0 {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
if name == "" || name == "." || !fs.ValidPath(name) || path.IsAbs(name) {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
detail, err := r.store.GetRun(req.Context(), runID)
|
|
if err != nil {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
known := false
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence == sequence {
|
|
known = true
|
|
break
|
|
}
|
|
}
|
|
if !known {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
root, err := os.OpenRoot(r.artifactsRoot)
|
|
if err != nil {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
defer root.Close()
|
|
runRoot := strconv.FormatInt(runID, 10)
|
|
// The public directory is shared by every stage; a name carrying the
|
|
// "public/" prefix maps there. Everything else maps to the turn directory
|
|
// for this visit, with the pre-turn <run>/<step> layout still readable for
|
|
// runs recorded before the file-based protocol landed.
|
|
var rel string
|
|
if strings.HasPrefix(name, "public/") {
|
|
rel = path.Join(runRoot, "public", strings.TrimPrefix(name, "public/"))
|
|
} else {
|
|
stageKey := ""
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence == sequence {
|
|
stageKey = step.Stage
|
|
break
|
|
}
|
|
}
|
|
turnDir := path.Join(runRoot, "stages", fmt.Sprintf("%02d-%s", sequence, stageKey))
|
|
if stageKey != "" && containedDirectory(root, turnDir) {
|
|
rel = path.Join(turnDir, name)
|
|
} else {
|
|
rel = path.Join(runRoot, strconv.Itoa(sequence), name)
|
|
}
|
|
}
|
|
if !containedDirectory(root, path.Dir(rel)) {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
full := rel
|
|
parts := strings.Split(full, "/")
|
|
for i := range parts {
|
|
info, lerr := root.Lstat(strings.Join(parts[:i+1], "/"))
|
|
if lerr != nil || info.Mode()&os.ModeSymlink != 0 {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
}
|
|
info, err := root.Lstat(full)
|
|
if err != nil || !info.Mode().IsRegular() {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
file, err := root.OpenFile(full, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
defer file.Close()
|
|
opened, err := file.Stat()
|
|
if err != nil || !opened.Mode().IsRegular() {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
// An artifact is untrusted generated content, not part of the control UI.
|
|
// Enforce the boundary on the response itself: opening the raw HTML/SVG in
|
|
// a new tab must be as safe as the embedded sandboxed preview. No scripts,
|
|
// network calls, forms, same-origin authority or top-level navigation.
|
|
w.Header().Set("Content-Security-Policy", "sandbox; default-src 'none'; script-src 'none'; connect-src 'none'; style-src 'unsafe-inline'; img-src 'self' data:; media-src 'self' data:; base-uri 'none'; form-action 'none'; frame-ancestors 'self'")
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
w.Header().Set("Cache-Control", "private, no-store")
|
|
http.ServeContent(w, req, path.Base(name), opened.ModTime(), file)
|
|
}
|