e37b864c40
Each model visit now gets <run>/stages/<seq>-<stage>/ with input.md, memory.md, output.md, private/ and a shared run-level public/. The model reads the two input files, follows the stage's instructions verbatim and returns complete markdown; only the # outcome section is parsed. - add internal/turn layout, render and output parsing - write input.md/memory.md before a visit and output.md from the final message; inventory all turn files as stage artifacts - accumulate completed turns and system-written public file additions in memory, including across retries and resume - remove the JSON envelope, strict field validation, line format and the repair ladder for model stages - republish pipeline authoring prompts and docs in the new style Checks: go build ./..., go vet ./..., go test -count=1 ./..., gofmt -l .
275 lines
7.9 KiB
Go
275 lines
7.9 KiB
Go
package server
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"log"
|
|
"mime"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// Artifact capture is deliberately small and best-effort. A stage's artifacts
|
|
// directory is evidence, never a gate: every scan error is swallowed and never
|
|
// changes the step's outcome.
|
|
const (
|
|
maxArtifactFiles = 200
|
|
maxArtifactFileBytes = 8 << 20
|
|
maxArtifactTotalBytes = 64 << 20
|
|
artifactSniffBytes = 512
|
|
)
|
|
|
|
// scanArtifacts returns a bounded inventory of one step's artifacts directory.
|
|
// The directory path is supplied by the caller because it was created next to
|
|
// the step; runID and sequence name the served URL. Optional public
|
|
// directories are inventoried with a "public/" name prefix so the shared
|
|
// artifacts are visible from the visit that produced them. A missing,
|
|
// unreadable, symlinked or oversized directory yields no entries and a zero
|
|
// count.
|
|
func (r *runner) scanArtifacts(runID int64, sequence int, directory string, publicDirs ...string) ([]protocol.StageArtifact, int, int) {
|
|
if directory == "" || r.artifactsRoot == "" {
|
|
return nil, 0, 0
|
|
}
|
|
root, err := os.OpenRoot(r.artifactsRoot)
|
|
if err != nil {
|
|
log.Printf("stage artifacts root unavailable: %v", err)
|
|
return nil, 0, 0
|
|
}
|
|
defer root.Close()
|
|
rel, err := artifactRel(root.Name(), directory)
|
|
if err != nil {
|
|
return nil, 0, 0
|
|
}
|
|
if !containedDirectory(root, rel) {
|
|
return nil, 0, 0
|
|
}
|
|
files := []protocol.StageArtifact{}
|
|
skipped, truncated := 0, 0
|
|
used := int64(0)
|
|
scan := func(relative, prefix string) {
|
|
_ = fs.WalkDir(root.FS(), relative, func(name string, entry fs.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
// A single unreadable entry must not abort the whole inventory.
|
|
return nil
|
|
}
|
|
if name == relative || entry.IsDir() {
|
|
return nil
|
|
}
|
|
if entry.Type()&fs.ModeSymlink != 0 || !entry.Type().IsRegular() {
|
|
skipped++
|
|
return nil
|
|
}
|
|
info, infoErr := entry.Info()
|
|
if infoErr != nil || !info.Mode().IsRegular() {
|
|
skipped++
|
|
return nil
|
|
}
|
|
if len(files) >= maxArtifactFiles {
|
|
skipped++
|
|
return fs.SkipAll
|
|
}
|
|
if info.Size() > maxArtifactFileBytes {
|
|
skipped++
|
|
return nil
|
|
}
|
|
if used+info.Size() > maxArtifactTotalBytes {
|
|
truncated++
|
|
return fs.SkipAll
|
|
}
|
|
artifactName := strings.TrimPrefix(name, relative+"/")
|
|
artifactName = prefix + artifactName
|
|
full := path.Join(relative, strings.TrimPrefix(artifactName, prefix))
|
|
contentType := mime.TypeByExtension(filepath.Ext(artifactName))
|
|
if contentType == "" {
|
|
contentType = sniffArtifact(root, full)
|
|
if contentType == "" {
|
|
skipped++
|
|
return nil
|
|
}
|
|
}
|
|
used += info.Size()
|
|
files = append(files, protocol.StageArtifact{
|
|
Name: artifactName,
|
|
SizeBytes: info.Size(),
|
|
ContentType: contentType,
|
|
URL: artifactURL(runID, sequence, artifactName),
|
|
})
|
|
return nil
|
|
})
|
|
}
|
|
scan(rel, "")
|
|
for _, public := range publicDirs {
|
|
if public == "" {
|
|
continue
|
|
}
|
|
publicRel, relErr := artifactRel(root.Name(), public)
|
|
if relErr != nil || !containedDirectory(root, publicRel) {
|
|
continue
|
|
}
|
|
scan(publicRel, "public/")
|
|
}
|
|
if len(files) == 0 {
|
|
files = nil
|
|
}
|
|
return files, skipped, truncated
|
|
}
|
|
|
|
// artifactRel returns the slash path of directory below the artifacts root. A
|
|
// directory outside the root is refused.
|
|
func artifactRel(root, directory string) (string, error) {
|
|
relative, err := filepath.Rel(root, directory)
|
|
if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(os.PathSeparator)) || relative == ".." || filepath.IsAbs(relative) {
|
|
return "", fmt.Errorf("artifact directory is outside the artifacts root")
|
|
}
|
|
return filepath.ToSlash(relative), nil
|
|
}
|
|
|
|
// containedDirectory reports whether rel names a real directory below root with
|
|
// no symlink in any component.
|
|
func containedDirectory(root *os.Root, rel string) bool {
|
|
if rel == "" || !fs.ValidPath(rel) || rel == "." {
|
|
return false
|
|
}
|
|
parts := strings.Split(rel, "/")
|
|
for i := range parts {
|
|
info, err := root.Lstat(strings.Join(parts[:i+1], "/"))
|
|
if err != nil || info.Mode()&os.ModeSymlink != 0 {
|
|
return false
|
|
}
|
|
if i == len(parts)-1 {
|
|
return info.IsDir()
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// sniffArtifact opens one regular file without following a final symlink and
|
|
// returns the detected content type from at most the first 512 bytes.
|
|
func sniffArtifact(root *os.Root, rel string) string {
|
|
file, err := root.OpenFile(rel, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer file.Close()
|
|
buffer := make([]byte, artifactSniffBytes)
|
|
n, err := io.ReadFull(file, buffer)
|
|
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
|
|
return ""
|
|
}
|
|
return http.DetectContentType(buffer[:n])
|
|
}
|
|
|
|
// artifactURL builds the server-relative URL for one artifact, escaping each
|
|
// path segment so a name cannot forge path structure.
|
|
func artifactURL(runID int64, sequence int, name string) string {
|
|
parts := strings.Split(name, "/")
|
|
for i, part := range parts {
|
|
parts[i] = url.PathEscape(part)
|
|
}
|
|
return "/artifacts/" + strconv.FormatInt(runID, 10) + "/" + strconv.Itoa(sequence) + "/" + strings.Join(parts, "/")
|
|
}
|
|
|
|
// serveArtifact serves one recorded step artifact. It refuses unknown runs or
|
|
// steps and any name that is absolute, non-local, symlinked in any component or
|
|
// not a regular file, so nothing outside <artifactsRoot>/<run>/<step> is served.
|
|
// http.ServeContent supplies Content-Type (extension then sniff), Content-Length,
|
|
// HEAD and Range handling from the opened *os.File.
|
|
func (r *runner) serveArtifact(w http.ResponseWriter, req *http.Request) {
|
|
name := req.PathValue("name")
|
|
runID, runErr := strconv.ParseInt(req.PathValue("run"), 10, 64)
|
|
sequence, stepErr := strconv.Atoi(req.PathValue("step"))
|
|
if runErr != nil || stepErr != nil || runID <= 0 || sequence <= 0 {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
if name == "" || name == "." || !fs.ValidPath(name) || path.IsAbs(name) {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
detail, err := r.store.GetRun(req.Context(), runID)
|
|
if err != nil {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
known := false
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence == sequence {
|
|
known = true
|
|
break
|
|
}
|
|
}
|
|
if !known {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
root, err := os.OpenRoot(r.artifactsRoot)
|
|
if err != nil {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
defer root.Close()
|
|
runRoot := strconv.FormatInt(runID, 10)
|
|
// The public directory is shared by every stage; a name carrying the
|
|
// "public/" prefix maps there. Everything else maps to the turn directory
|
|
// for this visit, with the pre-turn <run>/<step> layout still readable for
|
|
// runs recorded before the file-based protocol landed.
|
|
var rel string
|
|
if strings.HasPrefix(name, "public/") {
|
|
rel = path.Join(runRoot, "public", strings.TrimPrefix(name, "public/"))
|
|
} else {
|
|
stageKey := ""
|
|
for _, step := range detail.Steps {
|
|
if step.Sequence == sequence {
|
|
stageKey = step.Stage
|
|
break
|
|
}
|
|
}
|
|
turnDir := path.Join(runRoot, "stages", fmt.Sprintf("%02d-%s", sequence, stageKey))
|
|
if stageKey != "" && containedDirectory(root, turnDir) {
|
|
rel = path.Join(turnDir, name)
|
|
} else {
|
|
rel = path.Join(runRoot, strconv.Itoa(sequence), name)
|
|
}
|
|
}
|
|
if !containedDirectory(root, path.Dir(rel)) {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
full := rel
|
|
parts := strings.Split(full, "/")
|
|
for i := range parts {
|
|
info, lerr := root.Lstat(strings.Join(parts[:i+1], "/"))
|
|
if lerr != nil || info.Mode()&os.ModeSymlink != 0 {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
}
|
|
info, err := root.Lstat(full)
|
|
if err != nil || !info.Mode().IsRegular() {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
file, err := root.OpenFile(full, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
defer file.Close()
|
|
opened, err := file.Stat()
|
|
if err != nil || !opened.Mode().IsRegular() {
|
|
http.NotFound(w, req)
|
|
return
|
|
}
|
|
http.ServeContent(w, req, path.Base(name), opened.ModTime(), file)
|
|
}
|