feat(chat): files upload over HTTP before the prompt, up to 100 MiB each

This commit is contained in:
2026-10-23 21:25:41 +01:00
parent 39a68a00b3
commit e507f554c3
17 changed files with 634 additions and 74 deletions
+92 -48
View File
@@ -18,7 +18,7 @@ import (
// Files survive process restarts and asynchronous harness inbox admission.
// Only unleased files older than seven days are collected, on the next upload.
const chatAttachmentRetention = 7 * 24 * time.Hour
const chatAttachmentQuota = 64 << 20
const chatAttachmentQuota = 1 << 30
const chatAttachmentFileQuota = 512
var chatAttachmentStore = struct {
@@ -59,69 +59,54 @@ func prepareChatAttachments(sessionID string, attachments []protocol.ChatAttachm
if len(attachments) > maxChatAttachments {
return nil, nil, fmt.Errorf("at most %d attachments are allowed", maxChatAttachments)
}
dir := os.Getenv("PIPELINES_CHAT_ATTACHMENT_DIR")
if dir == "" {
home, err := os.UserHomeDir()
if err != nil {
return nil, nil, err
}
dir = filepath.Join(home, ".pipelines-chat-attachments")
}
if !filepath.IsAbs(dir) {
return nil, nil, fmt.Errorf("chat attachment storage must be an absolute path")
}
dir = filepath.Clean(dir)
chatAttachmentStore.Lock()
defer chatAttachmentStore.Unlock()
if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
return nil, nil, fmt.Errorf("prepare attachment directory: %w", err)
}
info, err := os.Lstat(dir)
if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 {
return nil, nil, fmt.Errorf("attachment storage must be a private directory, not a symlink")
}
root, err := os.OpenRoot(dir)
root, dir, stored, storedFiles, err := openChatAttachmentStore()
if err != nil {
return nil, nil, err
}
defer root.Close()
actual, err := root.Stat(".")
if err != nil || !os.SameFile(info, actual) {
return nil, nil, fmt.Errorf("chat attachment storage changed while opening")
}
entries, err := fs.ReadDir(root.FS(), ".")
if err != nil {
return nil, nil, err
}
var stored int64
var storedFiles int
for _, entry := range entries {
info, err := entry.Info()
if err != nil {
return nil, nil, err
}
if !info.Mode().IsRegular() {
continue
}
if !chatAttachmentStore.leased[filepath.Join(dir, entry.Name())] && time.Since(info.ModTime()) > chatAttachmentRetention {
if err := root.Remove(entry.Name()); err == nil {
continue
}
}
stored += info.Size()
storedFiles++
}
var files []opencode.PromptFile
var parts []protocol.ChatPart
var created []string
var created, referenced []string
cleanup := func() {
for _, path := range created {
_ = root.Remove(filepath.Base(path))
delete(chatAttachmentStore.leased, path)
}
for _, path := range referenced {
delete(chatAttachmentStore.leased, path)
}
}
total := 0
for index, attachment := range attachments {
if attachment.Upload != "" {
// A file sent earlier through POST /chat-uploads: only its stored
// name crosses the websocket.
upload := attachment.Upload
info, err := root.Lstat(upload)
if !chatUploadName.MatchString(upload) || err != nil || !info.Mode().IsRegular() {
cleanup()
return nil, nil, fmt.Errorf("attachment %q was not uploaded or has expired", attachment.Name)
}
path := filepath.Join(dir, upload)
if !chatAttachmentStore.leased[path] {
referenced = append(referenced, path)
}
chatAttachmentStore.leased[path] = true
name := safeChatAttachmentName(attachment.Name)
if strings.TrimSpace(attachment.Name) == "" {
name = safeChatAttachmentName(strings.SplitN(upload, "-", 3)[2])
}
mime := strings.TrimSpace(attachment.Mime)
if mime == "" {
mime = "application/octet-stream"
}
uri := (&url.URL{Scheme: "file", Path: path}).String()
files = append(files, opencode.NewPreparedPromptFile(uri, name))
parts = append(parts, protocol.ChatPart{Type: protocol.ChatPartFile, Name: name, Mime: mime, URI: uri})
continue
}
data := strings.TrimSpace(attachment.Data)
if len(data) > base64.StdEncoding.EncodedLen(maxChatAttachmentBytes) {
cleanup()
@@ -204,3 +189,62 @@ func safeChatAttachmentName(value string) string {
}
return name
}
// openChatAttachmentStore opens the private attachment directory, collects
// unleased files past retention and reports what the rest occupy. The caller
// holds chatAttachmentStore and closes the root.
func openChatAttachmentStore() (*os.Root, string, int64, int, error) {
dir := os.Getenv("PIPELINES_CHAT_ATTACHMENT_DIR")
if dir == "" {
home, err := os.UserHomeDir()
if err != nil {
return nil, "", 0, 0, err
}
dir = filepath.Join(home, ".pipelines-chat-attachments")
}
if !filepath.IsAbs(dir) {
return nil, "", 0, 0, fmt.Errorf("chat attachment storage must be an absolute path")
}
dir = filepath.Clean(dir)
if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
return nil, "", 0, 0, fmt.Errorf("prepare attachment directory: %w", err)
}
info, err := os.Lstat(dir)
if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 {
return nil, "", 0, 0, fmt.Errorf("attachment storage must be a private directory, not a symlink")
}
root, err := os.OpenRoot(dir)
if err != nil {
return nil, "", 0, 0, err
}
actual, err := root.Stat(".")
if err != nil || !os.SameFile(info, actual) {
root.Close()
return nil, "", 0, 0, fmt.Errorf("chat attachment storage changed while opening")
}
entries, err := fs.ReadDir(root.FS(), ".")
if err != nil {
root.Close()
return nil, "", 0, 0, err
}
var stored int64
var storedFiles int
for _, entry := range entries {
info, err := entry.Info()
if err != nil {
root.Close()
return nil, "", 0, 0, err
}
if !info.Mode().IsRegular() {
continue
}
if !chatAttachmentStore.leased[filepath.Join(dir, entry.Name())] && time.Since(info.ModTime()) > chatAttachmentRetention {
if err := root.Remove(entry.Name()); err == nil {
continue
}
}
stored += info.Size()
storedFiles++
}
return root, dir, stored, storedFiles, nil
}
+129
View File
@@ -0,0 +1,129 @@
package server
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
)
// maxChatUploadBytes bounds one file sent through POST /chat-uploads. Files
// travel over plain HTTP, outside the 2 MiB websocket frame, so a prompt can
// carry photos, logs and archives the inline base64 route cannot.
const maxChatUploadBytes = 100 << 20
// chatUploadHeader carries the file's name. Being a custom header, it also
// keeps a cross-site page from posting here without a CORS preflight, which
// neither the server nor its proxies answer.
const chatUploadHeader = "X-Pipelines-Upload-Name"
var chatUploadName = regexp.MustCompile(`^up-[0-9a-f]{32}-[^/\\]+$`)
type chatUploadReply struct {
Upload string `json:"upload"`
Name string `json:"name"`
Mime string `json:"mime"`
Size int64 `json:"size"`
}
// serveChatUpload stores the request body as one private chat attachment and
// returns the name a later chat prompt references in ChatAttachment.Upload.
func serveChatUpload(w http.ResponseWriter, r *http.Request) {
fail := func(status int, message string) {
http.Error(w, message, status)
}
rawName := r.Header.Get(chatUploadHeader)
if rawName == "" {
fail(http.StatusBadRequest, "missing "+chatUploadHeader)
return
}
if r.Header.Get("Sec-Fetch-Site") == "cross-site" {
fail(http.StatusForbidden, "cross-site upload refused")
return
}
decoded, err := url.PathUnescape(rawName)
if err != nil {
decoded = rawName
}
name := safeChatAttachmentName(decoded)
if r.ContentLength > maxChatUploadBytes {
fail(http.StatusRequestEntityTooLarge, "attachment exceeds the 100 MiB limit")
return
}
mime := strings.TrimSpace(r.Header.Get("Content-Type"))
if mime == "" {
mime = "application/octet-stream"
}
var token [16]byte
if _, err := rand.Read(token[:]); err != nil {
fail(http.StatusInternalServerError, "upload id unavailable")
return
}
stored := "up-" + hex.EncodeToString(token[:]) + "-" + name
chatAttachmentStore.Lock()
root, dir, used, usedFiles, err := openChatAttachmentStore()
if err != nil {
chatAttachmentStore.Unlock()
fail(http.StatusInternalServerError, err.Error())
return
}
defer root.Close()
if usedFiles+1 > chatAttachmentFileQuota || used+max(r.ContentLength, 0) > chatAttachmentQuota {
chatAttachmentStore.Unlock()
fail(http.StatusInsufficientStorage, "chat attachment storage quota exceeded; retry after retention cleanup")
return
}
file, err := root.OpenFile(stored, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
chatAttachmentStore.Unlock()
fail(http.StatusInternalServerError, "write attachment: "+err.Error())
return
}
// Leased while it is written so retention cannot collect it mid-upload.
path := filepath.Join(dir, stored)
chatAttachmentStore.leased[path] = true
chatAttachmentStore.Unlock()
release := func(remove bool) {
chatAttachmentStore.Lock()
delete(chatAttachmentStore.leased, path)
if remove {
_ = root.Remove(stored)
}
chatAttachmentStore.Unlock()
}
size, err := io.Copy(file, io.LimitReader(r.Body, maxChatUploadBytes+1))
if err == nil && size > maxChatUploadBytes {
err = errors.New("attachment exceeds the 100 MiB limit")
}
if err == nil {
err = file.Sync()
}
if closeErr := file.Close(); err == nil {
err = closeErr
}
if err == nil && size == 0 {
err = errors.New("attachment is empty")
}
if err != nil {
release(true)
status := http.StatusBadRequest
if size > maxChatUploadBytes {
status = http.StatusRequestEntityTooLarge
}
fail(status, err.Error())
return
}
release(false)
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
_ = json.NewEncoder(w).Encode(chatUploadReply{Upload: stored, Name: name, Mime: mime, Size: size})
}
+87
View File
@@ -0,0 +1,87 @@
package server
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func postChatUpload(t *testing.T, name, body string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader(body))
if name != "" {
req.Header.Set(chatUploadHeader, name)
}
req.Header.Set("Content-Type", "image/png")
rec := httptest.NewRecorder()
serveChatUpload(rec, req)
return rec
}
func TestChatUploadIsReferencedByAPrompt(t *testing.T) {
dir := filepath.Join(t.TempDir(), "uploads")
t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", dir)
// Larger than the inline 1 MiB route allows.
body := strings.Repeat("x", 3<<20)
rec := postChatUpload(t, "big%20photo.png", body)
if rec.Code != http.StatusOK {
t.Fatalf("upload status = %d %s", rec.Code, rec.Body)
}
var reply chatUploadReply
if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil {
t.Fatal(err)
}
if reply.Name != "big photo.png" || reply.Size != int64(len(body)) || reply.Mime != "image/png" || !chatUploadName.MatchString(reply.Upload) {
t.Fatalf("reply = %+v", reply)
}
info, err := os.Stat(filepath.Join(dir, reply.Upload))
if err != nil || info.Mode().Perm() != 0o600 || info.Size() != int64(len(body)) {
t.Fatalf("stored file = %v %v", info, err)
}
files, parts, err := prepareChatAttachments("ses_test", []protocol.ChatAttachment{{Name: "big photo.png", Mime: "image/png", Upload: reply.Upload}})
if err != nil {
t.Fatal(err)
}
if len(files) != 1 || len(parts) != 1 || parts[0].Name != "big photo.png" || !strings.Contains(parts[0].URI, reply.Upload[:35]) {
t.Fatalf("files = %+v parts = %+v", files, parts)
}
releaseChatAttachments(files)
}
func TestChatUploadRefusals(t *testing.T) {
t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", filepath.Join(t.TempDir(), "uploads"))
if rec := postChatUpload(t, "", "x"); rec.Code != http.StatusBadRequest {
t.Fatalf("missing name status = %d", rec.Code)
}
if rec := postChatUpload(t, "empty.txt", ""); rec.Code != http.StatusBadRequest {
t.Fatalf("empty status = %d", rec.Code)
}
req := httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader("x"))
req.Header.Set(chatUploadHeader, "a.txt")
req.Header.Set("Sec-Fetch-Site", "cross-site")
rec := httptest.NewRecorder()
serveChatUpload(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("cross-site status = %d", rec.Code)
}
req = httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader("x"))
req.Header.Set(chatUploadHeader, "a.txt")
req.ContentLength = maxChatUploadBytes + 1
rec = httptest.NewRecorder()
serveChatUpload(rec, req)
if rec.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized status = %d", rec.Code)
}
for _, upload := range []string{"../escape", "up-zz-x", "up-00000000000000000000000000000000-missing.txt"} {
if _, _, err := prepareChatAttachments("ses_test", []protocol.ChatAttachment{{Name: "x", Upload: upload}}); err == nil {
t.Fatalf("reference %q accepted", upload)
}
}
}
+1
View File
@@ -115,6 +115,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
runs.serveArtifact(w, r)
})
mux.HandleFunc("POST /chat-uploads", serveChatUpload)
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if runErr != nil {