feat(chat): files upload over HTTP before the prompt, up to 100 MiB each
This commit is contained in:
@@ -18,7 +18,7 @@ import (
|
||||
// Files survive process restarts and asynchronous harness inbox admission.
|
||||
// Only unleased files older than seven days are collected, on the next upload.
|
||||
const chatAttachmentRetention = 7 * 24 * time.Hour
|
||||
const chatAttachmentQuota = 64 << 20
|
||||
const chatAttachmentQuota = 1 << 30
|
||||
const chatAttachmentFileQuota = 512
|
||||
|
||||
var chatAttachmentStore = struct {
|
||||
@@ -59,69 +59,54 @@ func prepareChatAttachments(sessionID string, attachments []protocol.ChatAttachm
|
||||
if len(attachments) > maxChatAttachments {
|
||||
return nil, nil, fmt.Errorf("at most %d attachments are allowed", maxChatAttachments)
|
||||
}
|
||||
dir := os.Getenv("PIPELINES_CHAT_ATTACHMENT_DIR")
|
||||
if dir == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
dir = filepath.Join(home, ".pipelines-chat-attachments")
|
||||
}
|
||||
if !filepath.IsAbs(dir) {
|
||||
return nil, nil, fmt.Errorf("chat attachment storage must be an absolute path")
|
||||
}
|
||||
dir = filepath.Clean(dir)
|
||||
chatAttachmentStore.Lock()
|
||||
defer chatAttachmentStore.Unlock()
|
||||
if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
|
||||
return nil, nil, fmt.Errorf("prepare attachment directory: %w", err)
|
||||
}
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 {
|
||||
return nil, nil, fmt.Errorf("attachment storage must be a private directory, not a symlink")
|
||||
}
|
||||
root, err := os.OpenRoot(dir)
|
||||
root, dir, stored, storedFiles, err := openChatAttachmentStore()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer root.Close()
|
||||
actual, err := root.Stat(".")
|
||||
if err != nil || !os.SameFile(info, actual) {
|
||||
return nil, nil, fmt.Errorf("chat attachment storage changed while opening")
|
||||
}
|
||||
entries, err := fs.ReadDir(root.FS(), ".")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var stored int64
|
||||
var storedFiles int
|
||||
for _, entry := range entries {
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
if !chatAttachmentStore.leased[filepath.Join(dir, entry.Name())] && time.Since(info.ModTime()) > chatAttachmentRetention {
|
||||
if err := root.Remove(entry.Name()); err == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
stored += info.Size()
|
||||
storedFiles++
|
||||
}
|
||||
var files []opencode.PromptFile
|
||||
var parts []protocol.ChatPart
|
||||
var created []string
|
||||
var created, referenced []string
|
||||
cleanup := func() {
|
||||
for _, path := range created {
|
||||
_ = root.Remove(filepath.Base(path))
|
||||
delete(chatAttachmentStore.leased, path)
|
||||
}
|
||||
for _, path := range referenced {
|
||||
delete(chatAttachmentStore.leased, path)
|
||||
}
|
||||
}
|
||||
total := 0
|
||||
for index, attachment := range attachments {
|
||||
if attachment.Upload != "" {
|
||||
// A file sent earlier through POST /chat-uploads: only its stored
|
||||
// name crosses the websocket.
|
||||
upload := attachment.Upload
|
||||
info, err := root.Lstat(upload)
|
||||
if !chatUploadName.MatchString(upload) || err != nil || !info.Mode().IsRegular() {
|
||||
cleanup()
|
||||
return nil, nil, fmt.Errorf("attachment %q was not uploaded or has expired", attachment.Name)
|
||||
}
|
||||
path := filepath.Join(dir, upload)
|
||||
if !chatAttachmentStore.leased[path] {
|
||||
referenced = append(referenced, path)
|
||||
}
|
||||
chatAttachmentStore.leased[path] = true
|
||||
name := safeChatAttachmentName(attachment.Name)
|
||||
if strings.TrimSpace(attachment.Name) == "" {
|
||||
name = safeChatAttachmentName(strings.SplitN(upload, "-", 3)[2])
|
||||
}
|
||||
mime := strings.TrimSpace(attachment.Mime)
|
||||
if mime == "" {
|
||||
mime = "application/octet-stream"
|
||||
}
|
||||
uri := (&url.URL{Scheme: "file", Path: path}).String()
|
||||
files = append(files, opencode.NewPreparedPromptFile(uri, name))
|
||||
parts = append(parts, protocol.ChatPart{Type: protocol.ChatPartFile, Name: name, Mime: mime, URI: uri})
|
||||
continue
|
||||
}
|
||||
data := strings.TrimSpace(attachment.Data)
|
||||
if len(data) > base64.StdEncoding.EncodedLen(maxChatAttachmentBytes) {
|
||||
cleanup()
|
||||
@@ -204,3 +189,62 @@ func safeChatAttachmentName(value string) string {
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// openChatAttachmentStore opens the private attachment directory, collects
|
||||
// unleased files past retention and reports what the rest occupy. The caller
|
||||
// holds chatAttachmentStore and closes the root.
|
||||
func openChatAttachmentStore() (*os.Root, string, int64, int, error) {
|
||||
dir := os.Getenv("PIPELINES_CHAT_ATTACHMENT_DIR")
|
||||
if dir == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, "", 0, 0, err
|
||||
}
|
||||
dir = filepath.Join(home, ".pipelines-chat-attachments")
|
||||
}
|
||||
if !filepath.IsAbs(dir) {
|
||||
return nil, "", 0, 0, fmt.Errorf("chat attachment storage must be an absolute path")
|
||||
}
|
||||
dir = filepath.Clean(dir)
|
||||
if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
|
||||
return nil, "", 0, 0, fmt.Errorf("prepare attachment directory: %w", err)
|
||||
}
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 {
|
||||
return nil, "", 0, 0, fmt.Errorf("attachment storage must be a private directory, not a symlink")
|
||||
}
|
||||
root, err := os.OpenRoot(dir)
|
||||
if err != nil {
|
||||
return nil, "", 0, 0, err
|
||||
}
|
||||
actual, err := root.Stat(".")
|
||||
if err != nil || !os.SameFile(info, actual) {
|
||||
root.Close()
|
||||
return nil, "", 0, 0, fmt.Errorf("chat attachment storage changed while opening")
|
||||
}
|
||||
entries, err := fs.ReadDir(root.FS(), ".")
|
||||
if err != nil {
|
||||
root.Close()
|
||||
return nil, "", 0, 0, err
|
||||
}
|
||||
var stored int64
|
||||
var storedFiles int
|
||||
for _, entry := range entries {
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
root.Close()
|
||||
return nil, "", 0, 0, err
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
if !chatAttachmentStore.leased[filepath.Join(dir, entry.Name())] && time.Since(info.ModTime()) > chatAttachmentRetention {
|
||||
if err := root.Remove(entry.Name()); err == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
stored += info.Size()
|
||||
storedFiles++
|
||||
}
|
||||
return root, dir, stored, storedFiles, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// maxChatUploadBytes bounds one file sent through POST /chat-uploads. Files
|
||||
// travel over plain HTTP, outside the 2 MiB websocket frame, so a prompt can
|
||||
// carry photos, logs and archives the inline base64 route cannot.
|
||||
const maxChatUploadBytes = 100 << 20
|
||||
|
||||
// chatUploadHeader carries the file's name. Being a custom header, it also
|
||||
// keeps a cross-site page from posting here without a CORS preflight, which
|
||||
// neither the server nor its proxies answer.
|
||||
const chatUploadHeader = "X-Pipelines-Upload-Name"
|
||||
|
||||
var chatUploadName = regexp.MustCompile(`^up-[0-9a-f]{32}-[^/\\]+$`)
|
||||
|
||||
type chatUploadReply struct {
|
||||
Upload string `json:"upload"`
|
||||
Name string `json:"name"`
|
||||
Mime string `json:"mime"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// serveChatUpload stores the request body as one private chat attachment and
|
||||
// returns the name a later chat prompt references in ChatAttachment.Upload.
|
||||
func serveChatUpload(w http.ResponseWriter, r *http.Request) {
|
||||
fail := func(status int, message string) {
|
||||
http.Error(w, message, status)
|
||||
}
|
||||
rawName := r.Header.Get(chatUploadHeader)
|
||||
if rawName == "" {
|
||||
fail(http.StatusBadRequest, "missing "+chatUploadHeader)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("Sec-Fetch-Site") == "cross-site" {
|
||||
fail(http.StatusForbidden, "cross-site upload refused")
|
||||
return
|
||||
}
|
||||
decoded, err := url.PathUnescape(rawName)
|
||||
if err != nil {
|
||||
decoded = rawName
|
||||
}
|
||||
name := safeChatAttachmentName(decoded)
|
||||
if r.ContentLength > maxChatUploadBytes {
|
||||
fail(http.StatusRequestEntityTooLarge, "attachment exceeds the 100 MiB limit")
|
||||
return
|
||||
}
|
||||
mime := strings.TrimSpace(r.Header.Get("Content-Type"))
|
||||
if mime == "" {
|
||||
mime = "application/octet-stream"
|
||||
}
|
||||
var token [16]byte
|
||||
if _, err := rand.Read(token[:]); err != nil {
|
||||
fail(http.StatusInternalServerError, "upload id unavailable")
|
||||
return
|
||||
}
|
||||
stored := "up-" + hex.EncodeToString(token[:]) + "-" + name
|
||||
|
||||
chatAttachmentStore.Lock()
|
||||
root, dir, used, usedFiles, err := openChatAttachmentStore()
|
||||
if err != nil {
|
||||
chatAttachmentStore.Unlock()
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
defer root.Close()
|
||||
if usedFiles+1 > chatAttachmentFileQuota || used+max(r.ContentLength, 0) > chatAttachmentQuota {
|
||||
chatAttachmentStore.Unlock()
|
||||
fail(http.StatusInsufficientStorage, "chat attachment storage quota exceeded; retry after retention cleanup")
|
||||
return
|
||||
}
|
||||
file, err := root.OpenFile(stored, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
chatAttachmentStore.Unlock()
|
||||
fail(http.StatusInternalServerError, "write attachment: "+err.Error())
|
||||
return
|
||||
}
|
||||
// Leased while it is written so retention cannot collect it mid-upload.
|
||||
path := filepath.Join(dir, stored)
|
||||
chatAttachmentStore.leased[path] = true
|
||||
chatAttachmentStore.Unlock()
|
||||
release := func(remove bool) {
|
||||
chatAttachmentStore.Lock()
|
||||
delete(chatAttachmentStore.leased, path)
|
||||
if remove {
|
||||
_ = root.Remove(stored)
|
||||
}
|
||||
chatAttachmentStore.Unlock()
|
||||
}
|
||||
|
||||
size, err := io.Copy(file, io.LimitReader(r.Body, maxChatUploadBytes+1))
|
||||
if err == nil && size > maxChatUploadBytes {
|
||||
err = errors.New("attachment exceeds the 100 MiB limit")
|
||||
}
|
||||
if err == nil {
|
||||
err = file.Sync()
|
||||
}
|
||||
if closeErr := file.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err == nil && size == 0 {
|
||||
err = errors.New("attachment is empty")
|
||||
}
|
||||
if err != nil {
|
||||
release(true)
|
||||
status := http.StatusBadRequest
|
||||
if size > maxChatUploadBytes {
|
||||
status = http.StatusRequestEntityTooLarge
|
||||
}
|
||||
fail(status, err.Error())
|
||||
return
|
||||
}
|
||||
release(false)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
_ = json.NewEncoder(w).Encode(chatUploadReply{Upload: stored, Name: name, Mime: mime, Size: size})
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func postChatUpload(t *testing.T, name, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader(body))
|
||||
if name != "" {
|
||||
req.Header.Set(chatUploadHeader, name)
|
||||
}
|
||||
req.Header.Set("Content-Type", "image/png")
|
||||
rec := httptest.NewRecorder()
|
||||
serveChatUpload(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestChatUploadIsReferencedByAPrompt(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "uploads")
|
||||
t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", dir)
|
||||
// Larger than the inline 1 MiB route allows.
|
||||
body := strings.Repeat("x", 3<<20)
|
||||
rec := postChatUpload(t, "big%20photo.png", body)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("upload status = %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
var reply chatUploadReply
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reply.Name != "big photo.png" || reply.Size != int64(len(body)) || reply.Mime != "image/png" || !chatUploadName.MatchString(reply.Upload) {
|
||||
t.Fatalf("reply = %+v", reply)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(dir, reply.Upload))
|
||||
if err != nil || info.Mode().Perm() != 0o600 || info.Size() != int64(len(body)) {
|
||||
t.Fatalf("stored file = %v %v", info, err)
|
||||
}
|
||||
|
||||
files, parts, err := prepareChatAttachments("ses_test", []protocol.ChatAttachment{{Name: "big photo.png", Mime: "image/png", Upload: reply.Upload}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 1 || len(parts) != 1 || parts[0].Name != "big photo.png" || !strings.Contains(parts[0].URI, reply.Upload[:35]) {
|
||||
t.Fatalf("files = %+v parts = %+v", files, parts)
|
||||
}
|
||||
releaseChatAttachments(files)
|
||||
}
|
||||
|
||||
func TestChatUploadRefusals(t *testing.T) {
|
||||
t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", filepath.Join(t.TempDir(), "uploads"))
|
||||
if rec := postChatUpload(t, "", "x"); rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("missing name status = %d", rec.Code)
|
||||
}
|
||||
if rec := postChatUpload(t, "empty.txt", ""); rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("empty status = %d", rec.Code)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader("x"))
|
||||
req.Header.Set(chatUploadHeader, "a.txt")
|
||||
req.Header.Set("Sec-Fetch-Site", "cross-site")
|
||||
rec := httptest.NewRecorder()
|
||||
serveChatUpload(rec, req)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("cross-site status = %d", rec.Code)
|
||||
}
|
||||
req = httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader("x"))
|
||||
req.Header.Set(chatUploadHeader, "a.txt")
|
||||
req.ContentLength = maxChatUploadBytes + 1
|
||||
rec = httptest.NewRecorder()
|
||||
serveChatUpload(rec, req)
|
||||
if rec.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("oversized status = %d", rec.Code)
|
||||
}
|
||||
for _, upload := range []string{"../escape", "up-zz-x", "up-00000000000000000000000000000000-missing.txt"} {
|
||||
if _, _, err := prepareChatAttachments("ses_test", []protocol.ChatAttachment{{Name: "x", Upload: upload}}); err == nil {
|
||||
t.Fatalf("reference %q accepted", upload)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -115,6 +115,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
}
|
||||
runs.serveArtifact(w, r)
|
||||
})
|
||||
mux.HandleFunc("POST /chat-uploads", serveChatUpload)
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if runErr != nil {
|
||||
|
||||
Reference in New Issue
Block a user