feat(chat): files upload over HTTP before the prompt, up to 100 MiB each

This commit is contained in:
2026-10-23 21:25:41 +01:00
parent 39a68a00b3
commit e507f554c3
17 changed files with 634 additions and 74 deletions
@@ -6,6 +6,8 @@ import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import '../../../core/api_gateway.dart';
import '../../../core/transport/pipelines_client.dart';
import '../data/upload/chat_upload.dart';
import '../domain/chat_attachment_draft.dart';
import '../domain/chat_attachment_picker.dart';
import '../domain/chat_attachment_wire.dart';
@@ -101,6 +103,13 @@ class _ChatComposerBarState extends State<ChatComposerBar> {
final List<Uint8List> _bytes = [];
String? _attachmentError;
/// True while the draft's files upload ahead of the prompt.
bool _uploading = false;
/// Upload names by attachment instance, so a retried send reuses the files
/// already stored instead of uploading them again.
final Expando<String> _uploaded = Expando();
/// The draft text whose palette the operator closed (Escape) or completed
/// (a row or Enter). The palette stays hidden while the draft still equals
/// it, so any further typing offers the commands again.
@@ -392,14 +401,39 @@ class _ChatComposerBarState extends State<ChatComposerBar> {
/// accepted prompt clears them.
Future<void> _send() async {
final composer = widget.composer;
if (composer.sending || composer.draft.trim().isEmpty) return;
List<JsonMap> parts;
try {
parts = chatAttachmentParts(_attachments, read: _readAttachment);
} on ChatAttachmentWireException catch (error) {
setState(() => _attachmentError = error.message);
if (_uploading || composer.sending || composer.draft.trim().isEmpty) {
return;
}
List<JsonMap> parts;
final endpoint = configuredGatewayEndpoint(composer.gateway);
try {
if (endpoint.isEmpty || _attachments.isEmpty) {
parts = chatAttachmentParts(_attachments, read: _readAttachment);
} else {
// Each file goes to the server as its own request first; the prompt
// then names the stored files.
setState(() {
_uploading = true;
_attachmentError = null;
});
parts = await chatAttachmentUploadParts(
_attachments,
read: _readAttachment,
upload: (attachment, bytes) async => _uploaded[attachment] ??=
await uploadChatAttachment(endpoint, attachment, bytes),
);
}
} on ChatAttachmentWireException catch (error) {
if (mounted) {
setState(() {
_uploading = false;
_attachmentError = error.message;
});
}
return;
}
if (!mounted) return;
if (_uploading) setState(() => _uploading = false);
// Keep the caret in the field while the prompt is in flight, as before.
_focus.requestFocus();
final accepted = await composer.send(attachments: parts);
@@ -432,7 +466,7 @@ class _ChatComposerBarState extends State<ChatComposerBar> {
/// Adds each file through the landed attachment draft. A refusal keeps every
/// attachment already in the draft and shows the draft's own words; the
/// refused file's bytes are never retained. Picking and pasting share this
/// path so both apply the same name and 1 MiB decisions.
/// path so both apply the same name and size decisions.
void _addFiles(List<ChatPickedFile> files) {
if (!mounted) return;
setState(() {
@@ -527,6 +561,15 @@ class _ChatComposerBarState extends State<ChatComposerBar> {
),
),
if (!_attachments.isEmpty) _attachmentStrip(context, cupertino),
if (_uploading)
_notice(
context,
cupertino,
key: const Key('chat-composer-uploading'),
text: _attachments.attachments.length == 1
? 'Uploading 1 file…'
: 'Uploading ${_attachments.attachments.length} files…',
),
if (_attachmentError != null)
_notice(
context,
@@ -861,7 +904,7 @@ class _ChatComposerBarState extends State<ChatComposerBar> {
final stop = widget.running;
final enabled = stop
? !composer.stopping
: !composer.sending && composer.draft.trim().isNotEmpty;
: !_uploading && !composer.sending && composer.draft.trim().isNotEmpty;
void press() {
if (stop) {
unawaited(composer.stop());
@@ -0,0 +1,81 @@
import 'dart:convert';
import 'dart:typed_data';
import '../../../../core/transport/server_url.dart';
import '../../domain/chat_attachment_draft.dart';
import '../../domain/chat_attachment_wire.dart';
import 'chat_upload_stub.dart'
if (dart.library.io) 'chat_upload_io.dart'
if (dart.library.js_interop) 'chat_upload_web.dart'
as platform;
/// The server's reply to one stored chat file.
typedef ChatUploadReply = ({int status, String body});
/// Posts one file to the server's `/chat-uploads`: the bytes travel as their
/// own HTTP request, outside the websocket frame, and the reply names the
/// stored file a prompt then references.
Future<ChatUploadReply> postChatUpload(
Uri uri, {
required String name,
required String mime,
required Uint8List bytes,
}) => platform.postChatUpload(uri, name: name, mime: mime, bytes: bytes);
/// The header that carries the file name; the server refuses a post without it.
const chatUploadNameHeader = 'X-Pipelines-Upload-Name';
/// Uploads [bytes] for [attachment] to the server behind [endpoint] and
/// returns the stored name a prompt references. Every failure is a
/// [ChatAttachmentWireException] in words the composer shows as they are.
Future<String> uploadChatAttachment(
String endpoint,
ChatAttachment attachment,
Uint8List bytes,
) async {
final uri = ServerUrl.resolveArtifactUrl(endpoint, '/chat-uploads');
if (uri == null) {
throw const ChatAttachmentWireException(
'No server is configured to upload files to.',
);
}
final ChatUploadReply reply;
try {
reply = await postChatUpload(
uri,
name: attachment.name.trim().isEmpty ? 'attachment' : attachment.name,
mime: attachment.mime.trim().isEmpty
? 'application/octet-stream'
: attachment.mime,
bytes: bytes,
);
} catch (error) {
throw ChatAttachmentWireException(
'"${attachment.name}" could not be uploaded: $error',
);
}
if (reply.status == 401) {
throw const ChatAttachmentWireException(
'Sign in to upload files (Settings → Sign in).',
);
}
if (reply.status != 200) {
final reason = reply.body.trim();
throw ChatAttachmentWireException(
reason.isEmpty
? '"${attachment.name}" could not be uploaded (HTTP ${reply.status}).'
: '"${attachment.name}" could not be uploaded: $reason',
);
}
try {
final decoded = jsonDecode(reply.body);
final upload = decoded is Map ? decoded['upload'] : null;
if (upload is String && upload.isNotEmpty) return upload;
} on FormatException {
// Reported below.
}
throw ChatAttachmentWireException(
'"${attachment.name}" upload returned no file name.',
);
}
@@ -0,0 +1,31 @@
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import '../../../../core/session/server_session.dart';
import 'chat_upload.dart';
Future<ChatUploadReply> postChatUpload(
Uri uri, {
required String name,
required String mime,
required Uint8List bytes,
}) async {
final client = HttpClient()..connectionTimeout = const Duration(seconds: 15);
try {
final request = await client.postUrl(uri);
request.followRedirects = false;
// A phone behind the web login carries its kept session.
ServerSession.headers.forEach(request.headers.set);
request.headers
..set(chatUploadNameHeader, Uri.encodeComponent(name))
..set(HttpHeaders.contentTypeHeader, mime)
..contentLength = bytes.length;
request.add(bytes);
final response = await request.close();
final body = await response.transform(utf8.decoder).join();
return (status: response.statusCode, body: body);
} finally {
client.close(force: true);
}
}
@@ -0,0 +1,10 @@
import 'dart:typed_data';
import 'chat_upload.dart';
Future<ChatUploadReply> postChatUpload(
Uri uri, {
required String name,
required String mime,
required Uint8List bytes,
}) => throw UnsupportedError('Uploads are not supported on this platform.');
@@ -0,0 +1,46 @@
import 'dart:js_interop';
import 'dart:typed_data';
import 'chat_upload.dart';
@JS('fetch')
external JSPromise<_Response> _fetch(JSString input, _Init init);
extension type _Init._(JSObject _) implements JSObject {
external factory _Init({
JSString method,
JSAny headers,
JSAny body,
JSString credentials,
});
}
extension type _Response._(JSObject _) implements JSObject {
external int get status;
external JSPromise<JSString> text();
}
/// The browser keeps the login cookie; `same-origin` sends it only to the
/// page's own server.
Future<ChatUploadReply> postChatUpload(
Uri uri, {
required String name,
required String mime,
required Uint8List bytes,
}) async {
final response = await _fetch(
uri.toString().toJS,
_Init(
method: 'POST'.toJS,
headers: {
chatUploadNameHeader: Uri.encodeComponent(name),
'Content-Type': mime,
}.jsify()!,
body: bytes.toJS,
credentials: 'same-origin'.toJS,
),
).toDart;
final body = (await response.text().toDart).toDart;
return (status: response.status, body: body);
}
@@ -39,7 +39,7 @@ enum ChatAttachmentRefusal {
/// Keeping this attachment would take the draft past
/// [ChatAttachmentDraft.maxTotalBytes]. The wording matches the web composer.
totalTooLarge('Attachments are too large for web chat (1 MiB total).');
totalTooLarge('Attachments are too large (100 MiB total).');
const ChatAttachmentRefusal(this.message);
@@ -48,13 +48,14 @@ enum ChatAttachmentRefusal {
/// The files attached to one conversation's next message.
///
/// Mirrors the web composer's 1 MiB total budget across attachments: an
/// Files upload to the server one request each (`/chat-uploads`, 100 MiB a
/// file), so the draft allows 100 MiB in total across attachments: an
/// attachment that would exceed it is refused with its own reason, and every
/// attachment already in the draft stays. Attachments are memory-only; nothing
/// here persists, reads file bytes or sends a prompt.
class ChatAttachmentDraft {
/// The largest total the web composer allows across all attachments.
static const int maxTotalBytes = 1 << 20;
static const int maxTotalBytes = 100 << 20;
final List<ChatAttachment> _attachments = [];
@@ -8,6 +8,10 @@ import 'chat_attachment_draft.dart';
/// server applies the same bound.
const int maxChatAttachmentCount = 8;
/// The inline route's budget: base64 bytes inside the 2 MiB websocket frame.
/// Uploaded files ([chatAttachmentUploadParts]) are not bound by it.
const int maxInlineAttachmentBytes = 1 << 20;
/// The web's fallback name when an attachment's recorded name is blank.
const String chatAttachmentFallbackName = 'attachment';
@@ -45,7 +49,7 @@ typedef ChatAttachmentBytes = Uint8List? Function(ChatAttachment attachment);
/// at all; the web omits it for a text-only prompt. The count and the actual
/// byte lengths are checked before any part is encoded: more than
/// [maxChatAttachmentCount] files, an unreadable file, or a total over
/// [ChatAttachmentDraft.maxTotalBytes] throws [ChatAttachmentWireException].
/// [maxInlineAttachmentBytes] throws [ChatAttachmentWireException].
/// Actual bytes are used, not the recorded size, matching the web reader.
List<JsonMap> chatAttachmentParts(
ChatAttachmentDraft draft, {
@@ -71,9 +75,9 @@ List<JsonMap> chatAttachmentParts(
);
}
total += bytes.length;
if (total > ChatAttachmentDraft.maxTotalBytes) {
if (total > maxInlineAttachmentBytes) {
throw const ChatAttachmentWireException(
'Attachments are too large for web chat (1 MiB total).',
'Attachments are too large to send without a server (1 MiB total).',
);
}
contents.add(bytes);
@@ -98,6 +102,43 @@ List<JsonMap> chatAttachmentParts(
/// history, so this reports no decoded bytes and its
/// [ChatAttachment.sizeBytes] is 0 (or the record's own numeric `size` when a
/// caller supplied one). Nothing here builds UI or keeps file contents.
/// Stores one file on the server and returns its upload name.
typedef ChatAttachmentUpload =
Future<String> Function(ChatAttachment attachment, Uint8List bytes);
/// The prompt parts for files sent ahead through [upload]: each part names the
/// stored file instead of carrying its bytes, so size is bound by the server's
/// upload limit, not the websocket frame. Throws [ChatAttachmentWireException]
/// for an unreadable file; [upload]'s own exceptions pass through.
Future<List<JsonMap>> chatAttachmentUploadParts(
ChatAttachmentDraft draft, {
required ChatAttachmentBytes read,
required ChatAttachmentUpload upload,
}) async {
final attachments = draft.attachments;
if (attachments.isEmpty) return const [];
if (attachments.length > maxChatAttachmentCount) {
throw const ChatAttachmentWireException(
'At most 8 attachments are allowed.',
);
}
final parts = <JsonMap>[];
for (final attachment in attachments) {
final bytes = read(attachment);
if (bytes == null) {
throw ChatAttachmentWireException(
'"${chatAttachmentDisplayName(attachment.name, '')}" could not be read.',
);
}
parts.add({
'name': _wireText(attachment.name, chatAttachmentFallbackName),
'mime': _wireText(attachment.mime, chatAttachmentFallbackMime),
'upload': await upload(attachment, bytes),
});
}
return parts;
}
ChatAttachment? chatAttachmentFromHistoryPart(Object? part) {
if (part is! Map) return null;
if ('${part['type'] ?? ''}'.trim() != 'file') return null;
@@ -50,7 +50,11 @@ ChatPickedFile _file(
String name,
List<int> bytes, {
String mime = 'text/plain',
}) => ChatPickedFile(name: name, mime: mime, bytes: Uint8List.fromList(bytes));
}) => ChatPickedFile(
name: name,
mime: mime,
bytes: bytes is Uint8List ? bytes : Uint8List.fromList(bytes),
);
Finder get _field => find.byKey(const Key('chat-composer-field'));
@@ -159,7 +163,7 @@ void main() {
fake.paste([
_file(
'huge.bin',
List<int>.filled(ChatAttachmentDraft.maxTotalBytes + 1, 0),
Uint8List(ChatAttachmentDraft.maxTotalBytes + 1),
mime: 'application/octet-stream',
),
]);
@@ -184,7 +188,7 @@ void main() {
_file('kept.txt', [1]),
_file(
'huge.bin',
List<int>.filled(ChatAttachmentDraft.maxTotalBytes, 0),
Uint8List(ChatAttachmentDraft.maxTotalBytes),
mime: 'application/octet-stream',
),
]);
@@ -42,10 +42,15 @@ void main() {
expect(draft.attachments, [second]);
});
test('accepts exactly 1 MiB in total', () {
test('accepts exactly 100 MiB in total', () {
final draft = ChatAttachmentDraft();
expect(draft.add(attachment('large.bin', 700000)), isNull);
expect(
draft.add(
attachment('large.bin', ChatAttachmentDraft.maxTotalBytes - 348576),
),
isNull,
);
expect(draft.add(attachment('rest.bin', 348576)), isNull);
expect(draft.totalBytes, ChatAttachmentDraft.maxTotalBytes);
expect(draft.attachments, hasLength(2));
@@ -59,10 +64,7 @@ void main() {
final refusal = draft.add(attachment('extra.bin', 1));
expect(refusal, ChatAttachmentRefusal.totalTooLarge);
expect(
refusal!.message,
'Attachments are too large for web chat (1 MiB total).',
);
expect(refusal!.message, 'Attachments are too large (100 MiB total).');
expect(draft.attachments, [kept]);
expect(draft.totalBytes, ChatAttachmentDraft.maxTotalBytes);
});
@@ -121,14 +121,14 @@ void main() {
draft,
read: (_) {
reads += 1;
return Uint8List(ChatAttachmentDraft.maxTotalBytes + 1);
return Uint8List(maxInlineAttachmentBytes + 1);
},
),
throwsA(
isA<ChatAttachmentWireException>().having(
(error) => error.message,
'message',
'Attachments are too large for web chat (1 MiB total).',
'Attachments are too large to send without a server (1 MiB total).',
),
),
);
@@ -190,4 +190,35 @@ void main() {
isNull,
);
});
test(
'uploaded files are named in their parts, not carried as bytes',
() async {
final draft = ChatAttachmentDraft();
final big = ChatAttachment(
name: 'photo.png',
mime: 'image/png',
sizeBytes: maxInlineAttachmentBytes + 1,
);
draft.add(big);
final uploaded = <String>[];
final parts = await chatAttachmentUploadParts(
draft,
read: (_) => Uint8List(maxInlineAttachmentBytes + 1),
upload: (attachment, bytes) async {
uploaded.add(attachment.name);
return 'up-0123-${attachment.name}';
},
);
expect(uploaded, ['photo.png']);
expect(parts, [
{
'name': 'photo.png',
'mime': 'image/png',
'upload': 'up-0123-photo.png',
},
]);
},
);
}
+1 -1
View File
@@ -123,7 +123,7 @@ export async function start({ root = resolve('build/web'), upstream = 'http://12
if (!['GET', 'HEAD', 'POST'].includes(req.method)) {
res.setHeader('Allow', 'GET, HEAD, POST'); return reply(res, 405);
}
if (pathname === '/login' || pathname === '/logout' || pathname.startsWith('/artifacts/')) return proxy(req, res, target);
if (pathname === '/login' || pathname === '/logout' || pathname === '/chat-uploads' || pathname.startsWith('/artifacts/')) return proxy(req, res, target);
if (req.method !== 'GET' && req.method !== 'HEAD') return reply(res, 405);
authCheck(req, target, (authenticated) => {
if (res.destroyed) return;
+3
View File
@@ -325,6 +325,9 @@ type ChatAttachment struct {
Name string `json:"name"`
Mime string `json:"mime,omitempty"`
Data string `json:"data"`
// Upload names a file already sent through POST /chat-uploads; Data is
// then empty. Large files take this route instead of the websocket.
Upload string `json:"upload,omitempty"`
}
// ChatMessage is one stored turn returned by chat_messages. A user turn is a
+92 -48
View File
@@ -18,7 +18,7 @@ import (
// Files survive process restarts and asynchronous harness inbox admission.
// Only unleased files older than seven days are collected, on the next upload.
const chatAttachmentRetention = 7 * 24 * time.Hour
const chatAttachmentQuota = 64 << 20
const chatAttachmentQuota = 1 << 30
const chatAttachmentFileQuota = 512
var chatAttachmentStore = struct {
@@ -59,69 +59,54 @@ func prepareChatAttachments(sessionID string, attachments []protocol.ChatAttachm
if len(attachments) > maxChatAttachments {
return nil, nil, fmt.Errorf("at most %d attachments are allowed", maxChatAttachments)
}
dir := os.Getenv("PIPELINES_CHAT_ATTACHMENT_DIR")
if dir == "" {
home, err := os.UserHomeDir()
if err != nil {
return nil, nil, err
}
dir = filepath.Join(home, ".pipelines-chat-attachments")
}
if !filepath.IsAbs(dir) {
return nil, nil, fmt.Errorf("chat attachment storage must be an absolute path")
}
dir = filepath.Clean(dir)
chatAttachmentStore.Lock()
defer chatAttachmentStore.Unlock()
if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
return nil, nil, fmt.Errorf("prepare attachment directory: %w", err)
}
info, err := os.Lstat(dir)
if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 {
return nil, nil, fmt.Errorf("attachment storage must be a private directory, not a symlink")
}
root, err := os.OpenRoot(dir)
root, dir, stored, storedFiles, err := openChatAttachmentStore()
if err != nil {
return nil, nil, err
}
defer root.Close()
actual, err := root.Stat(".")
if err != nil || !os.SameFile(info, actual) {
return nil, nil, fmt.Errorf("chat attachment storage changed while opening")
}
entries, err := fs.ReadDir(root.FS(), ".")
if err != nil {
return nil, nil, err
}
var stored int64
var storedFiles int
for _, entry := range entries {
info, err := entry.Info()
if err != nil {
return nil, nil, err
}
if !info.Mode().IsRegular() {
continue
}
if !chatAttachmentStore.leased[filepath.Join(dir, entry.Name())] && time.Since(info.ModTime()) > chatAttachmentRetention {
if err := root.Remove(entry.Name()); err == nil {
continue
}
}
stored += info.Size()
storedFiles++
}
var files []opencode.PromptFile
var parts []protocol.ChatPart
var created []string
var created, referenced []string
cleanup := func() {
for _, path := range created {
_ = root.Remove(filepath.Base(path))
delete(chatAttachmentStore.leased, path)
}
for _, path := range referenced {
delete(chatAttachmentStore.leased, path)
}
}
total := 0
for index, attachment := range attachments {
if attachment.Upload != "" {
// A file sent earlier through POST /chat-uploads: only its stored
// name crosses the websocket.
upload := attachment.Upload
info, err := root.Lstat(upload)
if !chatUploadName.MatchString(upload) || err != nil || !info.Mode().IsRegular() {
cleanup()
return nil, nil, fmt.Errorf("attachment %q was not uploaded or has expired", attachment.Name)
}
path := filepath.Join(dir, upload)
if !chatAttachmentStore.leased[path] {
referenced = append(referenced, path)
}
chatAttachmentStore.leased[path] = true
name := safeChatAttachmentName(attachment.Name)
if strings.TrimSpace(attachment.Name) == "" {
name = safeChatAttachmentName(strings.SplitN(upload, "-", 3)[2])
}
mime := strings.TrimSpace(attachment.Mime)
if mime == "" {
mime = "application/octet-stream"
}
uri := (&url.URL{Scheme: "file", Path: path}).String()
files = append(files, opencode.NewPreparedPromptFile(uri, name))
parts = append(parts, protocol.ChatPart{Type: protocol.ChatPartFile, Name: name, Mime: mime, URI: uri})
continue
}
data := strings.TrimSpace(attachment.Data)
if len(data) > base64.StdEncoding.EncodedLen(maxChatAttachmentBytes) {
cleanup()
@@ -204,3 +189,62 @@ func safeChatAttachmentName(value string) string {
}
return name
}
// openChatAttachmentStore opens the private attachment directory, collects
// unleased files past retention and reports what the rest occupy. The caller
// holds chatAttachmentStore and closes the root.
func openChatAttachmentStore() (*os.Root, string, int64, int, error) {
dir := os.Getenv("PIPELINES_CHAT_ATTACHMENT_DIR")
if dir == "" {
home, err := os.UserHomeDir()
if err != nil {
return nil, "", 0, 0, err
}
dir = filepath.Join(home, ".pipelines-chat-attachments")
}
if !filepath.IsAbs(dir) {
return nil, "", 0, 0, fmt.Errorf("chat attachment storage must be an absolute path")
}
dir = filepath.Clean(dir)
if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
return nil, "", 0, 0, fmt.Errorf("prepare attachment directory: %w", err)
}
info, err := os.Lstat(dir)
if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 {
return nil, "", 0, 0, fmt.Errorf("attachment storage must be a private directory, not a symlink")
}
root, err := os.OpenRoot(dir)
if err != nil {
return nil, "", 0, 0, err
}
actual, err := root.Stat(".")
if err != nil || !os.SameFile(info, actual) {
root.Close()
return nil, "", 0, 0, fmt.Errorf("chat attachment storage changed while opening")
}
entries, err := fs.ReadDir(root.FS(), ".")
if err != nil {
root.Close()
return nil, "", 0, 0, err
}
var stored int64
var storedFiles int
for _, entry := range entries {
info, err := entry.Info()
if err != nil {
root.Close()
return nil, "", 0, 0, err
}
if !info.Mode().IsRegular() {
continue
}
if !chatAttachmentStore.leased[filepath.Join(dir, entry.Name())] && time.Since(info.ModTime()) > chatAttachmentRetention {
if err := root.Remove(entry.Name()); err == nil {
continue
}
}
stored += info.Size()
storedFiles++
}
return root, dir, stored, storedFiles, nil
}
+129
View File
@@ -0,0 +1,129 @@
package server
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
)
// maxChatUploadBytes bounds one file sent through POST /chat-uploads. Files
// travel over plain HTTP, outside the 2 MiB websocket frame, so a prompt can
// carry photos, logs and archives the inline base64 route cannot.
const maxChatUploadBytes = 100 << 20
// chatUploadHeader carries the file's name. Being a custom header, it also
// keeps a cross-site page from posting here without a CORS preflight, which
// neither the server nor its proxies answer.
const chatUploadHeader = "X-Pipelines-Upload-Name"
var chatUploadName = regexp.MustCompile(`^up-[0-9a-f]{32}-[^/\\]+$`)
type chatUploadReply struct {
Upload string `json:"upload"`
Name string `json:"name"`
Mime string `json:"mime"`
Size int64 `json:"size"`
}
// serveChatUpload stores the request body as one private chat attachment and
// returns the name a later chat prompt references in ChatAttachment.Upload.
func serveChatUpload(w http.ResponseWriter, r *http.Request) {
fail := func(status int, message string) {
http.Error(w, message, status)
}
rawName := r.Header.Get(chatUploadHeader)
if rawName == "" {
fail(http.StatusBadRequest, "missing "+chatUploadHeader)
return
}
if r.Header.Get("Sec-Fetch-Site") == "cross-site" {
fail(http.StatusForbidden, "cross-site upload refused")
return
}
decoded, err := url.PathUnescape(rawName)
if err != nil {
decoded = rawName
}
name := safeChatAttachmentName(decoded)
if r.ContentLength > maxChatUploadBytes {
fail(http.StatusRequestEntityTooLarge, "attachment exceeds the 100 MiB limit")
return
}
mime := strings.TrimSpace(r.Header.Get("Content-Type"))
if mime == "" {
mime = "application/octet-stream"
}
var token [16]byte
if _, err := rand.Read(token[:]); err != nil {
fail(http.StatusInternalServerError, "upload id unavailable")
return
}
stored := "up-" + hex.EncodeToString(token[:]) + "-" + name
chatAttachmentStore.Lock()
root, dir, used, usedFiles, err := openChatAttachmentStore()
if err != nil {
chatAttachmentStore.Unlock()
fail(http.StatusInternalServerError, err.Error())
return
}
defer root.Close()
if usedFiles+1 > chatAttachmentFileQuota || used+max(r.ContentLength, 0) > chatAttachmentQuota {
chatAttachmentStore.Unlock()
fail(http.StatusInsufficientStorage, "chat attachment storage quota exceeded; retry after retention cleanup")
return
}
file, err := root.OpenFile(stored, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
chatAttachmentStore.Unlock()
fail(http.StatusInternalServerError, "write attachment: "+err.Error())
return
}
// Leased while it is written so retention cannot collect it mid-upload.
path := filepath.Join(dir, stored)
chatAttachmentStore.leased[path] = true
chatAttachmentStore.Unlock()
release := func(remove bool) {
chatAttachmentStore.Lock()
delete(chatAttachmentStore.leased, path)
if remove {
_ = root.Remove(stored)
}
chatAttachmentStore.Unlock()
}
size, err := io.Copy(file, io.LimitReader(r.Body, maxChatUploadBytes+1))
if err == nil && size > maxChatUploadBytes {
err = errors.New("attachment exceeds the 100 MiB limit")
}
if err == nil {
err = file.Sync()
}
if closeErr := file.Close(); err == nil {
err = closeErr
}
if err == nil && size == 0 {
err = errors.New("attachment is empty")
}
if err != nil {
release(true)
status := http.StatusBadRequest
if size > maxChatUploadBytes {
status = http.StatusRequestEntityTooLarge
}
fail(status, err.Error())
return
}
release(false)
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
_ = json.NewEncoder(w).Encode(chatUploadReply{Upload: stored, Name: name, Mime: mime, Size: size})
}
+87
View File
@@ -0,0 +1,87 @@
package server
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func postChatUpload(t *testing.T, name, body string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader(body))
if name != "" {
req.Header.Set(chatUploadHeader, name)
}
req.Header.Set("Content-Type", "image/png")
rec := httptest.NewRecorder()
serveChatUpload(rec, req)
return rec
}
func TestChatUploadIsReferencedByAPrompt(t *testing.T) {
dir := filepath.Join(t.TempDir(), "uploads")
t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", dir)
// Larger than the inline 1 MiB route allows.
body := strings.Repeat("x", 3<<20)
rec := postChatUpload(t, "big%20photo.png", body)
if rec.Code != http.StatusOK {
t.Fatalf("upload status = %d %s", rec.Code, rec.Body)
}
var reply chatUploadReply
if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil {
t.Fatal(err)
}
if reply.Name != "big photo.png" || reply.Size != int64(len(body)) || reply.Mime != "image/png" || !chatUploadName.MatchString(reply.Upload) {
t.Fatalf("reply = %+v", reply)
}
info, err := os.Stat(filepath.Join(dir, reply.Upload))
if err != nil || info.Mode().Perm() != 0o600 || info.Size() != int64(len(body)) {
t.Fatalf("stored file = %v %v", info, err)
}
files, parts, err := prepareChatAttachments("ses_test", []protocol.ChatAttachment{{Name: "big photo.png", Mime: "image/png", Upload: reply.Upload}})
if err != nil {
t.Fatal(err)
}
if len(files) != 1 || len(parts) != 1 || parts[0].Name != "big photo.png" || !strings.Contains(parts[0].URI, reply.Upload[:35]) {
t.Fatalf("files = %+v parts = %+v", files, parts)
}
releaseChatAttachments(files)
}
func TestChatUploadRefusals(t *testing.T) {
t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", filepath.Join(t.TempDir(), "uploads"))
if rec := postChatUpload(t, "", "x"); rec.Code != http.StatusBadRequest {
t.Fatalf("missing name status = %d", rec.Code)
}
if rec := postChatUpload(t, "empty.txt", ""); rec.Code != http.StatusBadRequest {
t.Fatalf("empty status = %d", rec.Code)
}
req := httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader("x"))
req.Header.Set(chatUploadHeader, "a.txt")
req.Header.Set("Sec-Fetch-Site", "cross-site")
rec := httptest.NewRecorder()
serveChatUpload(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("cross-site status = %d", rec.Code)
}
req = httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader("x"))
req.Header.Set(chatUploadHeader, "a.txt")
req.ContentLength = maxChatUploadBytes + 1
rec = httptest.NewRecorder()
serveChatUpload(rec, req)
if rec.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized status = %d", rec.Code)
}
for _, upload := range []string{"../escape", "up-zz-x", "up-00000000000000000000000000000000-missing.txt"} {
if _, _, err := prepareChatAttachments("ses_test", []protocol.ChatAttachment{{Name: "x", Upload: upload}}); err == nil {
t.Fatalf("reference %q accepted", upload)
}
}
}
+1
View File
@@ -115,6 +115,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
}
runs.serveArtifact(w, r)
})
mux.HandleFunc("POST /chat-uploads", serveChatUpload)
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if runErr != nil {
+6
View File
@@ -386,6 +386,12 @@ const server = createServer(async (req, res) => {
proxyArtifact(req, res);
return;
}
// Chat files go to the server as their own request, outside the websocket
// frame limit; the server refuses a request without its custom name header.
if (pathname === "/chat-uploads" && req.method === "POST") {
proxyArtifact(req, res);
return;
}
if (pathname === "/favicon.ico") {
res.writeHead(204).end(); // no icon; keeps the console free of a 404
return;