From e507f554c3f8ee4b74517044f91d09aa7ed9af2a Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 23 Oct 2026 21:25:41 +0100 Subject: [PATCH] feat(chat): files upload over HTTP before the prompt, up to 100 MiB each --- .../chat/conversation/chat_composer_bar.dart | 59 +++++++- .../chat/data/upload/chat_upload.dart | 81 ++++++++++ .../chat/data/upload/chat_upload_io.dart | 31 ++++ .../chat/data/upload/chat_upload_stub.dart | 10 ++ .../chat/data/upload/chat_upload_web.dart | 46 ++++++ .../chat/domain/chat_attachment_draft.dart | 7 +- .../chat/domain/chat_attachment_wire.dart | 47 +++++- .../chat_composer_paste_test.dart | 10 +- .../domain/chat_attachment_draft_test.dart | 14 +- .../domain/chat_attachment_wire_test.dart | 35 ++++- app/tool/preview/server.mjs | 2 +- internal/protocol/chat.go | 3 + internal/server/chat_attachments.go | 140 ++++++++++++------ internal/server/chat_uploads.go | 129 ++++++++++++++++ internal/server/chat_uploads_test.go | 87 +++++++++++ internal/server/websocket.go | 1 + web/serve.mjs | 6 + 17 files changed, 634 insertions(+), 74 deletions(-) create mode 100644 app/lib/features/chat/data/upload/chat_upload.dart create mode 100644 app/lib/features/chat/data/upload/chat_upload_io.dart create mode 100644 app/lib/features/chat/data/upload/chat_upload_stub.dart create mode 100644 app/lib/features/chat/data/upload/chat_upload_web.dart create mode 100644 internal/server/chat_uploads.go create mode 100644 internal/server/chat_uploads_test.go diff --git a/app/lib/features/chat/conversation/chat_composer_bar.dart b/app/lib/features/chat/conversation/chat_composer_bar.dart index bf8bd5cc..b0b59c15 100644 --- a/app/lib/features/chat/conversation/chat_composer_bar.dart +++ b/app/lib/features/chat/conversation/chat_composer_bar.dart @@ -6,6 +6,8 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import '../../../core/api_gateway.dart'; +import '../../../core/transport/pipelines_client.dart'; +import '../data/upload/chat_upload.dart'; import '../domain/chat_attachment_draft.dart'; import '../domain/chat_attachment_picker.dart'; import '../domain/chat_attachment_wire.dart'; @@ -101,6 +103,13 @@ class _ChatComposerBarState extends State { final List _bytes = []; String? _attachmentError; + /// True while the draft's files upload ahead of the prompt. + bool _uploading = false; + + /// Upload names by attachment instance, so a retried send reuses the files + /// already stored instead of uploading them again. + final Expando _uploaded = Expando(); + /// The draft text whose palette the operator closed (Escape) or completed /// (a row or Enter). The palette stays hidden while the draft still equals /// it, so any further typing offers the commands again. @@ -392,14 +401,39 @@ class _ChatComposerBarState extends State { /// accepted prompt clears them. Future _send() async { final composer = widget.composer; - if (composer.sending || composer.draft.trim().isEmpty) return; - List parts; - try { - parts = chatAttachmentParts(_attachments, read: _readAttachment); - } on ChatAttachmentWireException catch (error) { - setState(() => _attachmentError = error.message); + if (_uploading || composer.sending || composer.draft.trim().isEmpty) { return; } + List parts; + final endpoint = configuredGatewayEndpoint(composer.gateway); + try { + if (endpoint.isEmpty || _attachments.isEmpty) { + parts = chatAttachmentParts(_attachments, read: _readAttachment); + } else { + // Each file goes to the server as its own request first; the prompt + // then names the stored files. + setState(() { + _uploading = true; + _attachmentError = null; + }); + parts = await chatAttachmentUploadParts( + _attachments, + read: _readAttachment, + upload: (attachment, bytes) async => _uploaded[attachment] ??= + await uploadChatAttachment(endpoint, attachment, bytes), + ); + } + } on ChatAttachmentWireException catch (error) { + if (mounted) { + setState(() { + _uploading = false; + _attachmentError = error.message; + }); + } + return; + } + if (!mounted) return; + if (_uploading) setState(() => _uploading = false); // Keep the caret in the field while the prompt is in flight, as before. _focus.requestFocus(); final accepted = await composer.send(attachments: parts); @@ -432,7 +466,7 @@ class _ChatComposerBarState extends State { /// Adds each file through the landed attachment draft. A refusal keeps every /// attachment already in the draft and shows the draft's own words; the /// refused file's bytes are never retained. Picking and pasting share this - /// path so both apply the same name and 1 MiB decisions. + /// path so both apply the same name and size decisions. void _addFiles(List files) { if (!mounted) return; setState(() { @@ -527,6 +561,15 @@ class _ChatComposerBarState extends State { ), ), if (!_attachments.isEmpty) _attachmentStrip(context, cupertino), + if (_uploading) + _notice( + context, + cupertino, + key: const Key('chat-composer-uploading'), + text: _attachments.attachments.length == 1 + ? 'Uploading 1 file…' + : 'Uploading ${_attachments.attachments.length} files…', + ), if (_attachmentError != null) _notice( context, @@ -861,7 +904,7 @@ class _ChatComposerBarState extends State { final stop = widget.running; final enabled = stop ? !composer.stopping - : !composer.sending && composer.draft.trim().isNotEmpty; + : !_uploading && !composer.sending && composer.draft.trim().isNotEmpty; void press() { if (stop) { unawaited(composer.stop()); diff --git a/app/lib/features/chat/data/upload/chat_upload.dart b/app/lib/features/chat/data/upload/chat_upload.dart new file mode 100644 index 00000000..481e07d1 --- /dev/null +++ b/app/lib/features/chat/data/upload/chat_upload.dart @@ -0,0 +1,81 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import '../../../../core/transport/server_url.dart'; +import '../../domain/chat_attachment_draft.dart'; +import '../../domain/chat_attachment_wire.dart'; + +import 'chat_upload_stub.dart' + if (dart.library.io) 'chat_upload_io.dart' + if (dart.library.js_interop) 'chat_upload_web.dart' + as platform; + +/// The server's reply to one stored chat file. +typedef ChatUploadReply = ({int status, String body}); + +/// Posts one file to the server's `/chat-uploads`: the bytes travel as their +/// own HTTP request, outside the websocket frame, and the reply names the +/// stored file a prompt then references. +Future postChatUpload( + Uri uri, { + required String name, + required String mime, + required Uint8List bytes, +}) => platform.postChatUpload(uri, name: name, mime: mime, bytes: bytes); + +/// The header that carries the file name; the server refuses a post without it. +const chatUploadNameHeader = 'X-Pipelines-Upload-Name'; + +/// Uploads [bytes] for [attachment] to the server behind [endpoint] and +/// returns the stored name a prompt references. Every failure is a +/// [ChatAttachmentWireException] in words the composer shows as they are. +Future uploadChatAttachment( + String endpoint, + ChatAttachment attachment, + Uint8List bytes, +) async { + final uri = ServerUrl.resolveArtifactUrl(endpoint, '/chat-uploads'); + if (uri == null) { + throw const ChatAttachmentWireException( + 'No server is configured to upload files to.', + ); + } + final ChatUploadReply reply; + try { + reply = await postChatUpload( + uri, + name: attachment.name.trim().isEmpty ? 'attachment' : attachment.name, + mime: attachment.mime.trim().isEmpty + ? 'application/octet-stream' + : attachment.mime, + bytes: bytes, + ); + } catch (error) { + throw ChatAttachmentWireException( + '"${attachment.name}" could not be uploaded: $error', + ); + } + if (reply.status == 401) { + throw const ChatAttachmentWireException( + 'Sign in to upload files (Settings → Sign in).', + ); + } + if (reply.status != 200) { + final reason = reply.body.trim(); + throw ChatAttachmentWireException( + reason.isEmpty + ? '"${attachment.name}" could not be uploaded (HTTP ${reply.status}).' + : '"${attachment.name}" could not be uploaded: $reason', + ); + } + try { + final decoded = jsonDecode(reply.body); + final upload = decoded is Map ? decoded['upload'] : null; + if (upload is String && upload.isNotEmpty) return upload; + } on FormatException { + // Reported below. + } + throw ChatAttachmentWireException( + '"${attachment.name}" upload returned no file name.', + ); +} diff --git a/app/lib/features/chat/data/upload/chat_upload_io.dart b/app/lib/features/chat/data/upload/chat_upload_io.dart new file mode 100644 index 00000000..56068d87 --- /dev/null +++ b/app/lib/features/chat/data/upload/chat_upload_io.dart @@ -0,0 +1,31 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import '../../../../core/session/server_session.dart'; +import 'chat_upload.dart'; + +Future postChatUpload( + Uri uri, { + required String name, + required String mime, + required Uint8List bytes, +}) async { + final client = HttpClient()..connectionTimeout = const Duration(seconds: 15); + try { + final request = await client.postUrl(uri); + request.followRedirects = false; + // A phone behind the web login carries its kept session. + ServerSession.headers.forEach(request.headers.set); + request.headers + ..set(chatUploadNameHeader, Uri.encodeComponent(name)) + ..set(HttpHeaders.contentTypeHeader, mime) + ..contentLength = bytes.length; + request.add(bytes); + final response = await request.close(); + final body = await response.transform(utf8.decoder).join(); + return (status: response.statusCode, body: body); + } finally { + client.close(force: true); + } +} diff --git a/app/lib/features/chat/data/upload/chat_upload_stub.dart b/app/lib/features/chat/data/upload/chat_upload_stub.dart new file mode 100644 index 00000000..7c4277e1 --- /dev/null +++ b/app/lib/features/chat/data/upload/chat_upload_stub.dart @@ -0,0 +1,10 @@ +import 'dart:typed_data'; + +import 'chat_upload.dart'; + +Future postChatUpload( + Uri uri, { + required String name, + required String mime, + required Uint8List bytes, +}) => throw UnsupportedError('Uploads are not supported on this platform.'); diff --git a/app/lib/features/chat/data/upload/chat_upload_web.dart b/app/lib/features/chat/data/upload/chat_upload_web.dart new file mode 100644 index 00000000..1ad6a14b --- /dev/null +++ b/app/lib/features/chat/data/upload/chat_upload_web.dart @@ -0,0 +1,46 @@ +import 'dart:js_interop'; +import 'dart:typed_data'; + +import 'chat_upload.dart'; + +@JS('fetch') +external JSPromise<_Response> _fetch(JSString input, _Init init); + +extension type _Init._(JSObject _) implements JSObject { + external factory _Init({ + JSString method, + JSAny headers, + JSAny body, + JSString credentials, + }); +} + +extension type _Response._(JSObject _) implements JSObject { + external int get status; + + external JSPromise text(); +} + +/// The browser keeps the login cookie; `same-origin` sends it only to the +/// page's own server. +Future postChatUpload( + Uri uri, { + required String name, + required String mime, + required Uint8List bytes, +}) async { + final response = await _fetch( + uri.toString().toJS, + _Init( + method: 'POST'.toJS, + headers: { + chatUploadNameHeader: Uri.encodeComponent(name), + 'Content-Type': mime, + }.jsify()!, + body: bytes.toJS, + credentials: 'same-origin'.toJS, + ), + ).toDart; + final body = (await response.text().toDart).toDart; + return (status: response.status, body: body); +} diff --git a/app/lib/features/chat/domain/chat_attachment_draft.dart b/app/lib/features/chat/domain/chat_attachment_draft.dart index 4942ce36..d8da524e 100644 --- a/app/lib/features/chat/domain/chat_attachment_draft.dart +++ b/app/lib/features/chat/domain/chat_attachment_draft.dart @@ -39,7 +39,7 @@ enum ChatAttachmentRefusal { /// Keeping this attachment would take the draft past /// [ChatAttachmentDraft.maxTotalBytes]. The wording matches the web composer. - totalTooLarge('Attachments are too large for web chat (1 MiB total).'); + totalTooLarge('Attachments are too large (100 MiB total).'); const ChatAttachmentRefusal(this.message); @@ -48,13 +48,14 @@ enum ChatAttachmentRefusal { /// The files attached to one conversation's next message. /// -/// Mirrors the web composer's 1 MiB total budget across attachments: an +/// Files upload to the server one request each (`/chat-uploads`, 100 MiB a +/// file), so the draft allows 100 MiB in total across attachments: an /// attachment that would exceed it is refused with its own reason, and every /// attachment already in the draft stays. Attachments are memory-only; nothing /// here persists, reads file bytes or sends a prompt. class ChatAttachmentDraft { /// The largest total the web composer allows across all attachments. - static const int maxTotalBytes = 1 << 20; + static const int maxTotalBytes = 100 << 20; final List _attachments = []; diff --git a/app/lib/features/chat/domain/chat_attachment_wire.dart b/app/lib/features/chat/domain/chat_attachment_wire.dart index 4a429921..cfa8e1c5 100644 --- a/app/lib/features/chat/domain/chat_attachment_wire.dart +++ b/app/lib/features/chat/domain/chat_attachment_wire.dart @@ -8,6 +8,10 @@ import 'chat_attachment_draft.dart'; /// server applies the same bound. const int maxChatAttachmentCount = 8; +/// The inline route's budget: base64 bytes inside the 2 MiB websocket frame. +/// Uploaded files ([chatAttachmentUploadParts]) are not bound by it. +const int maxInlineAttachmentBytes = 1 << 20; + /// The web's fallback name when an attachment's recorded name is blank. const String chatAttachmentFallbackName = 'attachment'; @@ -45,7 +49,7 @@ typedef ChatAttachmentBytes = Uint8List? Function(ChatAttachment attachment); /// at all; the web omits it for a text-only prompt. The count and the actual /// byte lengths are checked before any part is encoded: more than /// [maxChatAttachmentCount] files, an unreadable file, or a total over -/// [ChatAttachmentDraft.maxTotalBytes] throws [ChatAttachmentWireException]. +/// [maxInlineAttachmentBytes] throws [ChatAttachmentWireException]. /// Actual bytes are used, not the recorded size, matching the web reader. List chatAttachmentParts( ChatAttachmentDraft draft, { @@ -71,9 +75,9 @@ List chatAttachmentParts( ); } total += bytes.length; - if (total > ChatAttachmentDraft.maxTotalBytes) { + if (total > maxInlineAttachmentBytes) { throw const ChatAttachmentWireException( - 'Attachments are too large for web chat (1 MiB total).', + 'Attachments are too large to send without a server (1 MiB total).', ); } contents.add(bytes); @@ -98,6 +102,43 @@ List chatAttachmentParts( /// history, so this reports no decoded bytes and its /// [ChatAttachment.sizeBytes] is 0 (or the record's own numeric `size` when a /// caller supplied one). Nothing here builds UI or keeps file contents. +/// Stores one file on the server and returns its upload name. +typedef ChatAttachmentUpload = + Future Function(ChatAttachment attachment, Uint8List bytes); + +/// The prompt parts for files sent ahead through [upload]: each part names the +/// stored file instead of carrying its bytes, so size is bound by the server's +/// upload limit, not the websocket frame. Throws [ChatAttachmentWireException] +/// for an unreadable file; [upload]'s own exceptions pass through. +Future> chatAttachmentUploadParts( + ChatAttachmentDraft draft, { + required ChatAttachmentBytes read, + required ChatAttachmentUpload upload, +}) async { + final attachments = draft.attachments; + if (attachments.isEmpty) return const []; + if (attachments.length > maxChatAttachmentCount) { + throw const ChatAttachmentWireException( + 'At most 8 attachments are allowed.', + ); + } + final parts = []; + for (final attachment in attachments) { + final bytes = read(attachment); + if (bytes == null) { + throw ChatAttachmentWireException( + '"${chatAttachmentDisplayName(attachment.name, '')}" could not be read.', + ); + } + parts.add({ + 'name': _wireText(attachment.name, chatAttachmentFallbackName), + 'mime': _wireText(attachment.mime, chatAttachmentFallbackMime), + 'upload': await upload(attachment, bytes), + }); + } + return parts; +} + ChatAttachment? chatAttachmentFromHistoryPart(Object? part) { if (part is! Map) return null; if ('${part['type'] ?? ''}'.trim() != 'file') return null; diff --git a/app/test/features/chat/conversation/chat_composer_paste_test.dart b/app/test/features/chat/conversation/chat_composer_paste_test.dart index 0a6bcb01..2a9f2e91 100644 --- a/app/test/features/chat/conversation/chat_composer_paste_test.dart +++ b/app/test/features/chat/conversation/chat_composer_paste_test.dart @@ -50,7 +50,11 @@ ChatPickedFile _file( String name, List bytes, { String mime = 'text/plain', -}) => ChatPickedFile(name: name, mime: mime, bytes: Uint8List.fromList(bytes)); +}) => ChatPickedFile( + name: name, + mime: mime, + bytes: bytes is Uint8List ? bytes : Uint8List.fromList(bytes), +); Finder get _field => find.byKey(const Key('chat-composer-field')); @@ -159,7 +163,7 @@ void main() { fake.paste([ _file( 'huge.bin', - List.filled(ChatAttachmentDraft.maxTotalBytes + 1, 0), + Uint8List(ChatAttachmentDraft.maxTotalBytes + 1), mime: 'application/octet-stream', ), ]); @@ -184,7 +188,7 @@ void main() { _file('kept.txt', [1]), _file( 'huge.bin', - List.filled(ChatAttachmentDraft.maxTotalBytes, 0), + Uint8List(ChatAttachmentDraft.maxTotalBytes), mime: 'application/octet-stream', ), ]); diff --git a/app/test/features/chat/domain/chat_attachment_draft_test.dart b/app/test/features/chat/domain/chat_attachment_draft_test.dart index c4d35213..5636791a 100644 --- a/app/test/features/chat/domain/chat_attachment_draft_test.dart +++ b/app/test/features/chat/domain/chat_attachment_draft_test.dart @@ -42,10 +42,15 @@ void main() { expect(draft.attachments, [second]); }); - test('accepts exactly 1 MiB in total', () { + test('accepts exactly 100 MiB in total', () { final draft = ChatAttachmentDraft(); - expect(draft.add(attachment('large.bin', 700000)), isNull); + expect( + draft.add( + attachment('large.bin', ChatAttachmentDraft.maxTotalBytes - 348576), + ), + isNull, + ); expect(draft.add(attachment('rest.bin', 348576)), isNull); expect(draft.totalBytes, ChatAttachmentDraft.maxTotalBytes); expect(draft.attachments, hasLength(2)); @@ -59,10 +64,7 @@ void main() { final refusal = draft.add(attachment('extra.bin', 1)); expect(refusal, ChatAttachmentRefusal.totalTooLarge); - expect( - refusal!.message, - 'Attachments are too large for web chat (1 MiB total).', - ); + expect(refusal!.message, 'Attachments are too large (100 MiB total).'); expect(draft.attachments, [kept]); expect(draft.totalBytes, ChatAttachmentDraft.maxTotalBytes); }); diff --git a/app/test/features/chat/domain/chat_attachment_wire_test.dart b/app/test/features/chat/domain/chat_attachment_wire_test.dart index ee5cf497..784ba884 100644 --- a/app/test/features/chat/domain/chat_attachment_wire_test.dart +++ b/app/test/features/chat/domain/chat_attachment_wire_test.dart @@ -121,14 +121,14 @@ void main() { draft, read: (_) { reads += 1; - return Uint8List(ChatAttachmentDraft.maxTotalBytes + 1); + return Uint8List(maxInlineAttachmentBytes + 1); }, ), throwsA( isA().having( (error) => error.message, 'message', - 'Attachments are too large for web chat (1 MiB total).', + 'Attachments are too large to send without a server (1 MiB total).', ), ), ); @@ -190,4 +190,35 @@ void main() { isNull, ); }); + + test( + 'uploaded files are named in their parts, not carried as bytes', + () async { + final draft = ChatAttachmentDraft(); + final big = ChatAttachment( + name: 'photo.png', + mime: 'image/png', + sizeBytes: maxInlineAttachmentBytes + 1, + ); + draft.add(big); + final uploaded = []; + final parts = await chatAttachmentUploadParts( + draft, + read: (_) => Uint8List(maxInlineAttachmentBytes + 1), + upload: (attachment, bytes) async { + uploaded.add(attachment.name); + return 'up-0123-${attachment.name}'; + }, + ); + + expect(uploaded, ['photo.png']); + expect(parts, [ + { + 'name': 'photo.png', + 'mime': 'image/png', + 'upload': 'up-0123-photo.png', + }, + ]); + }, + ); } diff --git a/app/tool/preview/server.mjs b/app/tool/preview/server.mjs index abc8e55e..824b6c8c 100644 --- a/app/tool/preview/server.mjs +++ b/app/tool/preview/server.mjs @@ -123,7 +123,7 @@ export async function start({ root = resolve('build/web'), upstream = 'http://12 if (!['GET', 'HEAD', 'POST'].includes(req.method)) { res.setHeader('Allow', 'GET, HEAD, POST'); return reply(res, 405); } - if (pathname === '/login' || pathname === '/logout' || pathname.startsWith('/artifacts/')) return proxy(req, res, target); + if (pathname === '/login' || pathname === '/logout' || pathname === '/chat-uploads' || pathname.startsWith('/artifacts/')) return proxy(req, res, target); if (req.method !== 'GET' && req.method !== 'HEAD') return reply(res, 405); authCheck(req, target, (authenticated) => { if (res.destroyed) return; diff --git a/internal/protocol/chat.go b/internal/protocol/chat.go index 3816a201..d2911126 100644 --- a/internal/protocol/chat.go +++ b/internal/protocol/chat.go @@ -325,6 +325,9 @@ type ChatAttachment struct { Name string `json:"name"` Mime string `json:"mime,omitempty"` Data string `json:"data"` + // Upload names a file already sent through POST /chat-uploads; Data is + // then empty. Large files take this route instead of the websocket. + Upload string `json:"upload,omitempty"` } // ChatMessage is one stored turn returned by chat_messages. A user turn is a diff --git a/internal/server/chat_attachments.go b/internal/server/chat_attachments.go index fadb22a5..0e851582 100644 --- a/internal/server/chat_attachments.go +++ b/internal/server/chat_attachments.go @@ -18,7 +18,7 @@ import ( // Files survive process restarts and asynchronous harness inbox admission. // Only unleased files older than seven days are collected, on the next upload. const chatAttachmentRetention = 7 * 24 * time.Hour -const chatAttachmentQuota = 64 << 20 +const chatAttachmentQuota = 1 << 30 const chatAttachmentFileQuota = 512 var chatAttachmentStore = struct { @@ -59,69 +59,54 @@ func prepareChatAttachments(sessionID string, attachments []protocol.ChatAttachm if len(attachments) > maxChatAttachments { return nil, nil, fmt.Errorf("at most %d attachments are allowed", maxChatAttachments) } - dir := os.Getenv("PIPELINES_CHAT_ATTACHMENT_DIR") - if dir == "" { - home, err := os.UserHomeDir() - if err != nil { - return nil, nil, err - } - dir = filepath.Join(home, ".pipelines-chat-attachments") - } - if !filepath.IsAbs(dir) { - return nil, nil, fmt.Errorf("chat attachment storage must be an absolute path") - } - dir = filepath.Clean(dir) chatAttachmentStore.Lock() defer chatAttachmentStore.Unlock() - if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) { - return nil, nil, fmt.Errorf("prepare attachment directory: %w", err) - } - info, err := os.Lstat(dir) - if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 { - return nil, nil, fmt.Errorf("attachment storage must be a private directory, not a symlink") - } - root, err := os.OpenRoot(dir) + root, dir, stored, storedFiles, err := openChatAttachmentStore() if err != nil { return nil, nil, err } defer root.Close() - actual, err := root.Stat(".") - if err != nil || !os.SameFile(info, actual) { - return nil, nil, fmt.Errorf("chat attachment storage changed while opening") - } - entries, err := fs.ReadDir(root.FS(), ".") - if err != nil { - return nil, nil, err - } - var stored int64 - var storedFiles int - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return nil, nil, err - } - if !info.Mode().IsRegular() { - continue - } - if !chatAttachmentStore.leased[filepath.Join(dir, entry.Name())] && time.Since(info.ModTime()) > chatAttachmentRetention { - if err := root.Remove(entry.Name()); err == nil { - continue - } - } - stored += info.Size() - storedFiles++ - } var files []opencode.PromptFile var parts []protocol.ChatPart - var created []string + var created, referenced []string cleanup := func() { for _, path := range created { _ = root.Remove(filepath.Base(path)) delete(chatAttachmentStore.leased, path) } + for _, path := range referenced { + delete(chatAttachmentStore.leased, path) + } } total := 0 for index, attachment := range attachments { + if attachment.Upload != "" { + // A file sent earlier through POST /chat-uploads: only its stored + // name crosses the websocket. + upload := attachment.Upload + info, err := root.Lstat(upload) + if !chatUploadName.MatchString(upload) || err != nil || !info.Mode().IsRegular() { + cleanup() + return nil, nil, fmt.Errorf("attachment %q was not uploaded or has expired", attachment.Name) + } + path := filepath.Join(dir, upload) + if !chatAttachmentStore.leased[path] { + referenced = append(referenced, path) + } + chatAttachmentStore.leased[path] = true + name := safeChatAttachmentName(attachment.Name) + if strings.TrimSpace(attachment.Name) == "" { + name = safeChatAttachmentName(strings.SplitN(upload, "-", 3)[2]) + } + mime := strings.TrimSpace(attachment.Mime) + if mime == "" { + mime = "application/octet-stream" + } + uri := (&url.URL{Scheme: "file", Path: path}).String() + files = append(files, opencode.NewPreparedPromptFile(uri, name)) + parts = append(parts, protocol.ChatPart{Type: protocol.ChatPartFile, Name: name, Mime: mime, URI: uri}) + continue + } data := strings.TrimSpace(attachment.Data) if len(data) > base64.StdEncoding.EncodedLen(maxChatAttachmentBytes) { cleanup() @@ -204,3 +189,62 @@ func safeChatAttachmentName(value string) string { } return name } + +// openChatAttachmentStore opens the private attachment directory, collects +// unleased files past retention and reports what the rest occupy. The caller +// holds chatAttachmentStore and closes the root. +func openChatAttachmentStore() (*os.Root, string, int64, int, error) { + dir := os.Getenv("PIPELINES_CHAT_ATTACHMENT_DIR") + if dir == "" { + home, err := os.UserHomeDir() + if err != nil { + return nil, "", 0, 0, err + } + dir = filepath.Join(home, ".pipelines-chat-attachments") + } + if !filepath.IsAbs(dir) { + return nil, "", 0, 0, fmt.Errorf("chat attachment storage must be an absolute path") + } + dir = filepath.Clean(dir) + if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) { + return nil, "", 0, 0, fmt.Errorf("prepare attachment directory: %w", err) + } + info, err := os.Lstat(dir) + if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 { + return nil, "", 0, 0, fmt.Errorf("attachment storage must be a private directory, not a symlink") + } + root, err := os.OpenRoot(dir) + if err != nil { + return nil, "", 0, 0, err + } + actual, err := root.Stat(".") + if err != nil || !os.SameFile(info, actual) { + root.Close() + return nil, "", 0, 0, fmt.Errorf("chat attachment storage changed while opening") + } + entries, err := fs.ReadDir(root.FS(), ".") + if err != nil { + root.Close() + return nil, "", 0, 0, err + } + var stored int64 + var storedFiles int + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + root.Close() + return nil, "", 0, 0, err + } + if !info.Mode().IsRegular() { + continue + } + if !chatAttachmentStore.leased[filepath.Join(dir, entry.Name())] && time.Since(info.ModTime()) > chatAttachmentRetention { + if err := root.Remove(entry.Name()); err == nil { + continue + } + } + stored += info.Size() + storedFiles++ + } + return root, dir, stored, storedFiles, nil +} diff --git a/internal/server/chat_uploads.go b/internal/server/chat_uploads.go new file mode 100644 index 00000000..ab3c9516 --- /dev/null +++ b/internal/server/chat_uploads.go @@ -0,0 +1,129 @@ +package server + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" +) + +// maxChatUploadBytes bounds one file sent through POST /chat-uploads. Files +// travel over plain HTTP, outside the 2 MiB websocket frame, so a prompt can +// carry photos, logs and archives the inline base64 route cannot. +const maxChatUploadBytes = 100 << 20 + +// chatUploadHeader carries the file's name. Being a custom header, it also +// keeps a cross-site page from posting here without a CORS preflight, which +// neither the server nor its proxies answer. +const chatUploadHeader = "X-Pipelines-Upload-Name" + +var chatUploadName = regexp.MustCompile(`^up-[0-9a-f]{32}-[^/\\]+$`) + +type chatUploadReply struct { + Upload string `json:"upload"` + Name string `json:"name"` + Mime string `json:"mime"` + Size int64 `json:"size"` +} + +// serveChatUpload stores the request body as one private chat attachment and +// returns the name a later chat prompt references in ChatAttachment.Upload. +func serveChatUpload(w http.ResponseWriter, r *http.Request) { + fail := func(status int, message string) { + http.Error(w, message, status) + } + rawName := r.Header.Get(chatUploadHeader) + if rawName == "" { + fail(http.StatusBadRequest, "missing "+chatUploadHeader) + return + } + if r.Header.Get("Sec-Fetch-Site") == "cross-site" { + fail(http.StatusForbidden, "cross-site upload refused") + return + } + decoded, err := url.PathUnescape(rawName) + if err != nil { + decoded = rawName + } + name := safeChatAttachmentName(decoded) + if r.ContentLength > maxChatUploadBytes { + fail(http.StatusRequestEntityTooLarge, "attachment exceeds the 100 MiB limit") + return + } + mime := strings.TrimSpace(r.Header.Get("Content-Type")) + if mime == "" { + mime = "application/octet-stream" + } + var token [16]byte + if _, err := rand.Read(token[:]); err != nil { + fail(http.StatusInternalServerError, "upload id unavailable") + return + } + stored := "up-" + hex.EncodeToString(token[:]) + "-" + name + + chatAttachmentStore.Lock() + root, dir, used, usedFiles, err := openChatAttachmentStore() + if err != nil { + chatAttachmentStore.Unlock() + fail(http.StatusInternalServerError, err.Error()) + return + } + defer root.Close() + if usedFiles+1 > chatAttachmentFileQuota || used+max(r.ContentLength, 0) > chatAttachmentQuota { + chatAttachmentStore.Unlock() + fail(http.StatusInsufficientStorage, "chat attachment storage quota exceeded; retry after retention cleanup") + return + } + file, err := root.OpenFile(stored, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + chatAttachmentStore.Unlock() + fail(http.StatusInternalServerError, "write attachment: "+err.Error()) + return + } + // Leased while it is written so retention cannot collect it mid-upload. + path := filepath.Join(dir, stored) + chatAttachmentStore.leased[path] = true + chatAttachmentStore.Unlock() + release := func(remove bool) { + chatAttachmentStore.Lock() + delete(chatAttachmentStore.leased, path) + if remove { + _ = root.Remove(stored) + } + chatAttachmentStore.Unlock() + } + + size, err := io.Copy(file, io.LimitReader(r.Body, maxChatUploadBytes+1)) + if err == nil && size > maxChatUploadBytes { + err = errors.New("attachment exceeds the 100 MiB limit") + } + if err == nil { + err = file.Sync() + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err == nil && size == 0 { + err = errors.New("attachment is empty") + } + if err != nil { + release(true) + status := http.StatusBadRequest + if size > maxChatUploadBytes { + status = http.StatusRequestEntityTooLarge + } + fail(status, err.Error()) + return + } + release(false) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _ = json.NewEncoder(w).Encode(chatUploadReply{Upload: stored, Name: name, Mime: mime, Size: size}) +} diff --git a/internal/server/chat_uploads_test.go b/internal/server/chat_uploads_test.go new file mode 100644 index 00000000..e90e65a4 --- /dev/null +++ b/internal/server/chat_uploads_test.go @@ -0,0 +1,87 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func postChatUpload(t *testing.T, name, body string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader(body)) + if name != "" { + req.Header.Set(chatUploadHeader, name) + } + req.Header.Set("Content-Type", "image/png") + rec := httptest.NewRecorder() + serveChatUpload(rec, req) + return rec +} + +func TestChatUploadIsReferencedByAPrompt(t *testing.T) { + dir := filepath.Join(t.TempDir(), "uploads") + t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", dir) + // Larger than the inline 1 MiB route allows. + body := strings.Repeat("x", 3<<20) + rec := postChatUpload(t, "big%20photo.png", body) + if rec.Code != http.StatusOK { + t.Fatalf("upload status = %d %s", rec.Code, rec.Body) + } + var reply chatUploadReply + if err := json.Unmarshal(rec.Body.Bytes(), &reply); err != nil { + t.Fatal(err) + } + if reply.Name != "big photo.png" || reply.Size != int64(len(body)) || reply.Mime != "image/png" || !chatUploadName.MatchString(reply.Upload) { + t.Fatalf("reply = %+v", reply) + } + info, err := os.Stat(filepath.Join(dir, reply.Upload)) + if err != nil || info.Mode().Perm() != 0o600 || info.Size() != int64(len(body)) { + t.Fatalf("stored file = %v %v", info, err) + } + + files, parts, err := prepareChatAttachments("ses_test", []protocol.ChatAttachment{{Name: "big photo.png", Mime: "image/png", Upload: reply.Upload}}) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 || len(parts) != 1 || parts[0].Name != "big photo.png" || !strings.Contains(parts[0].URI, reply.Upload[:35]) { + t.Fatalf("files = %+v parts = %+v", files, parts) + } + releaseChatAttachments(files) +} + +func TestChatUploadRefusals(t *testing.T) { + t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", filepath.Join(t.TempDir(), "uploads")) + if rec := postChatUpload(t, "", "x"); rec.Code != http.StatusBadRequest { + t.Fatalf("missing name status = %d", rec.Code) + } + if rec := postChatUpload(t, "empty.txt", ""); rec.Code != http.StatusBadRequest { + t.Fatalf("empty status = %d", rec.Code) + } + req := httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader("x")) + req.Header.Set(chatUploadHeader, "a.txt") + req.Header.Set("Sec-Fetch-Site", "cross-site") + rec := httptest.NewRecorder() + serveChatUpload(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("cross-site status = %d", rec.Code) + } + req = httptest.NewRequest(http.MethodPost, "/chat-uploads", strings.NewReader("x")) + req.Header.Set(chatUploadHeader, "a.txt") + req.ContentLength = maxChatUploadBytes + 1 + rec = httptest.NewRecorder() + serveChatUpload(rec, req) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized status = %d", rec.Code) + } + for _, upload := range []string{"../escape", "up-zz-x", "up-00000000000000000000000000000000-missing.txt"} { + if _, _, err := prepareChatAttachments("ses_test", []protocol.ChatAttachment{{Name: "x", Upload: upload}}); err == nil { + t.Fatalf("reference %q accepted", upload) + } + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 7cc2204e..62e13dd9 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -115,6 +115,7 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } runs.serveArtifact(w, r) }) + mux.HandleFunc("POST /chat-uploads", serveChatUpload) mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if runErr != nil { diff --git a/web/serve.mjs b/web/serve.mjs index fd9766db..891dfd40 100644 --- a/web/serve.mjs +++ b/web/serve.mjs @@ -386,6 +386,12 @@ const server = createServer(async (req, res) => { proxyArtifact(req, res); return; } + // Chat files go to the server as their own request, outside the websocket + // frame limit; the server refuses a request without its custom name header. + if (pathname === "/chat-uploads" && req.method === "POST") { + proxyArtifact(req, res); + return; + } if (pathname === "/favicon.ico") { res.writeHead(204).end(); // no icon; keeps the console free of a 404 return;