diff --git a/docs/Organisation-Konzept.md b/docs/Organisation-Konzept.md index 9bf74a5..5f3fda4 100644 --- a/docs/Organisation-Konzept.md +++ b/docs/Organisation-Konzept.md @@ -797,6 +797,16 @@ beidem. PDF-Anhang, eine Preisliste eine Tabelle. Text aus Anhängen zu holen ist die unspektakulärste und am häufigsten gebrauchte fehlende Fähigkeit. Ein PDF vom Lieferanten ist dabei so wenig vertrauenswürdig wie eine Webseite (#193). + + > **Umgesetzt (#692) für Word, Excel und Text.** `DocumentText` im Core holt den Text + > aus `.docx` (Absätze, Tabellen als Zeilen) und `.xlsx` (je Blatt, jede Zelle an ihrer + > Spalte, gespeicherte Werte statt Formeln) sowie aus Text- und CSV-Dateien — ohne + > DTD, ohne externe Verweise, mit Obergrenze je Teil gegen Zip-Bomben. Mail listet in + > `read_message` die Anhänge, `read_attachment` liefert ihren Text, `save_attachment` + > legt die Datei im eigenen Arbeitsbereich ab (nie über eine vorhandene, nie außerhalb). + > FileRW `read`/`read_many` gibt Word- und Excel-Dateien als Text zurück; schreiben kann + > es sie weiterhin nicht. Der Text ist Fremdinhalt und wird von der Engine gerahmt. + > **Offen: PDF** — ein verlässlicher PDF-Textauszug ist eine Bibliotheksentscheidung. 3. **Der MCP-Client (#197) bleibt nützlich** — aber als *eine* Fähigkeit unter anderen, für Dienste, die ohnehin als Server bereitstehen. Nicht als Anbindungsstrategie. diff --git a/src/ClawdDotNet.Core/Documents/DocumentText.cs b/src/ClawdDotNet.Core/Documents/DocumentText.cs new file mode 100644 index 0000000..6d71a77 --- /dev/null +++ b/src/ClawdDotNet.Core/Documents/DocumentText.cs @@ -0,0 +1,412 @@ +using System.IO.Compression; +using System.Text; +using System.Xml; + +namespace ClawdDotNet.Core.Documents; + +/// What came out of a document — the text, and what the reader should know about it. +public sealed record DocumentExtraction(bool Ok, string Text, string Kind, bool Truncated, string? Problem = null) +{ + public static DocumentExtraction Fail(string kind, string problem) => new(false, "", kind, false, problem); +} + +/// +/// #692: The text of what really comes in — an order confirmation as a Word file, a price list +/// as a spreadsheet. The least spectacular and most used missing ability of a company's agents. +/// +/// What comes out is foreign content. A supplier's file is no more trustworthy than +/// a web page; the engine frames every tool result as such (#193), this class only turns bytes +/// into text. It runs nothing in the file: no macros, no formulas (the stored value is read), +/// no external references, no DTDs. +/// +/// Bounded. Office files are zip archives, and a zip can unpack to anything. Every +/// part is read up to , the whole up to the caller's character limit. +/// +/// PDF is not read here yet: a reliable PDF text extractor is a library decision, not a +/// hundred lines of code — fonts with their own encodings come out as garbage otherwise. +/// +public static class DocumentText +{ + public const long MaxPartBytes = 32L * 1024 * 1024; + + private static readonly HashSet PlainText = new(StringComparer.OrdinalIgnoreCase) + { + ".txt", ".csv", ".tsv", ".md", ".json", ".xml", ".html", ".htm", ".log", ".yaml", ".yml", ".ini", ".eml" + }; + + /// Whether reads this kind of file. + public static bool Supports(string fileName) + => PlainText.Contains(Path.GetExtension(fileName)) || IsOfficeDocument(fileName); + + /// Whether the file is a document whose text needs extracting (not plain text). + public static bool IsOfficeDocument(string fileName) + => Path.GetExtension(fileName).ToLowerInvariant() is ".docx" or ".xlsx"; + + public static DocumentExtraction Extract(byte[] content, string fileName, int maxChars) + { + var ext = Path.GetExtension(fileName).ToLowerInvariant(); + try + { + return ext switch + { + ".docx" => Bounded(Docx(content), "Word document", maxChars), + ".xlsx" => Bounded(Xlsx(content), "spreadsheet", maxChars), + ".pdf" => DocumentExtraction.Fail("PDF", "PDF text cannot be read in this version yet."), + _ when PlainText.Contains(ext) => Bounded(Decode(content), "text", maxChars), + _ => DocumentExtraction.Fail(ext.TrimStart('.'), $"'{ext}' files cannot be read as text.") + }; + } + catch (Exception ex) when (ex is InvalidDataException or XmlException or IOException or InvalidOperationException) + { + return DocumentExtraction.Fail(ext.TrimStart('.'), $"The file could not be read: {ex.Message}"); + } + } + + private static DocumentExtraction Bounded(string text, string kind, int maxChars) + { + var limit = Math.Max(1, maxChars); + return text.Length <= limit + ? new DocumentExtraction(true, text, kind, false) + : new DocumentExtraction(true, text[..limit], kind, true); + } + + /// UTF-8 with or without BOM, UTF-16 by BOM, otherwise Latin-1 — never an exception for odd bytes. + private static string Decode(byte[] content) + { + if (content.Length >= 2 && content[0] == 0xFF && content[1] == 0xFE) + return Encoding.Unicode.GetString(content, 2, content.Length - 2); + if (content.Length >= 2 && content[0] == 0xFE && content[1] == 0xFF) + return Encoding.BigEndianUnicode.GetString(content, 2, content.Length - 2); + var start = content.Length >= 3 && content[0] == 0xEF && content[1] == 0xBB && content[2] == 0xBF ? 3 : 0; + try + { + return new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(content, start, content.Length - start); + } + catch (DecoderFallbackException) + { + return Encoding.Latin1.GetString(content); + } + } + + private static bool IsText(XmlNodeType type) + => type is XmlNodeType.Text or XmlNodeType.CDATA or XmlNodeType.SignificantWhitespace or XmlNodeType.Whitespace; + + // ─── Word ─── + + private const string W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"; + + private static string Docx(byte[] content) + { + using var zip = new ZipArchive(new MemoryStream(content), ZipArchiveMode.Read); + var part = zip.GetEntry("word/document.xml") ?? throw new InvalidDataException("This is no Word document (word/document.xml is missing)."); + + var sb = new StringBuilder(); + var cells = new List(); + var paragraph = new StringBuilder(); + var tableDepth = 0; + var inText = false; + using var reader = Xml(part); + while (reader.Read()) + { + if (inText && IsText(reader.NodeType)) + { + paragraph.Append(reader.Value); + continue; + } + if (reader.NamespaceURI != W) + continue; + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.LocalName) + { + case "t": + inText = !reader.IsEmptyElement; + break; + case "tab": + paragraph.Append('\t'); + break; + case "br" or "cr": + paragraph.Append('\n'); + break; + case "tbl": + tableDepth++; + break; + } + } + else if (reader.NodeType == XmlNodeType.EndElement) + { + switch (reader.LocalName) + { + case "t": + inText = false; + break; + case "p": + EndParagraph(); + break; + case "tc": + cells.Add(paragraph.ToString().Trim()); + paragraph.Clear(); + break; + case "tr": + sb.AppendLine("| " + string.Join(" | ", cells) + " |"); + cells.Clear(); + break; + case "tbl": + tableDepth--; + sb.AppendLine(); + break; + } + } + } + EndParagraph(); + return sb.ToString().Trim(); + + void EndParagraph() + { + if (tableDepth > 0) + { + // Inside a cell, paragraphs stay together until the cell ends. + if (paragraph.Length > 0 && paragraph[^1] != ' ') + paragraph.Append(' '); + return; + } + if (paragraph.Length > 0) + sb.AppendLine(paragraph.ToString()); + paragraph.Clear(); + } + } + + // ─── Excel ─── + + private const string S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; + private const string R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + private const string Pkg = "http://schemas.openxmlformats.org/package/2006/relationships"; + + private static string Xlsx(byte[] content) + { + using var zip = new ZipArchive(new MemoryStream(content), ZipArchiveMode.Read); + var workbook = zip.GetEntry("xl/workbook.xml") ?? throw new InvalidDataException("This is no spreadsheet (xl/workbook.xml is missing)."); + + var shared = zip.GetEntry("xl/sharedStrings.xml") is { } strings ? SharedStrings(strings) : []; + + var targets = new Dictionary(StringComparer.Ordinal); + if (zip.GetEntry("xl/_rels/workbook.xml.rels") is { } rels) + { + using var reader = Xml(rels); + while (reader.Read()) + if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "Relationship" && reader.NamespaceURI == Pkg + && reader.GetAttribute("Id") is { } id && reader.GetAttribute("Target") is { } target) + targets[id] = target.StartsWith('/') ? target.TrimStart('/') : "xl/" + target; + } + + var sheets = new List<(string Name, string Path)>(); + using (var reader = Xml(workbook)) + while (reader.Read()) + if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "sheet" && reader.NamespaceURI == S + && reader.GetAttribute("id", R) is { } rid && targets.TryGetValue(rid, out var path)) + sheets.Add((reader.GetAttribute("name") ?? path, path)); + + var sb = new StringBuilder(); + foreach (var (name, path) in sheets) + { + if (zip.GetEntry(path) is not { } sheet) + continue; + sb.AppendLine($"## Sheet: {name}"); + foreach (var row in Rows(sheet, shared)) + sb.AppendLine(string.Join('\t', row)); + sb.AppendLine(); + } + return sb.ToString().Trim(); + } + + /// The shared strings, in order — rich text joined, phonetic hints left out. + private static List SharedStrings(ZipArchiveEntry entry) + { + var shared = new List(); + using var reader = Xml(entry); + StringBuilder? item = null; + var inText = false; + var phonetic = 0; + while (reader.Read()) + { + if (inText && phonetic == 0 && IsText(reader.NodeType)) + { + item?.Append(reader.Value); + continue; + } + if (reader.NamespaceURI != S) + continue; + if (reader.NodeType == XmlNodeType.Element) + { + if (reader.LocalName == "si") + item = new StringBuilder(); + else if (reader.LocalName == "t") + inText = !reader.IsEmptyElement; + else if (reader.LocalName == "rPh" && !reader.IsEmptyElement) + phonetic++; + } + else if (reader.NodeType == XmlNodeType.EndElement) + { + if (reader.LocalName == "t") + inText = false; + else if (reader.LocalName == "rPh") + phonetic--; + else if (reader.LocalName == "si" && item is not null) + { + shared.Add(item.ToString()); + item = null; + } + } + } + return shared; + } + + /// The rows of a sheet, each cell at its column — the value as stored, formulas not run. + private static List> Rows(ZipArchiveEntry sheet, List shared) + { + var rows = new List>(); + using var reader = Xml(sheet); + List? row = null; + string? type = null, reference = null; + StringBuilder? value = null; + var inValue = false; + while (reader.Read()) + { + if (inValue && IsText(reader.NodeType)) + { + value?.Append(reader.Value); + continue; + } + if (reader.NamespaceURI != S) + continue; + if (reader.NodeType == XmlNodeType.Element) + { + switch (reader.LocalName) + { + case "row": + row = []; + if (reader.IsEmptyElement) + { + rows.Add(row); + row = null; + } + break; + case "c": + type = reader.GetAttribute("t"); + reference = reader.GetAttribute("r"); + value = new StringBuilder(); + if (reader.IsEmptyElement) + Place(row, reference, ""); + break; + case "v" or "t": + inValue = !reader.IsEmptyElement; + break; + } + } + else if (reader.NodeType == XmlNodeType.EndElement) + { + switch (reader.LocalName) + { + case "v" or "t": + inValue = false; + break; + case "c": + Place(row, reference, Cell(type, value?.ToString(), shared)); + value = null; + break; + case "row" when row is not null: + rows.Add(row); + row = null; + break; + } + } + } + return rows; + } + + private static string Cell(string? type, string? value, List shared) + => type switch + { + "s" when int.TryParse(value, out var i) && i >= 0 && i < shared.Count => shared[i], + "b" => value == "1" ? "TRUE" : "FALSE", + _ => value ?? "" + }; + + /// Puts the value at its column (from "C7"), filling gaps with empty cells. + private static void Place(List? row, string? reference, string value) + { + if (row is null) + return; + var column = row.Count; + if (!string.IsNullOrEmpty(reference)) + { + var letters = 0; + var number = 0; + foreach (var ch in reference) + { + if (ch is < 'A' or > 'Z') + break; + number = number * 26 + (ch - 'A' + 1); + letters++; + } + if (letters is > 0 and <= 3) + column = number - 1; + } + // A column beyond the sheet's width, or one already filled, goes at the end. + if (column > 16_384 || column < row.Count) + column = row.Count; + while (row.Count < column) + row.Add(""); + row.Add(value); + } + + // ─── shared ─── + + /// A reader for one part: no DTD, no external entities, bounded size. + private static XmlReader Xml(ZipArchiveEntry entry) + { + if (entry.Length > MaxPartBytes) + throw new InvalidDataException($"'{entry.FullName}' unpacks to {entry.Length / 1024 / 1024} MB — more than a document should."); + return XmlReader.Create(new BoundedStream(entry.Open(), MaxPartBytes), new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Prohibit, + XmlResolver = null, + IgnoreComments = true, + IgnoreProcessingInstructions = true, + CloseInput = true + }); + } + + /// Stops reading at a limit even if the archive lied about the size. + private sealed class BoundedStream(Stream inner, long limit) : Stream + { + private long _read; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _read; set => throw new NotSupportedException(); } + + public override int Read(byte[] buffer, int offset, int count) + { + var n = inner.Read(buffer, offset, count); + _read += n; + if (_read > limit) + throw new InvalidDataException("A part of the document unpacks to more than a document should."); + return n; + } + + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + inner.Dispose(); + base.Dispose(disposing); + } + } +} diff --git a/src/ClawdDotNet.Tools.FileRW/FileRWTool.cs b/src/ClawdDotNet.Tools.FileRW/FileRWTool.cs index 5459e70..a7dae6d 100644 --- a/src/ClawdDotNet.Tools.FileRW/FileRWTool.cs +++ b/src/ClawdDotNet.Tools.FileRW/FileRWTool.cs @@ -592,7 +592,12 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe ? je.EnumerateArray().Select(x => x.GetString()?.ToLowerInvariant()).ToList() : new List { ".txt", ".json", ".md", ".html", ".js", ".css" }; // Default - if (!allowedExtensions.Contains(extension)) + // #692: A Word file or a spreadsheet is read as its extracted text, never raw and never + // written by this tool — so reading it needs no place on the list. + var reading = input.ValueKind == JsonValueKind.Object && input.TryGetProperty("action", out var act) + && act.ValueKind == JsonValueKind.String && act.GetString() is "read" or "read_many"; + if (!allowedExtensions.Contains(extension) + && !(reading && ClawdDotNet.Core.Documents.DocumentText.IsOfficeDocument(fullPath))) { throw new UnauthorizedAccessException($"File extension '{extension}' is not allowed in workspace '{area.Name}'."); } @@ -668,7 +673,7 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe await fileLock.WaitAsync(ct); try { - var content = await File.ReadAllTextAsync(path, ct); + var content = await ReadContentAsync(path, ct); NoteRead(path, area, context); var relativePath = Path.GetRelativePath(rootPath, path); return ToolResult.Ok($"[{area.Name}:/{relativePath}]\n{content}"); @@ -679,6 +684,20 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe } } + /// + /// #692: A Word file or a spreadsheet is read as its text — a price list saved from a mail is + /// a table, not a heap of zip bytes. Everything else as before. + /// + private static async Task ReadContentAsync(string path, CancellationToken ct) + { + if (!ClawdDotNet.Core.Documents.DocumentText.IsOfficeDocument(path)) + return await File.ReadAllTextAsync(path, ct); + var extracted = ClawdDotNet.Core.Documents.DocumentText.Extract(await File.ReadAllBytesAsync(path, ct), path, MaxZeichenJeAufruf); + return extracted.Ok + ? $"(text of a {extracted.Kind}{(extracted.Truncated ? $", first {MaxZeichenJeAufruf} characters" : "")})\n{extracted.Text}" + : $"({extracted.Problem})"; + } + /// Hoechstzahl Dateien je read_many. public const int MaxDateienJeAufruf = 50; @@ -730,7 +749,7 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe string inhalt; try { - inhalt = await File.ReadAllTextAsync(pfad, ct); + inhalt = await ReadContentAsync(pfad, ct); NoteRead(pfad, area, context); } finally { sperre.Release(); } diff --git a/src/ClawdDotNet.Tools.Mail/MailTool.cs b/src/ClawdDotNet.Tools.Mail/MailTool.cs index d997828..61cbfd0 100644 --- a/src/ClawdDotNet.Tools.Mail/MailTool.cs +++ b/src/ClawdDotNet.Tools.Mail/MailTool.cs @@ -18,7 +18,7 @@ public sealed class MailTool : IAgentTool, IToolJobProvider /// #715: Brings text from outside into the context — mails from anyone. public bool ReadsForeignContent => true; - public string Description => "Allows sending and receiving emails via SMTP and IMAP."; + public string Description => "Allows sending and receiving emails via SMTP and IMAP. read_message lists a message's attachments; read_attachment returns an attachment's text (Word, Excel, text and CSV files); save_attachment stores it in your workspace."; public JsonElement InputSchema { get; } = JsonDocument.Parse(""" { @@ -26,7 +26,7 @@ public sealed class MailTool : IAgentTool, IToolJobProvider "properties": { "action": { "type": "string", - "enum": ["send", "read_inbox", "read_message", "mark_read"], + "enum": ["send", "read_inbox", "read_message", "mark_read", "read_attachment", "save_attachment"], "description": "The action to perform" }, "to": { @@ -47,7 +47,15 @@ public sealed class MailTool : IAgentTool, IToolJobProvider }, "messageId": { "type": "string", - "description": "The message ID (for read_message or mark_read)" + "description": "The message ID (for read_message, mark_read, read_attachment, save_attachment)" + }, + "attachment": { + "type": "string", + "description": "read_attachment, save_attachment: the attachment's number from read_message, or its file name" + }, + "fileName": { + "type": "string", + "description": "save_attachment: the file name in your workspace (default: the attachment's name)" } }, "required": ["action"] @@ -70,6 +78,7 @@ public sealed class MailTool : IAgentTool, IToolJobProvider "read_inbox" => await HandleReadInboxAsync(input, context, ct), "read_message" => await HandleReadMessageAsync(input, context, ct), "mark_read" => await HandleMarkReadAsync(input, context, ct), + "read_attachment" or "save_attachment" => await HandleAttachmentAsync(action, input, context, ct), _ => ToolResult.Fail($"Unknown action: {action}") }; } @@ -165,13 +174,98 @@ public sealed class MailTool : IAgentTool, IToolJobProvider Date = msg.Date, From = msg.From.ToString(), Subject = msg.Subject, - Body = msg.TextBody + Body = msg.TextBody, + // #692: what came with it — read with read_attachment, kept with save_attachment. + Attachments = Attachments(msg).Select((a, i) => new + { + Number = i + 1, + Name = a.FileName ?? $"attachment-{i + 1}", + Type = a.ContentType.MimeType, + Readable = ClawdDotNet.Core.Documents.DocumentText.Supports(a.FileName ?? "") + }).ToList() }; await client.DisconnectAsync(true, ct); return ToolResult.Ok(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true })); } + /// The largest attachment that is read or saved. + private const long MaxAttachmentBytes = 25L * 1024 * 1024; + + /// The longest text an attachment hands to the model. + private const int MaxAttachmentChars = 60_000; + + private static List Attachments(MimeMessage msg) + => msg.Attachments.OfType().ToList(); + + /// + /// #692: The text of what really comes in — an order confirmation as a Word file, a price list + /// as a spreadsheet — or the file itself in the workspace. The text is a supplier's text: + /// the engine frames it as foreign content like every tool result. + /// + private async Task HandleAttachmentAsync(string action, JsonElement input, AgentToolContext context, CancellationToken ct) + { + if (!input.TryGetProperty("messageId", out var mi) || !int.TryParse(mi.GetString(), out var index)) + return ToolResult.Fail("'messageId' (index) is required."); + var wanted = input.TryGetProperty("attachment", out var at) ? at.ToString().Trim() : ""; + if (wanted.Length == 0) + return ToolResult.Fail("'attachment' is required — its number from read_message, or its file name."); + + if (!TryReadAccess(context, "imapHost", "imapPort", 993, out var access, out var fehlt)) + return ToolResult.Fail(fehlt); + + using var client = new ImapClient(); + await client.ConnectAsync(access.Host, access.Port, true, ct); + await client.AuthenticateAsync(access.UserName, access.Password, ct); + await client.Inbox!.OpenAsync(FolderAccess.ReadOnly, ct); + var msg = await client.Inbox!.GetMessageAsync(index, ct); + await client.DisconnectAsync(true, ct); + + var attachments = Attachments(msg); + var part = int.TryParse(wanted, out var number) && number >= 1 && number <= attachments.Count + ? attachments[number - 1] + : attachments.FirstOrDefault(a => string.Equals(a.FileName, wanted, StringComparison.OrdinalIgnoreCase)); + if (part is null) + return ToolResult.Fail($"Message {index} has no attachment '{wanted}' — it has {attachments.Count}."); + + using var buffer = new MemoryStream(); + await part.Content.DecodeToAsync(buffer, ct); + if (buffer.Length > MaxAttachmentBytes) + return ToolResult.Fail($"The attachment is {buffer.Length / 1024 / 1024} MB — more than {MaxAttachmentBytes / 1024 / 1024} MB are not taken."); + var name = part.FileName ?? $"attachment-{index}"; + + if (action == "read_attachment") + { + var text = ClawdDotNet.Core.Documents.DocumentText.Extract(buffer.ToArray(), name, MaxAttachmentChars); + if (!text.Ok) + return ToolResult.Fail($"{name}: {text.Problem} Keep it with save_attachment if a human should look at it."); + return ToolResult.Ok($"Attachment '{name}' ({text.Kind}) of message {index}" + + (text.Truncated ? $", first {MaxAttachmentChars} characters" : "") + ":\n" + text.Text); + } + + if (string.IsNullOrWhiteSpace(context.WorkspacePath)) + return ToolResult.Fail("You have no workspace to keep the attachment in."); + var requested = input.TryGetProperty("fileName", out var fn) ? fn.GetString() : null; + var fileName = SafeFileName(string.IsNullOrWhiteSpace(requested) ? name : requested); + Directory.CreateDirectory(context.WorkspacePath); + var target = Path.Combine(context.WorkspacePath, fileName); + // Never over an existing file: name.pdf, name (2).pdf, ... + for (var n = 2; File.Exists(target); n++) + target = Path.Combine(context.WorkspacePath, $"{Path.GetFileNameWithoutExtension(fileName)} ({n}){Path.GetExtension(fileName)}"); + await File.WriteAllBytesAsync(target, buffer.ToArray(), ct); + return ToolResult.Ok($"Attachment '{name}' of message {index} saved in your workspace as '{Path.GetFileName(target)}' ({buffer.Length} bytes)."); + } + + /// A file name only — no directory, nothing that leaves the workspace. + public static string SafeFileName(string name) + { + // The same answer on every system: the last segment, and the characters any of them refuses replaced. + var bare = name.Replace('\\', '/').Split('/').Last().Trim(); + bare = new string(bare.Select(c => c < 32 || "<>:\"|?*".Contains(c) ? '_' : c).ToArray()); + bare = bare.TrimStart('.'); + return bare.Length == 0 ? "attachment" : bare.Length > 120 ? bare[..120] : bare; + } + private async Task HandleMarkReadAsync(JsonElement input, AgentToolContext context, CancellationToken ct) { if (!input.TryGetProperty("messageId", out var mi) || !int.TryParse(mi.GetString(), out var index)) diff --git a/tests/ClawdDotNet.Core.Tests/Documents/DocumentTextTests.cs b/tests/ClawdDotNet.Core.Tests/Documents/DocumentTextTests.cs new file mode 100644 index 0000000..f1a510f --- /dev/null +++ b/tests/ClawdDotNet.Core.Tests/Documents/DocumentTextTests.cs @@ -0,0 +1,118 @@ +using System.IO.Compression; +using System.Text; +using ClawdDotNet.Core.Documents; +using Shouldly; + +namespace ClawdDotNet.Core.Tests.Documents; + +/// +/// #692: The text of what really comes in — a Word order confirmation, a price list as a +/// spreadsheet — without running anything in the file and without being unpacked to death. +/// +public sealed class DocumentTextTests +{ + private const string W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"; + private const string S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; + + public static byte[] Zip(params (string Name, string Content)[] parts) + { + using var buffer = new MemoryStream(); + using (var zip = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + foreach (var (name, content) in parts) + { + using var stream = zip.CreateEntry(name).Open(); + stream.Write(Encoding.UTF8.GetBytes(content)); + } + return buffer.ToArray(); + } + + public static byte[] Docx(string body) + => Zip(("word/document.xml", $"""{body}""")); + + [Fact] + public void A_word_document_comes_out_as_paragraphs_and_table_rows() + { + var doc = Docx(""" + Order confirmation + No. 4711due 2026-10-15 + + ItemQty + Paper A440 + + Thank you. + """); + + var text = DocumentText.Extract(doc, "confirmation.docx", 10_000); + + text.Ok.ShouldBeTrue(text.Problem); + text.Kind.ShouldBe("Word document"); + text.Text.ShouldContain("Order confirmation"); + text.Text.ShouldContain("No. 4711\tdue 2026-10-15"); + text.Text.ShouldContain("| Item | Qty |"); + text.Text.ShouldContain("| Paper A4 | 40 |"); + text.Text.ShouldEndWith("Thank you."); + } + + [Fact] + public void A_spreadsheet_comes_out_sheet_by_sheet_with_every_cell_at_its_column() + { + var xlsx = Zip( + ("xl/workbook.xml", $""""""), + ("xl/_rels/workbook.xml.rels", """"""), + ("xl/sharedStrings.xml", $"""ItemPaperignoredPrice"""), + ("xl/worksheets/sheet1.xml", $""" + + 02 + 11B2*34.5 + + """), + ("xl/worksheets/sheet2.xml", $"""Order by Friday""")); + + var text = DocumentText.Extract(xlsx, "prices.xlsx", 10_000); + + text.Ok.ShouldBeTrue(text.Problem); + text.Text.ShouldContain("## Sheet: Prices\nItem\t\tPrice\nPaper\tTRUE\t4.5".Replace("\n", Environment.NewLine)); + text.Text.ShouldContain("## Sheet: Notes"); + text.Text.ShouldContain("Order by Friday"); + text.Text.ShouldNotContain("ignored", customMessage: "phonetic hints are no content"); + text.Text.ShouldNotContain("B2*3", customMessage: "the stored value is read, the formula is not"); + } + + [Fact] + public void A_document_with_a_dtd_is_refused_not_resolved() + { + var doc = Zip(("word/document.xml", $"""]>&e;""")); + + var text = DocumentText.Extract(doc, "evil.docx", 10_000); + + text.Ok.ShouldBeFalse(); + text.Problem!.ShouldContain("could not be read"); + } + + [Fact] + public void A_part_that_unpacks_beyond_the_limit_is_stopped() + { + var huge = new string('a', (int)DocumentText.MaxPartBytes + 1024); + var doc = Docx($"{huge}"); + + DocumentText.Extract(doc, "bomb.docx", 100).Ok.ShouldBeFalse(); + } + + [Fact] + public void Text_is_cut_at_the_limit_and_odd_bytes_are_no_error() + { + var cut = DocumentText.Extract(Encoding.UTF8.GetBytes(new string('x', 500)), "a.csv", 100); + cut.Truncated.ShouldBeTrue(); + cut.Text.Length.ShouldBe(100); + + DocumentText.Extract([0x50, 0x72, 0x65, 0x69, 0x73, 0x20, 0xE4], "latin.txt", 100).Text.ShouldBe("Preis ä"); + } + + [Fact] + public void Pdf_and_unknown_files_are_named_not_guessed() + { + DocumentText.Extract([1, 2, 3], "invoice.pdf", 100).Problem!.ShouldContain("PDF"); + DocumentText.Extract([1, 2, 3], "image.png", 100).Ok.ShouldBeFalse(); + DocumentText.Extract([1, 2, 3], "broken.docx", 100).Ok.ShouldBeFalse(); + } +} diff --git a/tests/ClawdDotNet.Tools.Tests/ClawdDotNet.Tools.Tests.csproj b/tests/ClawdDotNet.Tools.Tests/ClawdDotNet.Tools.Tests.csproj index 325cb44..63d4f11 100644 --- a/tests/ClawdDotNet.Tools.Tests/ClawdDotNet.Tools.Tests.csproj +++ b/tests/ClawdDotNet.Tools.Tests/ClawdDotNet.Tools.Tests.csproj @@ -25,6 +25,7 @@ + diff --git a/tests/ClawdDotNet.Tools.Tests/FileRW/DocumentReadTests.cs b/tests/ClawdDotNet.Tools.Tests/FileRW/DocumentReadTests.cs new file mode 100644 index 0000000..74ef0a8 --- /dev/null +++ b/tests/ClawdDotNet.Tools.Tests/FileRW/DocumentReadTests.cs @@ -0,0 +1,62 @@ +using System.IO.Compression; +using System.Text; +using System.Text.Json; +using ClawdDotNet.Core.Tools; +using ClawdDotNet.Tools.FileRW; +using ClawdDotNet.Tools.Mail; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace ClawdDotNet.Tools.Tests.FileRW; + +/// #692: A Word file in the workspace is read as its text, and an attachment's name never leaves it. +public sealed class DocumentReadTests : IDisposable +{ + private readonly string _workspace = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N"), "Workspace"); + + public DocumentReadTests() => Directory.CreateDirectory(_workspace); + + public void Dispose() + { + try { Directory.Delete(Path.GetDirectoryName(_workspace)!, recursive: true); } + catch { /* cleanup is secondary */ } + } + + [Fact] + public async Task Read_returns_the_text_of_a_word_document() + { + using (var zip = ZipFile.Open(Path.Combine(_workspace, "order.docx"), ZipArchiveMode.Create)) + using (var stream = zip.CreateEntry("word/document.xml").Open()) + stream.Write(Encoding.UTF8.GetBytes( + """40 x Paper A4""")); + + var context = new AgentToolContext("buyer", "inst", new Dictionary(), new NullStateStore(), NullLogger.Instance, + CancellationToken.None, WorkspacePath: _workspace); + var result = await new FileRWTool().ExecuteAsync(JsonDocument.Parse("""{"action":"read","path":"order.docx"}""").RootElement, context, CancellationToken.None); + + result.Success.ShouldBeTrue(result.ErrorMessage); + result.Content.ShouldContain("text of a Word document"); + result.Content.ShouldContain("40 x Paper A4"); + } + + [Fact] + public async Task Writing_a_word_document_stays_refused() + { + var context = new AgentToolContext("buyer", "inst", new Dictionary(), new NullStateStore(), NullLogger.Instance, + CancellationToken.None, WorkspacePath: _workspace); + var result = await new FileRWTool().ExecuteAsync( + JsonDocument.Parse("""{"action":"write","path":"fake.docx","content":"x"}""").RootElement, context, CancellationToken.None); + + result.Success.ShouldBeFalse("only reading a document needs no place on the list"); + } + + [Theory] + [InlineData("../../etc/passwd", "passwd")] + [InlineData("..\\..\\Windows\\win.ini", "win.ini")] + [InlineData("C:/x/invoice.pdf", "invoice.pdf")] + [InlineData(".hidden", "hidden")] + [InlineData("", "attachment")] + [InlineData("a:b?c.txt", "a_b_c.txt")] + public void An_attachment_name_is_a_file_name_and_nothing_more(string given, string expected) + => MailTool.SafeFileName(given).ShouldBe(expected); +}