Werkzeugkasten: Anhaenge lesen - Word, Excel, Text (#692)
- Core/Documents/DocumentText: text of .docx (paragraphs, tables as rows), .xlsx (sheet by sheet, every cell at its column, stored values not formulas) and plain text/CSV. No DTD, no external references, every part bounded against zip bombs. PDF is named as not readable yet. - Mail: read_message lists attachments; read_attachment returns the text; save_attachment keeps the file in the agent's workspace (never over an existing file, never outside). - FileRW read/read_many return Word and Excel files as text; writing them stays refused. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
edf3ff19ea
commit
72d855efcf
@@ -797,6 +797,16 @@ beidem.
|
||||
PDF-Anhang, eine Preisliste eine Tabelle. Text aus Anhängen zu holen ist die
|
||||
unspektakulärste und am häufigsten gebrauchte fehlende Fähigkeit. Ein PDF vom
|
||||
Lieferanten ist dabei so wenig vertrauenswürdig wie eine Webseite (#193).
|
||||
|
||||
> **Umgesetzt (#692) für Word, Excel und Text.** `DocumentText` im Core holt den Text
|
||||
> aus `.docx` (Absätze, Tabellen als Zeilen) und `.xlsx` (je Blatt, jede Zelle an ihrer
|
||||
> Spalte, gespeicherte Werte statt Formeln) sowie aus Text- und CSV-Dateien — ohne
|
||||
> DTD, ohne externe Verweise, mit Obergrenze je Teil gegen Zip-Bomben. Mail listet in
|
||||
> `read_message` die Anhänge, `read_attachment` liefert ihren Text, `save_attachment`
|
||||
> legt die Datei im eigenen Arbeitsbereich ab (nie über eine vorhandene, nie außerhalb).
|
||||
> FileRW `read`/`read_many` gibt Word- und Excel-Dateien als Text zurück; schreiben kann
|
||||
> es sie weiterhin nicht. Der Text ist Fremdinhalt und wird von der Engine gerahmt.
|
||||
> **Offen: PDF** — ein verlässlicher PDF-Textauszug ist eine Bibliotheksentscheidung.
|
||||
3. **Der MCP-Client (#197) bleibt nützlich** — aber als *eine* Fähigkeit unter anderen,
|
||||
für Dienste, die ohnehin als Server bereitstehen. Nicht als Anbindungsstrategie.
|
||||
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using System.Xml;
|
||||
|
||||
namespace ClawdDotNet.Core.Documents;
|
||||
|
||||
/// <summary>What came out of a document — the text, and what the reader should know about it.</summary>
|
||||
public sealed record DocumentExtraction(bool Ok, string Text, string Kind, bool Truncated, string? Problem = null)
|
||||
{
|
||||
public static DocumentExtraction Fail(string kind, string problem) => new(false, "", kind, false, problem);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #692: The text of what really comes in — an order confirmation as a Word file, a price list
|
||||
/// as a spreadsheet. The least spectacular and most used missing ability of a company's agents.
|
||||
///
|
||||
/// <para><b>What comes out is foreign content.</b> A supplier's file is no more trustworthy than
|
||||
/// a web page; the engine frames every tool result as such (#193), this class only turns bytes
|
||||
/// into text. It runs nothing in the file: no macros, no formulas (the stored value is read),
|
||||
/// no external references, no DTDs.</para>
|
||||
///
|
||||
/// <para><b>Bounded.</b> Office files are zip archives, and a zip can unpack to anything. Every
|
||||
/// part is read up to <see cref="MaxPartBytes"/>, the whole up to the caller's character limit.</para>
|
||||
///
|
||||
/// <para>PDF is not read here yet: a reliable PDF text extractor is a library decision, not a
|
||||
/// hundred lines of code — fonts with their own encodings come out as garbage otherwise.</para>
|
||||
/// </summary>
|
||||
public static class DocumentText
|
||||
{
|
||||
public const long MaxPartBytes = 32L * 1024 * 1024;
|
||||
|
||||
private static readonly HashSet<string> PlainText = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
".txt", ".csv", ".tsv", ".md", ".json", ".xml", ".html", ".htm", ".log", ".yaml", ".yml", ".ini", ".eml"
|
||||
};
|
||||
|
||||
/// <summary>Whether <see cref="Extract"/> reads this kind of file.</summary>
|
||||
public static bool Supports(string fileName)
|
||||
=> PlainText.Contains(Path.GetExtension(fileName)) || IsOfficeDocument(fileName);
|
||||
|
||||
/// <summary>Whether the file is a document whose text needs extracting (not plain text).</summary>
|
||||
public static bool IsOfficeDocument(string fileName)
|
||||
=> Path.GetExtension(fileName).ToLowerInvariant() is ".docx" or ".xlsx";
|
||||
|
||||
public static DocumentExtraction Extract(byte[] content, string fileName, int maxChars)
|
||||
{
|
||||
var ext = Path.GetExtension(fileName).ToLowerInvariant();
|
||||
try
|
||||
{
|
||||
return ext switch
|
||||
{
|
||||
".docx" => Bounded(Docx(content), "Word document", maxChars),
|
||||
".xlsx" => Bounded(Xlsx(content), "spreadsheet", maxChars),
|
||||
".pdf" => DocumentExtraction.Fail("PDF", "PDF text cannot be read in this version yet."),
|
||||
_ when PlainText.Contains(ext) => Bounded(Decode(content), "text", maxChars),
|
||||
_ => DocumentExtraction.Fail(ext.TrimStart('.'), $"'{ext}' files cannot be read as text.")
|
||||
};
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or XmlException or IOException or InvalidOperationException)
|
||||
{
|
||||
return DocumentExtraction.Fail(ext.TrimStart('.'), $"The file could not be read: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static DocumentExtraction Bounded(string text, string kind, int maxChars)
|
||||
{
|
||||
var limit = Math.Max(1, maxChars);
|
||||
return text.Length <= limit
|
||||
? new DocumentExtraction(true, text, kind, false)
|
||||
: new DocumentExtraction(true, text[..limit], kind, true);
|
||||
}
|
||||
|
||||
/// <summary>UTF-8 with or without BOM, UTF-16 by BOM, otherwise Latin-1 — never an exception for odd bytes.</summary>
|
||||
private static string Decode(byte[] content)
|
||||
{
|
||||
if (content.Length >= 2 && content[0] == 0xFF && content[1] == 0xFE)
|
||||
return Encoding.Unicode.GetString(content, 2, content.Length - 2);
|
||||
if (content.Length >= 2 && content[0] == 0xFE && content[1] == 0xFF)
|
||||
return Encoding.BigEndianUnicode.GetString(content, 2, content.Length - 2);
|
||||
var start = content.Length >= 3 && content[0] == 0xEF && content[1] == 0xBB && content[2] == 0xBF ? 3 : 0;
|
||||
try
|
||||
{
|
||||
return new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(content, start, content.Length - start);
|
||||
}
|
||||
catch (DecoderFallbackException)
|
||||
{
|
||||
return Encoding.Latin1.GetString(content);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsText(XmlNodeType type)
|
||||
=> type is XmlNodeType.Text or XmlNodeType.CDATA or XmlNodeType.SignificantWhitespace or XmlNodeType.Whitespace;
|
||||
|
||||
// ─── Word ───
|
||||
|
||||
private const string W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main";
|
||||
|
||||
private static string Docx(byte[] content)
|
||||
{
|
||||
using var zip = new ZipArchive(new MemoryStream(content), ZipArchiveMode.Read);
|
||||
var part = zip.GetEntry("word/document.xml") ?? throw new InvalidDataException("This is no Word document (word/document.xml is missing).");
|
||||
|
||||
var sb = new StringBuilder();
|
||||
var cells = new List<string>();
|
||||
var paragraph = new StringBuilder();
|
||||
var tableDepth = 0;
|
||||
var inText = false;
|
||||
using var reader = Xml(part);
|
||||
while (reader.Read())
|
||||
{
|
||||
if (inText && IsText(reader.NodeType))
|
||||
{
|
||||
paragraph.Append(reader.Value);
|
||||
continue;
|
||||
}
|
||||
if (reader.NamespaceURI != W)
|
||||
continue;
|
||||
if (reader.NodeType == XmlNodeType.Element)
|
||||
{
|
||||
switch (reader.LocalName)
|
||||
{
|
||||
case "t":
|
||||
inText = !reader.IsEmptyElement;
|
||||
break;
|
||||
case "tab":
|
||||
paragraph.Append('\t');
|
||||
break;
|
||||
case "br" or "cr":
|
||||
paragraph.Append('\n');
|
||||
break;
|
||||
case "tbl":
|
||||
tableDepth++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (reader.NodeType == XmlNodeType.EndElement)
|
||||
{
|
||||
switch (reader.LocalName)
|
||||
{
|
||||
case "t":
|
||||
inText = false;
|
||||
break;
|
||||
case "p":
|
||||
EndParagraph();
|
||||
break;
|
||||
case "tc":
|
||||
cells.Add(paragraph.ToString().Trim());
|
||||
paragraph.Clear();
|
||||
break;
|
||||
case "tr":
|
||||
sb.AppendLine("| " + string.Join(" | ", cells) + " |");
|
||||
cells.Clear();
|
||||
break;
|
||||
case "tbl":
|
||||
tableDepth--;
|
||||
sb.AppendLine();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
EndParagraph();
|
||||
return sb.ToString().Trim();
|
||||
|
||||
void EndParagraph()
|
||||
{
|
||||
if (tableDepth > 0)
|
||||
{
|
||||
// Inside a cell, paragraphs stay together until the cell ends.
|
||||
if (paragraph.Length > 0 && paragraph[^1] != ' ')
|
||||
paragraph.Append(' ');
|
||||
return;
|
||||
}
|
||||
if (paragraph.Length > 0)
|
||||
sb.AppendLine(paragraph.ToString());
|
||||
paragraph.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Excel ───
|
||||
|
||||
private const string S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
|
||||
private const string R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships";
|
||||
private const string Pkg = "http://schemas.openxmlformats.org/package/2006/relationships";
|
||||
|
||||
private static string Xlsx(byte[] content)
|
||||
{
|
||||
using var zip = new ZipArchive(new MemoryStream(content), ZipArchiveMode.Read);
|
||||
var workbook = zip.GetEntry("xl/workbook.xml") ?? throw new InvalidDataException("This is no spreadsheet (xl/workbook.xml is missing).");
|
||||
|
||||
var shared = zip.GetEntry("xl/sharedStrings.xml") is { } strings ? SharedStrings(strings) : [];
|
||||
|
||||
var targets = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
if (zip.GetEntry("xl/_rels/workbook.xml.rels") is { } rels)
|
||||
{
|
||||
using var reader = Xml(rels);
|
||||
while (reader.Read())
|
||||
if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "Relationship" && reader.NamespaceURI == Pkg
|
||||
&& reader.GetAttribute("Id") is { } id && reader.GetAttribute("Target") is { } target)
|
||||
targets[id] = target.StartsWith('/') ? target.TrimStart('/') : "xl/" + target;
|
||||
}
|
||||
|
||||
var sheets = new List<(string Name, string Path)>();
|
||||
using (var reader = Xml(workbook))
|
||||
while (reader.Read())
|
||||
if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "sheet" && reader.NamespaceURI == S
|
||||
&& reader.GetAttribute("id", R) is { } rid && targets.TryGetValue(rid, out var path))
|
||||
sheets.Add((reader.GetAttribute("name") ?? path, path));
|
||||
|
||||
var sb = new StringBuilder();
|
||||
foreach (var (name, path) in sheets)
|
||||
{
|
||||
if (zip.GetEntry(path) is not { } sheet)
|
||||
continue;
|
||||
sb.AppendLine($"## Sheet: {name}");
|
||||
foreach (var row in Rows(sheet, shared))
|
||||
sb.AppendLine(string.Join('\t', row));
|
||||
sb.AppendLine();
|
||||
}
|
||||
return sb.ToString().Trim();
|
||||
}
|
||||
|
||||
/// <summary>The shared strings, in order — rich text joined, phonetic hints left out.</summary>
|
||||
private static List<string> SharedStrings(ZipArchiveEntry entry)
|
||||
{
|
||||
var shared = new List<string>();
|
||||
using var reader = Xml(entry);
|
||||
StringBuilder? item = null;
|
||||
var inText = false;
|
||||
var phonetic = 0;
|
||||
while (reader.Read())
|
||||
{
|
||||
if (inText && phonetic == 0 && IsText(reader.NodeType))
|
||||
{
|
||||
item?.Append(reader.Value);
|
||||
continue;
|
||||
}
|
||||
if (reader.NamespaceURI != S)
|
||||
continue;
|
||||
if (reader.NodeType == XmlNodeType.Element)
|
||||
{
|
||||
if (reader.LocalName == "si")
|
||||
item = new StringBuilder();
|
||||
else if (reader.LocalName == "t")
|
||||
inText = !reader.IsEmptyElement;
|
||||
else if (reader.LocalName == "rPh" && !reader.IsEmptyElement)
|
||||
phonetic++;
|
||||
}
|
||||
else if (reader.NodeType == XmlNodeType.EndElement)
|
||||
{
|
||||
if (reader.LocalName == "t")
|
||||
inText = false;
|
||||
else if (reader.LocalName == "rPh")
|
||||
phonetic--;
|
||||
else if (reader.LocalName == "si" && item is not null)
|
||||
{
|
||||
shared.Add(item.ToString());
|
||||
item = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
return shared;
|
||||
}
|
||||
|
||||
/// <summary>The rows of a sheet, each cell at its column — the value as stored, formulas not run.</summary>
|
||||
private static List<List<string>> Rows(ZipArchiveEntry sheet, List<string> shared)
|
||||
{
|
||||
var rows = new List<List<string>>();
|
||||
using var reader = Xml(sheet);
|
||||
List<string>? row = null;
|
||||
string? type = null, reference = null;
|
||||
StringBuilder? value = null;
|
||||
var inValue = false;
|
||||
while (reader.Read())
|
||||
{
|
||||
if (inValue && IsText(reader.NodeType))
|
||||
{
|
||||
value?.Append(reader.Value);
|
||||
continue;
|
||||
}
|
||||
if (reader.NamespaceURI != S)
|
||||
continue;
|
||||
if (reader.NodeType == XmlNodeType.Element)
|
||||
{
|
||||
switch (reader.LocalName)
|
||||
{
|
||||
case "row":
|
||||
row = [];
|
||||
if (reader.IsEmptyElement)
|
||||
{
|
||||
rows.Add(row);
|
||||
row = null;
|
||||
}
|
||||
break;
|
||||
case "c":
|
||||
type = reader.GetAttribute("t");
|
||||
reference = reader.GetAttribute("r");
|
||||
value = new StringBuilder();
|
||||
if (reader.IsEmptyElement)
|
||||
Place(row, reference, "");
|
||||
break;
|
||||
case "v" or "t":
|
||||
inValue = !reader.IsEmptyElement;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (reader.NodeType == XmlNodeType.EndElement)
|
||||
{
|
||||
switch (reader.LocalName)
|
||||
{
|
||||
case "v" or "t":
|
||||
inValue = false;
|
||||
break;
|
||||
case "c":
|
||||
Place(row, reference, Cell(type, value?.ToString(), shared));
|
||||
value = null;
|
||||
break;
|
||||
case "row" when row is not null:
|
||||
rows.Add(row);
|
||||
row = null;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
private static string Cell(string? type, string? value, List<string> shared)
|
||||
=> type switch
|
||||
{
|
||||
"s" when int.TryParse(value, out var i) && i >= 0 && i < shared.Count => shared[i],
|
||||
"b" => value == "1" ? "TRUE" : "FALSE",
|
||||
_ => value ?? ""
|
||||
};
|
||||
|
||||
/// <summary>Puts the value at its column (from "C7"), filling gaps with empty cells.</summary>
|
||||
private static void Place(List<string>? row, string? reference, string value)
|
||||
{
|
||||
if (row is null)
|
||||
return;
|
||||
var column = row.Count;
|
||||
if (!string.IsNullOrEmpty(reference))
|
||||
{
|
||||
var letters = 0;
|
||||
var number = 0;
|
||||
foreach (var ch in reference)
|
||||
{
|
||||
if (ch is < 'A' or > 'Z')
|
||||
break;
|
||||
number = number * 26 + (ch - 'A' + 1);
|
||||
letters++;
|
||||
}
|
||||
if (letters is > 0 and <= 3)
|
||||
column = number - 1;
|
||||
}
|
||||
// A column beyond the sheet's width, or one already filled, goes at the end.
|
||||
if (column > 16_384 || column < row.Count)
|
||||
column = row.Count;
|
||||
while (row.Count < column)
|
||||
row.Add("");
|
||||
row.Add(value);
|
||||
}
|
||||
|
||||
// ─── shared ───
|
||||
|
||||
/// <summary>A reader for one part: no DTD, no external entities, bounded size.</summary>
|
||||
private static XmlReader Xml(ZipArchiveEntry entry)
|
||||
{
|
||||
if (entry.Length > MaxPartBytes)
|
||||
throw new InvalidDataException($"'{entry.FullName}' unpacks to {entry.Length / 1024 / 1024} MB — more than a document should.");
|
||||
return XmlReader.Create(new BoundedStream(entry.Open(), MaxPartBytes), new XmlReaderSettings
|
||||
{
|
||||
DtdProcessing = DtdProcessing.Prohibit,
|
||||
XmlResolver = null,
|
||||
IgnoreComments = true,
|
||||
IgnoreProcessingInstructions = true,
|
||||
CloseInput = true
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Stops reading at a limit even if the archive lied about the size.</summary>
|
||||
private sealed class BoundedStream(Stream inner, long limit) : Stream
|
||||
{
|
||||
private long _read;
|
||||
|
||||
public override bool CanRead => true;
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => throw new NotSupportedException();
|
||||
public override long Position { get => _read; set => throw new NotSupportedException(); }
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
var n = inner.Read(buffer, offset, count);
|
||||
_read += n;
|
||||
if (_read > limit)
|
||||
throw new InvalidDataException("A part of the document unpacks to more than a document should.");
|
||||
return n;
|
||||
}
|
||||
|
||||
public override void Flush() { }
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
inner.Dispose();
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -592,7 +592,12 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe
|
||||
? je.EnumerateArray().Select(x => x.GetString()?.ToLowerInvariant()).ToList()
|
||||
: new List<string?> { ".txt", ".json", ".md", ".html", ".js", ".css" }; // Default
|
||||
|
||||
if (!allowedExtensions.Contains(extension))
|
||||
// #692: A Word file or a spreadsheet is read as its extracted text, never raw and never
|
||||
// written by this tool — so reading it needs no place on the list.
|
||||
var reading = input.ValueKind == JsonValueKind.Object && input.TryGetProperty("action", out var act)
|
||||
&& act.ValueKind == JsonValueKind.String && act.GetString() is "read" or "read_many";
|
||||
if (!allowedExtensions.Contains(extension)
|
||||
&& !(reading && ClawdDotNet.Core.Documents.DocumentText.IsOfficeDocument(fullPath)))
|
||||
{
|
||||
throw new UnauthorizedAccessException($"File extension '{extension}' is not allowed in workspace '{area.Name}'.");
|
||||
}
|
||||
@@ -668,7 +673,7 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe
|
||||
await fileLock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var content = await File.ReadAllTextAsync(path, ct);
|
||||
var content = await ReadContentAsync(path, ct);
|
||||
NoteRead(path, area, context);
|
||||
var relativePath = Path.GetRelativePath(rootPath, path);
|
||||
return ToolResult.Ok($"[{area.Name}:/{relativePath}]\n{content}");
|
||||
@@ -679,6 +684,20 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #692: A Word file or a spreadsheet is read as its text — a price list saved from a mail is
|
||||
/// a table, not a heap of zip bytes. Everything else as before.
|
||||
/// </summary>
|
||||
private static async Task<string> ReadContentAsync(string path, CancellationToken ct)
|
||||
{
|
||||
if (!ClawdDotNet.Core.Documents.DocumentText.IsOfficeDocument(path))
|
||||
return await File.ReadAllTextAsync(path, ct);
|
||||
var extracted = ClawdDotNet.Core.Documents.DocumentText.Extract(await File.ReadAllBytesAsync(path, ct), path, MaxZeichenJeAufruf);
|
||||
return extracted.Ok
|
||||
? $"(text of a {extracted.Kind}{(extracted.Truncated ? $", first {MaxZeichenJeAufruf} characters" : "")})\n{extracted.Text}"
|
||||
: $"({extracted.Problem})";
|
||||
}
|
||||
|
||||
/// <summary>Hoechstzahl Dateien je <c>read_many</c>.</summary>
|
||||
public const int MaxDateienJeAufruf = 50;
|
||||
|
||||
@@ -730,7 +749,7 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe
|
||||
string inhalt;
|
||||
try
|
||||
{
|
||||
inhalt = await File.ReadAllTextAsync(pfad, ct);
|
||||
inhalt = await ReadContentAsync(pfad, ct);
|
||||
NoteRead(pfad, area, context);
|
||||
}
|
||||
finally { sperre.Release(); }
|
||||
|
||||
@@ -18,7 +18,7 @@ public sealed class MailTool : IAgentTool, IToolJobProvider
|
||||
/// <summary>#715: Brings text from outside into the context — mails from anyone.</summary>
|
||||
public bool ReadsForeignContent => true;
|
||||
|
||||
public string Description => "Allows sending and receiving emails via SMTP and IMAP.";
|
||||
public string Description => "Allows sending and receiving emails via SMTP and IMAP. read_message lists a message's attachments; read_attachment returns an attachment's text (Word, Excel, text and CSV files); save_attachment stores it in your workspace.";
|
||||
|
||||
public JsonElement InputSchema { get; } = JsonDocument.Parse("""
|
||||
{
|
||||
@@ -26,7 +26,7 @@ public sealed class MailTool : IAgentTool, IToolJobProvider
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["send", "read_inbox", "read_message", "mark_read"],
|
||||
"enum": ["send", "read_inbox", "read_message", "mark_read", "read_attachment", "save_attachment"],
|
||||
"description": "The action to perform"
|
||||
},
|
||||
"to": {
|
||||
@@ -47,7 +47,15 @@ public sealed class MailTool : IAgentTool, IToolJobProvider
|
||||
},
|
||||
"messageId": {
|
||||
"type": "string",
|
||||
"description": "The message ID (for read_message or mark_read)"
|
||||
"description": "The message ID (for read_message, mark_read, read_attachment, save_attachment)"
|
||||
},
|
||||
"attachment": {
|
||||
"type": "string",
|
||||
"description": "read_attachment, save_attachment: the attachment's number from read_message, or its file name"
|
||||
},
|
||||
"fileName": {
|
||||
"type": "string",
|
||||
"description": "save_attachment: the file name in your workspace (default: the attachment's name)"
|
||||
}
|
||||
},
|
||||
"required": ["action"]
|
||||
@@ -70,6 +78,7 @@ public sealed class MailTool : IAgentTool, IToolJobProvider
|
||||
"read_inbox" => await HandleReadInboxAsync(input, context, ct),
|
||||
"read_message" => await HandleReadMessageAsync(input, context, ct),
|
||||
"mark_read" => await HandleMarkReadAsync(input, context, ct),
|
||||
"read_attachment" or "save_attachment" => await HandleAttachmentAsync(action, input, context, ct),
|
||||
_ => ToolResult.Fail($"Unknown action: {action}")
|
||||
};
|
||||
}
|
||||
@@ -165,13 +174,98 @@ public sealed class MailTool : IAgentTool, IToolJobProvider
|
||||
Date = msg.Date,
|
||||
From = msg.From.ToString(),
|
||||
Subject = msg.Subject,
|
||||
Body = msg.TextBody
|
||||
Body = msg.TextBody,
|
||||
// #692: what came with it — read with read_attachment, kept with save_attachment.
|
||||
Attachments = Attachments(msg).Select((a, i) => new
|
||||
{
|
||||
Number = i + 1,
|
||||
Name = a.FileName ?? $"attachment-{i + 1}",
|
||||
Type = a.ContentType.MimeType,
|
||||
Readable = ClawdDotNet.Core.Documents.DocumentText.Supports(a.FileName ?? "")
|
||||
}).ToList()
|
||||
};
|
||||
|
||||
await client.DisconnectAsync(true, ct);
|
||||
return ToolResult.Ok(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }));
|
||||
}
|
||||
|
||||
/// <summary>The largest attachment that is read or saved.</summary>
|
||||
private const long MaxAttachmentBytes = 25L * 1024 * 1024;
|
||||
|
||||
/// <summary>The longest text an attachment hands to the model.</summary>
|
||||
private const int MaxAttachmentChars = 60_000;
|
||||
|
||||
private static List<MimePart> Attachments(MimeMessage msg)
|
||||
=> msg.Attachments.OfType<MimePart>().ToList();
|
||||
|
||||
/// <summary>
|
||||
/// #692: The text of what really comes in — an order confirmation as a Word file, a price list
|
||||
/// as a spreadsheet — or the file itself in the workspace. The text is a supplier's text:
|
||||
/// the engine frames it as foreign content like every tool result.
|
||||
/// </summary>
|
||||
private async Task<ToolResult> HandleAttachmentAsync(string action, JsonElement input, AgentToolContext context, CancellationToken ct)
|
||||
{
|
||||
if (!input.TryGetProperty("messageId", out var mi) || !int.TryParse(mi.GetString(), out var index))
|
||||
return ToolResult.Fail("'messageId' (index) is required.");
|
||||
var wanted = input.TryGetProperty("attachment", out var at) ? at.ToString().Trim() : "";
|
||||
if (wanted.Length == 0)
|
||||
return ToolResult.Fail("'attachment' is required — its number from read_message, or its file name.");
|
||||
|
||||
if (!TryReadAccess(context, "imapHost", "imapPort", 993, out var access, out var fehlt))
|
||||
return ToolResult.Fail(fehlt);
|
||||
|
||||
using var client = new ImapClient();
|
||||
await client.ConnectAsync(access.Host, access.Port, true, ct);
|
||||
await client.AuthenticateAsync(access.UserName, access.Password, ct);
|
||||
await client.Inbox!.OpenAsync(FolderAccess.ReadOnly, ct);
|
||||
var msg = await client.Inbox!.GetMessageAsync(index, ct);
|
||||
await client.DisconnectAsync(true, ct);
|
||||
|
||||
var attachments = Attachments(msg);
|
||||
var part = int.TryParse(wanted, out var number) && number >= 1 && number <= attachments.Count
|
||||
? attachments[number - 1]
|
||||
: attachments.FirstOrDefault(a => string.Equals(a.FileName, wanted, StringComparison.OrdinalIgnoreCase));
|
||||
if (part is null)
|
||||
return ToolResult.Fail($"Message {index} has no attachment '{wanted}' — it has {attachments.Count}.");
|
||||
|
||||
using var buffer = new MemoryStream();
|
||||
await part.Content.DecodeToAsync(buffer, ct);
|
||||
if (buffer.Length > MaxAttachmentBytes)
|
||||
return ToolResult.Fail($"The attachment is {buffer.Length / 1024 / 1024} MB — more than {MaxAttachmentBytes / 1024 / 1024} MB are not taken.");
|
||||
var name = part.FileName ?? $"attachment-{index}";
|
||||
|
||||
if (action == "read_attachment")
|
||||
{
|
||||
var text = ClawdDotNet.Core.Documents.DocumentText.Extract(buffer.ToArray(), name, MaxAttachmentChars);
|
||||
if (!text.Ok)
|
||||
return ToolResult.Fail($"{name}: {text.Problem} Keep it with save_attachment if a human should look at it.");
|
||||
return ToolResult.Ok($"Attachment '{name}' ({text.Kind}) of message {index}" +
|
||||
(text.Truncated ? $", first {MaxAttachmentChars} characters" : "") + ":\n" + text.Text);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(context.WorkspacePath))
|
||||
return ToolResult.Fail("You have no workspace to keep the attachment in.");
|
||||
var requested = input.TryGetProperty("fileName", out var fn) ? fn.GetString() : null;
|
||||
var fileName = SafeFileName(string.IsNullOrWhiteSpace(requested) ? name : requested);
|
||||
Directory.CreateDirectory(context.WorkspacePath);
|
||||
var target = Path.Combine(context.WorkspacePath, fileName);
|
||||
// Never over an existing file: name.pdf, name (2).pdf, ...
|
||||
for (var n = 2; File.Exists(target); n++)
|
||||
target = Path.Combine(context.WorkspacePath, $"{Path.GetFileNameWithoutExtension(fileName)} ({n}){Path.GetExtension(fileName)}");
|
||||
await File.WriteAllBytesAsync(target, buffer.ToArray(), ct);
|
||||
return ToolResult.Ok($"Attachment '{name}' of message {index} saved in your workspace as '{Path.GetFileName(target)}' ({buffer.Length} bytes).");
|
||||
}
|
||||
|
||||
/// <summary>A file name only — no directory, nothing that leaves the workspace.</summary>
|
||||
public static string SafeFileName(string name)
|
||||
{
|
||||
// The same answer on every system: the last segment, and the characters any of them refuses replaced.
|
||||
var bare = name.Replace('\\', '/').Split('/').Last().Trim();
|
||||
bare = new string(bare.Select(c => c < 32 || "<>:\"|?*".Contains(c) ? '_' : c).ToArray());
|
||||
bare = bare.TrimStart('.');
|
||||
return bare.Length == 0 ? "attachment" : bare.Length > 120 ? bare[..120] : bare;
|
||||
}
|
||||
|
||||
private async Task<ToolResult> HandleMarkReadAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||||
{
|
||||
if (!input.TryGetProperty("messageId", out var mi) || !int.TryParse(mi.GetString(), out var index))
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using ClawdDotNet.Core.Documents;
|
||||
using Shouldly;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Documents;
|
||||
|
||||
/// <summary>
|
||||
/// #692: The text of what really comes in — a Word order confirmation, a price list as a
|
||||
/// spreadsheet — without running anything in the file and without being unpacked to death.
|
||||
/// </summary>
|
||||
public sealed class DocumentTextTests
|
||||
{
|
||||
private const string W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main";
|
||||
private const string S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
|
||||
|
||||
public static byte[] Zip(params (string Name, string Content)[] parts)
|
||||
{
|
||||
using var buffer = new MemoryStream();
|
||||
using (var zip = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true))
|
||||
foreach (var (name, content) in parts)
|
||||
{
|
||||
using var stream = zip.CreateEntry(name).Open();
|
||||
stream.Write(Encoding.UTF8.GetBytes(content));
|
||||
}
|
||||
return buffer.ToArray();
|
||||
}
|
||||
|
||||
public static byte[] Docx(string body)
|
||||
=> Zip(("word/document.xml", $"""<?xml version="1.0" encoding="UTF-8"?><w:document xmlns:w="{W}"><w:body>{body}</w:body></w:document>"""));
|
||||
|
||||
[Fact]
|
||||
public void A_word_document_comes_out_as_paragraphs_and_table_rows()
|
||||
{
|
||||
var doc = Docx("""
|
||||
<w:p><w:r><w:t>Order confirmation</w:t></w:r></w:p>
|
||||
<w:p><w:r><w:t xml:space="preserve">No. </w:t></w:r><w:r><w:t>4711</w:t></w:r><w:r><w:tab/><w:t>due 2026-10-15</w:t></w:r></w:p>
|
||||
<w:tbl>
|
||||
<w:tr><w:tc><w:p><w:r><w:t>Item</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Qty</w:t></w:r></w:p></w:tc></w:tr>
|
||||
<w:tr><w:tc><w:p><w:r><w:t>Paper A4</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>40</w:t></w:r></w:p></w:tc></w:tr>
|
||||
</w:tbl>
|
||||
<w:p><w:r><w:t>Thank you.</w:t></w:r></w:p>
|
||||
""");
|
||||
|
||||
var text = DocumentText.Extract(doc, "confirmation.docx", 10_000);
|
||||
|
||||
text.Ok.ShouldBeTrue(text.Problem);
|
||||
text.Kind.ShouldBe("Word document");
|
||||
text.Text.ShouldContain("Order confirmation");
|
||||
text.Text.ShouldContain("No. 4711\tdue 2026-10-15");
|
||||
text.Text.ShouldContain("| Item | Qty |");
|
||||
text.Text.ShouldContain("| Paper A4 | 40 |");
|
||||
text.Text.ShouldEndWith("Thank you.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_spreadsheet_comes_out_sheet_by_sheet_with_every_cell_at_its_column()
|
||||
{
|
||||
var xlsx = Zip(
|
||||
("xl/workbook.xml", $"""<workbook xmlns="{S}" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Prices" sheetId="1" r:id="rId1"/><sheet name="Notes" sheetId="2" r:id="rId2"/></sheets></workbook>"""),
|
||||
("xl/_rels/workbook.xml.rels", """<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Target="worksheets/sheet1.xml"/><Relationship Id="rId2" Target="/xl/worksheets/sheet2.xml"/></Relationships>"""),
|
||||
("xl/sharedStrings.xml", $"""<sst xmlns="{S}"><si><t>Item</t></si><si><r><t>Pa</t></r><r><t>per</t></r><rPh><t>ignored</t></rPh></si><si><t>Price</t></si></sst>"""),
|
||||
("xl/worksheets/sheet1.xml", $"""
|
||||
<worksheet xmlns="{S}"><sheetData>
|
||||
<row r="1"><c r="A1" t="s"><v>0</v></c><c r="C1" t="s"><v>2</v></c></row>
|
||||
<row r="2"><c r="A2" t="s"><v>1</v></c><c r="B2" t="b"><v>1</v></c><c r="C2"><f>B2*3</f><v>4.5</v></c></row>
|
||||
</sheetData></worksheet>
|
||||
"""),
|
||||
("xl/worksheets/sheet2.xml", $"""<worksheet xmlns="{S}"><sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>Order by Friday</t></is></c></row></sheetData></worksheet>"""));
|
||||
|
||||
var text = DocumentText.Extract(xlsx, "prices.xlsx", 10_000);
|
||||
|
||||
text.Ok.ShouldBeTrue(text.Problem);
|
||||
text.Text.ShouldContain("## Sheet: Prices\nItem\t\tPrice\nPaper\tTRUE\t4.5".Replace("\n", Environment.NewLine));
|
||||
text.Text.ShouldContain("## Sheet: Notes");
|
||||
text.Text.ShouldContain("Order by Friday");
|
||||
text.Text.ShouldNotContain("ignored", customMessage: "phonetic hints are no content");
|
||||
text.Text.ShouldNotContain("B2*3", customMessage: "the stored value is read, the formula is not");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_document_with_a_dtd_is_refused_not_resolved()
|
||||
{
|
||||
var doc = Zip(("word/document.xml", $"""<?xml version="1.0"?><!DOCTYPE x [<!ENTITY e SYSTEM "file:///etc/passwd">]><w:document xmlns:w="{W}"><w:body><w:p><w:r><w:t>&e;</w:t></w:r></w:p></w:body></w:document>"""));
|
||||
|
||||
var text = DocumentText.Extract(doc, "evil.docx", 10_000);
|
||||
|
||||
text.Ok.ShouldBeFalse();
|
||||
text.Problem!.ShouldContain("could not be read");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_part_that_unpacks_beyond_the_limit_is_stopped()
|
||||
{
|
||||
var huge = new string('a', (int)DocumentText.MaxPartBytes + 1024);
|
||||
var doc = Docx($"<w:p><w:r><w:t>{huge}</w:t></w:r></w:p>");
|
||||
|
||||
DocumentText.Extract(doc, "bomb.docx", 100).Ok.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Text_is_cut_at_the_limit_and_odd_bytes_are_no_error()
|
||||
{
|
||||
var cut = DocumentText.Extract(Encoding.UTF8.GetBytes(new string('x', 500)), "a.csv", 100);
|
||||
cut.Truncated.ShouldBeTrue();
|
||||
cut.Text.Length.ShouldBe(100);
|
||||
|
||||
DocumentText.Extract([0x50, 0x72, 0x65, 0x69, 0x73, 0x20, 0xE4], "latin.txt", 100).Text.ShouldBe("Preis ä");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pdf_and_unknown_files_are_named_not_guessed()
|
||||
{
|
||||
DocumentText.Extract([1, 2, 3], "invoice.pdf", 100).Problem!.ShouldContain("PDF");
|
||||
DocumentText.Extract([1, 2, 3], "image.png", 100).Ok.ShouldBeFalse();
|
||||
DocumentText.Extract([1, 2, 3], "broken.docx", 100).Ok.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@
|
||||
<ProjectReference Include="..\..\src\ClawdDotNet.Tools.FileRW\ClawdDotNet.Tools.FileRW.csproj" />
|
||||
<ProjectReference Include="..\..\src\ClawdDotNet.Tools.FTP\ClawdDotNet.Tools.FTP.csproj" />
|
||||
<ProjectReference Include="..\..\src\ClawdDotNet.Tools.RocketChat\ClawdDotNet.Tools.RocketChat.csproj" />
|
||||
<ProjectReference Include="..\..\src\ClawdDotNet.Tools.Mail\ClawdDotNet.Tools.Mail.csproj" />
|
||||
<ProjectReference Include="..\..\src\ClawdDotNet.Tools.Memory\ClawdDotNet.Tools.Memory.csproj" />
|
||||
<ProjectReference Include="..\..\src\ClawdDotNet.Tools.SocialMediaManager\ClawdDotNet.Tools.SocialMediaManager.csproj" />
|
||||
<ProjectReference Include="..\..\src\ClawdDotNet.Tools.Taskboard\ClawdDotNet.Tools.Taskboard.csproj" />
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using ClawdDotNet.Core.Tools;
|
||||
using ClawdDotNet.Tools.FileRW;
|
||||
using ClawdDotNet.Tools.Mail;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
|
||||
namespace ClawdDotNet.Tools.Tests.FileRW;
|
||||
|
||||
/// <summary>#692: A Word file in the workspace is read as its text, and an attachment's name never leaves it.</summary>
|
||||
public sealed class DocumentReadTests : IDisposable
|
||||
{
|
||||
private readonly string _workspace = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N"), "Workspace");
|
||||
|
||||
public DocumentReadTests() => Directory.CreateDirectory(_workspace);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try { Directory.Delete(Path.GetDirectoryName(_workspace)!, recursive: true); }
|
||||
catch { /* cleanup is secondary */ }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Read_returns_the_text_of_a_word_document()
|
||||
{
|
||||
using (var zip = ZipFile.Open(Path.Combine(_workspace, "order.docx"), ZipArchiveMode.Create))
|
||||
using (var stream = zip.CreateEntry("word/document.xml").Open())
|
||||
stream.Write(Encoding.UTF8.GetBytes(
|
||||
"""<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>40 x Paper A4</w:t></w:r></w:p></w:body></w:document>"""));
|
||||
|
||||
var context = new AgentToolContext("buyer", "inst", new Dictionary<string, object?>(), new NullStateStore(), NullLogger.Instance,
|
||||
CancellationToken.None, WorkspacePath: _workspace);
|
||||
var result = await new FileRWTool().ExecuteAsync(JsonDocument.Parse("""{"action":"read","path":"order.docx"}""").RootElement, context, CancellationToken.None);
|
||||
|
||||
result.Success.ShouldBeTrue(result.ErrorMessage);
|
||||
result.Content.ShouldContain("text of a Word document");
|
||||
result.Content.ShouldContain("40 x Paper A4");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Writing_a_word_document_stays_refused()
|
||||
{
|
||||
var context = new AgentToolContext("buyer", "inst", new Dictionary<string, object?>(), new NullStateStore(), NullLogger.Instance,
|
||||
CancellationToken.None, WorkspacePath: _workspace);
|
||||
var result = await new FileRWTool().ExecuteAsync(
|
||||
JsonDocument.Parse("""{"action":"write","path":"fake.docx","content":"x"}""").RootElement, context, CancellationToken.None);
|
||||
|
||||
result.Success.ShouldBeFalse("only reading a document needs no place on the list");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("../../etc/passwd", "passwd")]
|
||||
[InlineData("..\\..\\Windows\\win.ini", "win.ini")]
|
||||
[InlineData("C:/x/invoice.pdf", "invoice.pdf")]
|
||||
[InlineData(".hidden", "hidden")]
|
||||
[InlineData("", "attachment")]
|
||||
[InlineData("a:b?c.txt", "a_b_c.txt")]
|
||||
public void An_attachment_name_is_a_file_name_and_nothing_more(string given, string expected)
|
||||
=> MailTool.SafeFileName(given).ShouldBe(expected);
|
||||
}
|
||||
Reference in New Issue
Block a user