Werkzeugkasten: Anhaenge lesen - Word, Excel, Text (#692)
- Core/Documents/DocumentText: text of .docx (paragraphs, tables as rows), .xlsx (sheet by sheet, every cell at its column, stored values not formulas) and plain text/CSV. No DTD, no external references, every part bounded against zip bombs. PDF is named as not readable yet. - Mail: read_message lists attachments; read_attachment returns the text; save_attachment keeps the file in the agent's workspace (never over an existing file, never outside). - FileRW read/read_many return Word and Excel files as text; writing them stays refused. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
edf3ff19ea
commit
72d855efcf
@@ -0,0 +1,412 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using System.Xml;
|
||||
|
||||
namespace ClawdDotNet.Core.Documents;
|
||||
|
||||
/// <summary>What came out of a document — the text, and what the reader should know about it.</summary>
|
||||
public sealed record DocumentExtraction(bool Ok, string Text, string Kind, bool Truncated, string? Problem = null)
|
||||
{
|
||||
public static DocumentExtraction Fail(string kind, string problem) => new(false, "", kind, false, problem);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #692: The text of what really comes in — an order confirmation as a Word file, a price list
|
||||
/// as a spreadsheet. The least spectacular and most used missing ability of a company's agents.
|
||||
///
|
||||
/// <para><b>What comes out is foreign content.</b> A supplier's file is no more trustworthy than
|
||||
/// a web page; the engine frames every tool result as such (#193), this class only turns bytes
|
||||
/// into text. It runs nothing in the file: no macros, no formulas (the stored value is read),
|
||||
/// no external references, no DTDs.</para>
|
||||
///
|
||||
/// <para><b>Bounded.</b> Office files are zip archives, and a zip can unpack to anything. Every
|
||||
/// part is read up to <see cref="MaxPartBytes"/>, the whole up to the caller's character limit.</para>
|
||||
///
|
||||
/// <para>PDF is not read here yet: a reliable PDF text extractor is a library decision, not a
|
||||
/// hundred lines of code — fonts with their own encodings come out as garbage otherwise.</para>
|
||||
/// </summary>
|
||||
public static class DocumentText
|
||||
{
|
||||
public const long MaxPartBytes = 32L * 1024 * 1024;
|
||||
|
||||
private static readonly HashSet<string> PlainText = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
".txt", ".csv", ".tsv", ".md", ".json", ".xml", ".html", ".htm", ".log", ".yaml", ".yml", ".ini", ".eml"
|
||||
};
|
||||
|
||||
/// <summary>Whether <see cref="Extract"/> reads this kind of file.</summary>
|
||||
public static bool Supports(string fileName)
|
||||
=> PlainText.Contains(Path.GetExtension(fileName)) || IsOfficeDocument(fileName);
|
||||
|
||||
/// <summary>Whether the file is a document whose text needs extracting (not plain text).</summary>
|
||||
public static bool IsOfficeDocument(string fileName)
|
||||
=> Path.GetExtension(fileName).ToLowerInvariant() is ".docx" or ".xlsx";
|
||||
|
||||
public static DocumentExtraction Extract(byte[] content, string fileName, int maxChars)
|
||||
{
|
||||
var ext = Path.GetExtension(fileName).ToLowerInvariant();
|
||||
try
|
||||
{
|
||||
return ext switch
|
||||
{
|
||||
".docx" => Bounded(Docx(content), "Word document", maxChars),
|
||||
".xlsx" => Bounded(Xlsx(content), "spreadsheet", maxChars),
|
||||
".pdf" => DocumentExtraction.Fail("PDF", "PDF text cannot be read in this version yet."),
|
||||
_ when PlainText.Contains(ext) => Bounded(Decode(content), "text", maxChars),
|
||||
_ => DocumentExtraction.Fail(ext.TrimStart('.'), $"'{ext}' files cannot be read as text.")
|
||||
};
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or XmlException or IOException or InvalidOperationException)
|
||||
{
|
||||
return DocumentExtraction.Fail(ext.TrimStart('.'), $"The file could not be read: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static DocumentExtraction Bounded(string text, string kind, int maxChars)
|
||||
{
|
||||
var limit = Math.Max(1, maxChars);
|
||||
return text.Length <= limit
|
||||
? new DocumentExtraction(true, text, kind, false)
|
||||
: new DocumentExtraction(true, text[..limit], kind, true);
|
||||
}
|
||||
|
||||
/// <summary>UTF-8 with or without BOM, UTF-16 by BOM, otherwise Latin-1 — never an exception for odd bytes.</summary>
|
||||
private static string Decode(byte[] content)
|
||||
{
|
||||
if (content.Length >= 2 && content[0] == 0xFF && content[1] == 0xFE)
|
||||
return Encoding.Unicode.GetString(content, 2, content.Length - 2);
|
||||
if (content.Length >= 2 && content[0] == 0xFE && content[1] == 0xFF)
|
||||
return Encoding.BigEndianUnicode.GetString(content, 2, content.Length - 2);
|
||||
var start = content.Length >= 3 && content[0] == 0xEF && content[1] == 0xBB && content[2] == 0xBF ? 3 : 0;
|
||||
try
|
||||
{
|
||||
return new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(content, start, content.Length - start);
|
||||
}
|
||||
catch (DecoderFallbackException)
|
||||
{
|
||||
return Encoding.Latin1.GetString(content);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsText(XmlNodeType type)
|
||||
=> type is XmlNodeType.Text or XmlNodeType.CDATA or XmlNodeType.SignificantWhitespace or XmlNodeType.Whitespace;
|
||||
|
||||
// ─── Word ───
|
||||
|
||||
private const string W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main";
|
||||
|
||||
private static string Docx(byte[] content)
|
||||
{
|
||||
using var zip = new ZipArchive(new MemoryStream(content), ZipArchiveMode.Read);
|
||||
var part = zip.GetEntry("word/document.xml") ?? throw new InvalidDataException("This is no Word document (word/document.xml is missing).");
|
||||
|
||||
var sb = new StringBuilder();
|
||||
var cells = new List<string>();
|
||||
var paragraph = new StringBuilder();
|
||||
var tableDepth = 0;
|
||||
var inText = false;
|
||||
using var reader = Xml(part);
|
||||
while (reader.Read())
|
||||
{
|
||||
if (inText && IsText(reader.NodeType))
|
||||
{
|
||||
paragraph.Append(reader.Value);
|
||||
continue;
|
||||
}
|
||||
if (reader.NamespaceURI != W)
|
||||
continue;
|
||||
if (reader.NodeType == XmlNodeType.Element)
|
||||
{
|
||||
switch (reader.LocalName)
|
||||
{
|
||||
case "t":
|
||||
inText = !reader.IsEmptyElement;
|
||||
break;
|
||||
case "tab":
|
||||
paragraph.Append('\t');
|
||||
break;
|
||||
case "br" or "cr":
|
||||
paragraph.Append('\n');
|
||||
break;
|
||||
case "tbl":
|
||||
tableDepth++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (reader.NodeType == XmlNodeType.EndElement)
|
||||
{
|
||||
switch (reader.LocalName)
|
||||
{
|
||||
case "t":
|
||||
inText = false;
|
||||
break;
|
||||
case "p":
|
||||
EndParagraph();
|
||||
break;
|
||||
case "tc":
|
||||
cells.Add(paragraph.ToString().Trim());
|
||||
paragraph.Clear();
|
||||
break;
|
||||
case "tr":
|
||||
sb.AppendLine("| " + string.Join(" | ", cells) + " |");
|
||||
cells.Clear();
|
||||
break;
|
||||
case "tbl":
|
||||
tableDepth--;
|
||||
sb.AppendLine();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
EndParagraph();
|
||||
return sb.ToString().Trim();
|
||||
|
||||
void EndParagraph()
|
||||
{
|
||||
if (tableDepth > 0)
|
||||
{
|
||||
// Inside a cell, paragraphs stay together until the cell ends.
|
||||
if (paragraph.Length > 0 && paragraph[^1] != ' ')
|
||||
paragraph.Append(' ');
|
||||
return;
|
||||
}
|
||||
if (paragraph.Length > 0)
|
||||
sb.AppendLine(paragraph.ToString());
|
||||
paragraph.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Excel ───
|
||||
|
||||
private const string S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
|
||||
private const string R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships";
|
||||
private const string Pkg = "http://schemas.openxmlformats.org/package/2006/relationships";
|
||||
|
||||
private static string Xlsx(byte[] content)
|
||||
{
|
||||
using var zip = new ZipArchive(new MemoryStream(content), ZipArchiveMode.Read);
|
||||
var workbook = zip.GetEntry("xl/workbook.xml") ?? throw new InvalidDataException("This is no spreadsheet (xl/workbook.xml is missing).");
|
||||
|
||||
var shared = zip.GetEntry("xl/sharedStrings.xml") is { } strings ? SharedStrings(strings) : [];
|
||||
|
||||
var targets = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
if (zip.GetEntry("xl/_rels/workbook.xml.rels") is { } rels)
|
||||
{
|
||||
using var reader = Xml(rels);
|
||||
while (reader.Read())
|
||||
if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "Relationship" && reader.NamespaceURI == Pkg
|
||||
&& reader.GetAttribute("Id") is { } id && reader.GetAttribute("Target") is { } target)
|
||||
targets[id] = target.StartsWith('/') ? target.TrimStart('/') : "xl/" + target;
|
||||
}
|
||||
|
||||
var sheets = new List<(string Name, string Path)>();
|
||||
using (var reader = Xml(workbook))
|
||||
while (reader.Read())
|
||||
if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "sheet" && reader.NamespaceURI == S
|
||||
&& reader.GetAttribute("id", R) is { } rid && targets.TryGetValue(rid, out var path))
|
||||
sheets.Add((reader.GetAttribute("name") ?? path, path));
|
||||
|
||||
var sb = new StringBuilder();
|
||||
foreach (var (name, path) in sheets)
|
||||
{
|
||||
if (zip.GetEntry(path) is not { } sheet)
|
||||
continue;
|
||||
sb.AppendLine($"## Sheet: {name}");
|
||||
foreach (var row in Rows(sheet, shared))
|
||||
sb.AppendLine(string.Join('\t', row));
|
||||
sb.AppendLine();
|
||||
}
|
||||
return sb.ToString().Trim();
|
||||
}
|
||||
|
||||
/// <summary>The shared strings, in order — rich text joined, phonetic hints left out.</summary>
|
||||
private static List<string> SharedStrings(ZipArchiveEntry entry)
|
||||
{
|
||||
var shared = new List<string>();
|
||||
using var reader = Xml(entry);
|
||||
StringBuilder? item = null;
|
||||
var inText = false;
|
||||
var phonetic = 0;
|
||||
while (reader.Read())
|
||||
{
|
||||
if (inText && phonetic == 0 && IsText(reader.NodeType))
|
||||
{
|
||||
item?.Append(reader.Value);
|
||||
continue;
|
||||
}
|
||||
if (reader.NamespaceURI != S)
|
||||
continue;
|
||||
if (reader.NodeType == XmlNodeType.Element)
|
||||
{
|
||||
if (reader.LocalName == "si")
|
||||
item = new StringBuilder();
|
||||
else if (reader.LocalName == "t")
|
||||
inText = !reader.IsEmptyElement;
|
||||
else if (reader.LocalName == "rPh" && !reader.IsEmptyElement)
|
||||
phonetic++;
|
||||
}
|
||||
else if (reader.NodeType == XmlNodeType.EndElement)
|
||||
{
|
||||
if (reader.LocalName == "t")
|
||||
inText = false;
|
||||
else if (reader.LocalName == "rPh")
|
||||
phonetic--;
|
||||
else if (reader.LocalName == "si" && item is not null)
|
||||
{
|
||||
shared.Add(item.ToString());
|
||||
item = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
return shared;
|
||||
}
|
||||
|
||||
/// <summary>The rows of a sheet, each cell at its column — the value as stored, formulas not run.</summary>
|
||||
private static List<List<string>> Rows(ZipArchiveEntry sheet, List<string> shared)
|
||||
{
|
||||
var rows = new List<List<string>>();
|
||||
using var reader = Xml(sheet);
|
||||
List<string>? row = null;
|
||||
string? type = null, reference = null;
|
||||
StringBuilder? value = null;
|
||||
var inValue = false;
|
||||
while (reader.Read())
|
||||
{
|
||||
if (inValue && IsText(reader.NodeType))
|
||||
{
|
||||
value?.Append(reader.Value);
|
||||
continue;
|
||||
}
|
||||
if (reader.NamespaceURI != S)
|
||||
continue;
|
||||
if (reader.NodeType == XmlNodeType.Element)
|
||||
{
|
||||
switch (reader.LocalName)
|
||||
{
|
||||
case "row":
|
||||
row = [];
|
||||
if (reader.IsEmptyElement)
|
||||
{
|
||||
rows.Add(row);
|
||||
row = null;
|
||||
}
|
||||
break;
|
||||
case "c":
|
||||
type = reader.GetAttribute("t");
|
||||
reference = reader.GetAttribute("r");
|
||||
value = new StringBuilder();
|
||||
if (reader.IsEmptyElement)
|
||||
Place(row, reference, "");
|
||||
break;
|
||||
case "v" or "t":
|
||||
inValue = !reader.IsEmptyElement;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (reader.NodeType == XmlNodeType.EndElement)
|
||||
{
|
||||
switch (reader.LocalName)
|
||||
{
|
||||
case "v" or "t":
|
||||
inValue = false;
|
||||
break;
|
||||
case "c":
|
||||
Place(row, reference, Cell(type, value?.ToString(), shared));
|
||||
value = null;
|
||||
break;
|
||||
case "row" when row is not null:
|
||||
rows.Add(row);
|
||||
row = null;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
private static string Cell(string? type, string? value, List<string> shared)
|
||||
=> type switch
|
||||
{
|
||||
"s" when int.TryParse(value, out var i) && i >= 0 && i < shared.Count => shared[i],
|
||||
"b" => value == "1" ? "TRUE" : "FALSE",
|
||||
_ => value ?? ""
|
||||
};
|
||||
|
||||
/// <summary>Puts the value at its column (from "C7"), filling gaps with empty cells.</summary>
|
||||
private static void Place(List<string>? row, string? reference, string value)
|
||||
{
|
||||
if (row is null)
|
||||
return;
|
||||
var column = row.Count;
|
||||
if (!string.IsNullOrEmpty(reference))
|
||||
{
|
||||
var letters = 0;
|
||||
var number = 0;
|
||||
foreach (var ch in reference)
|
||||
{
|
||||
if (ch is < 'A' or > 'Z')
|
||||
break;
|
||||
number = number * 26 + (ch - 'A' + 1);
|
||||
letters++;
|
||||
}
|
||||
if (letters is > 0 and <= 3)
|
||||
column = number - 1;
|
||||
}
|
||||
// A column beyond the sheet's width, or one already filled, goes at the end.
|
||||
if (column > 16_384 || column < row.Count)
|
||||
column = row.Count;
|
||||
while (row.Count < column)
|
||||
row.Add("");
|
||||
row.Add(value);
|
||||
}
|
||||
|
||||
// ─── shared ───
|
||||
|
||||
/// <summary>A reader for one part: no DTD, no external entities, bounded size.</summary>
|
||||
private static XmlReader Xml(ZipArchiveEntry entry)
|
||||
{
|
||||
if (entry.Length > MaxPartBytes)
|
||||
throw new InvalidDataException($"'{entry.FullName}' unpacks to {entry.Length / 1024 / 1024} MB — more than a document should.");
|
||||
return XmlReader.Create(new BoundedStream(entry.Open(), MaxPartBytes), new XmlReaderSettings
|
||||
{
|
||||
DtdProcessing = DtdProcessing.Prohibit,
|
||||
XmlResolver = null,
|
||||
IgnoreComments = true,
|
||||
IgnoreProcessingInstructions = true,
|
||||
CloseInput = true
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Stops reading at a limit even if the archive lied about the size.</summary>
|
||||
private sealed class BoundedStream(Stream inner, long limit) : Stream
|
||||
{
|
||||
private long _read;
|
||||
|
||||
public override bool CanRead => true;
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => throw new NotSupportedException();
|
||||
public override long Position { get => _read; set => throw new NotSupportedException(); }
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
var n = inner.Read(buffer, offset, count);
|
||||
_read += n;
|
||||
if (_read > limit)
|
||||
throw new InvalidDataException("A part of the document unpacks to more than a document should.");
|
||||
return n;
|
||||
}
|
||||
|
||||
public override void Flush() { }
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
inner.Dispose();
|
||||
base.Dispose(disposing);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user