From 347b35fc44270314e6cd4798f5a3d540ecfb5d11 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 1 Oct 2026 18:45:03 +0200 Subject: [PATCH] Werkzeugkasten: PDF-Text aus Anhaengen und Dateien (#692) - DocumentText reads PDF with PdfPig 0.1.16 (managed, Apache-2.0): page by page in reading order, at most 300 pages; a PDF without a text layer is named as probably scanned; broken and protected files are named. - Mail read_attachment and FileRW read/read_many take PDF like Word/Excel. - nuget.config maps exactly "PdfPig" - "UglyToad.PdfPig" is a package of a different account and stays out. Co-Authored-By: Claude Opus 5.5 --- NuGet.Config | 3 ++ docs/Organisation-Konzept.md | 7 ++- src/ClawdDotNet.Core/ClawdDotNet.Core.csproj | 5 ++ .../Documents/DocumentText.cs | 49 +++++++++++++++++-- src/ClawdDotNet.Tools.Mail/MailTool.cs | 2 +- .../Documents/DocumentTextTests.cs | 32 +++++++++++- 6 files changed, 91 insertions(+), 7 deletions(-) diff --git a/NuGet.Config b/NuGet.Config index c8045a0..ee7daf3 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -28,6 +28,9 @@ + + diff --git a/docs/Organisation-Konzept.md b/docs/Organisation-Konzept.md index 5f3fda4..6bad2f2 100644 --- a/docs/Organisation-Konzept.md +++ b/docs/Organisation-Konzept.md @@ -806,7 +806,12 @@ beidem. > legt die Datei im eigenen Arbeitsbereich ab (nie über eine vorhandene, nie außerhalb). > FileRW `read`/`read_many` gibt Word- und Excel-Dateien als Text zurück; schreiben kann > es sie weiterhin nicht. Der Text ist Fremdinhalt und wird von der Engine gerahmt. - > **Offen: PDF** — ein verlässlicher PDF-Textauszug ist eine Bibliotheksentscheidung. + > **PDF** liest PdfPig (Paket `PdfPig`, Apache-2.0, nur verwalteter Code; Entscheidung des + > Nutzers vom 2026-10-01 für eine lokale Bibliothek statt der Auslese über OpenRouter): + > Seite für Seite in Leserichtung, höchstens 300 Seiten. Ein gescanntes PDF ohne Textebene + > wird als solches benannt — OCR gibt es nicht. Achtung beim Paketnamen: unter + > `UglyToad.PdfPig` liegt eine Vorabversion eines fremden Kontos; `nuget.config` lässt nur + > genau `PdfPig` zu. 3. **Der MCP-Client (#197) bleibt nützlich** — aber als *eine* Fähigkeit unter anderen, für Dienste, die ohnehin als Server bereitstehen. Nicht als Anbindungsstrategie. diff --git a/src/ClawdDotNet.Core/ClawdDotNet.Core.csproj b/src/ClawdDotNet.Core/ClawdDotNet.Core.csproj index 1829da5..dd61c68 100644 --- a/src/ClawdDotNet.Core/ClawdDotNet.Core.csproj +++ b/src/ClawdDotNet.Core/ClawdDotNet.Core.csproj @@ -23,6 +23,11 @@ + + + diff --git a/src/ClawdDotNet.Core/Documents/DocumentText.cs b/src/ClawdDotNet.Core/Documents/DocumentText.cs index 6d71a77..2c669da 100644 --- a/src/ClawdDotNet.Core/Documents/DocumentText.cs +++ b/src/ClawdDotNet.Core/Documents/DocumentText.cs @@ -22,8 +22,8 @@ public sealed record DocumentExtraction(bool Ok, string Text, string Kind, bool /// Bounded. Office files are zip archives, and a zip can unpack to anything. Every /// part is read up to , the whole up to the caller's character limit. /// -/// PDF is not read here yet: a reliable PDF text extractor is a library decision, not a -/// hundred lines of code — fonts with their own encodings come out as garbage otherwise. +/// PDF is read with PdfPig (managed code only): the text of each page in reading +/// order. A scanned PDF has no text layer and is named as such — reading it would need OCR. /// public static class DocumentText { @@ -40,7 +40,10 @@ public static class DocumentText /// Whether the file is a document whose text needs extracting (not plain text). public static bool IsOfficeDocument(string fileName) - => Path.GetExtension(fileName).ToLowerInvariant() is ".docx" or ".xlsx"; + => Path.GetExtension(fileName).ToLowerInvariant() is ".docx" or ".xlsx" or ".pdf"; + + /// Pages read from one PDF — an invoice has a few, a catalogue is not read whole. + public const int MaxPdfPages = 300; public static DocumentExtraction Extract(byte[] content, string fileName, int maxChars) { @@ -51,7 +54,7 @@ public static class DocumentText { ".docx" => Bounded(Docx(content), "Word document", maxChars), ".xlsx" => Bounded(Xlsx(content), "spreadsheet", maxChars), - ".pdf" => DocumentExtraction.Fail("PDF", "PDF text cannot be read in this version yet."), + ".pdf" => Pdf(content, maxChars), _ when PlainText.Contains(ext) => Bounded(Decode(content), "text", maxChars), _ => DocumentExtraction.Fail(ext.TrimStart('.'), $"'{ext}' files cannot be read as text.") }; @@ -62,6 +65,44 @@ public static class DocumentText } } + // ─── PDF ─── + + private static DocumentExtraction Pdf(byte[] content, int maxChars) + { + try + { + using var document = UglyToad.PdfPig.PdfDocument.Open(content, new UglyToad.PdfPig.ParsingOptions { UseLenientParsing = true }); + var sb = new StringBuilder(); + var pages = document.NumberOfPages; + var read = 0; + foreach (var page in document.GetPages()) + { + if (read >= MaxPdfPages || sb.Length > maxChars) + break; + read++; + var text = UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor.ContentOrderTextExtractor.GetText(page).Trim(); + if (pages > 1) + sb.AppendLine($"## Page {page.Number}"); + sb.AppendLine(text).AppendLine(); + } + + var all = sb.ToString().Trim(); + if (all.Replace("## Page", "").All(c => char.IsWhiteSpace(c) || char.IsDigit(c))) + return DocumentExtraction.Fail("PDF", "The PDF has no text layer — it is probably scanned, and reading it would need OCR."); + if (read < pages) + all += $"\n\n(Read {read} of {pages} pages.)"; + var bounded = Bounded(all, "PDF", maxChars); + return bounded with { Truncated = bounded.Truncated || read < pages }; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + // PdfPig throws its own exceptions for broken and encrypted files. + return DocumentExtraction.Fail("PDF", ex.Message.Contains("password", StringComparison.OrdinalIgnoreCase) || ex.Message.Contains("encrypt", StringComparison.OrdinalIgnoreCase) + ? "The PDF is protected by a password." + : $"The PDF could not be read: {ex.Message}"); + } + } + private static DocumentExtraction Bounded(string text, string kind, int maxChars) { var limit = Math.Max(1, maxChars); diff --git a/src/ClawdDotNet.Tools.Mail/MailTool.cs b/src/ClawdDotNet.Tools.Mail/MailTool.cs index 61cbfd0..c33afc9 100644 --- a/src/ClawdDotNet.Tools.Mail/MailTool.cs +++ b/src/ClawdDotNet.Tools.Mail/MailTool.cs @@ -18,7 +18,7 @@ public sealed class MailTool : IAgentTool, IToolJobProvider /// #715: Brings text from outside into the context — mails from anyone. public bool ReadsForeignContent => true; - public string Description => "Allows sending and receiving emails via SMTP and IMAP. read_message lists a message's attachments; read_attachment returns an attachment's text (Word, Excel, text and CSV files); save_attachment stores it in your workspace."; + public string Description => "Allows sending and receiving emails via SMTP and IMAP. read_message lists a message's attachments; read_attachment returns an attachment's text (PDF, Word, Excel, text and CSV files); save_attachment stores it in your workspace."; public JsonElement InputSchema { get; } = JsonDocument.Parse(""" { diff --git a/tests/ClawdDotNet.Core.Tests/Documents/DocumentTextTests.cs b/tests/ClawdDotNet.Core.Tests/Documents/DocumentTextTests.cs index f1a510f..8a6bca9 100644 --- a/tests/ClawdDotNet.Core.Tests/Documents/DocumentTextTests.cs +++ b/tests/ClawdDotNet.Core.Tests/Documents/DocumentTextTests.cs @@ -109,10 +109,40 @@ public sealed class DocumentTextTests } [Fact] - public void Pdf_and_unknown_files_are_named_not_guessed() + public void Broken_and_unknown_files_are_named_not_guessed() { DocumentText.Extract([1, 2, 3], "invoice.pdf", 100).Problem!.ShouldContain("PDF"); DocumentText.Extract([1, 2, 3], "image.png", 100).Ok.ShouldBeFalse(); DocumentText.Extract([1, 2, 3], "broken.docx", 100).Ok.ShouldBeFalse(); } + + private static byte[] Pdf(params string?[] pages) + { + var builder = new UglyToad.PdfPig.Writer.PdfDocumentBuilder(); + var font = builder.AddStandard14Font(UglyToad.PdfPig.Fonts.Standard14Fonts.Standard14Font.Helvetica); + foreach (var text in pages) + { + var page = builder.AddPage(UglyToad.PdfPig.Content.PageSize.A4); + if (text is not null) + page.AddText(text, 12, new UglyToad.PdfPig.Core.PdfPoint(40, 700), font); + } + return builder.Build(); + } + + [Fact] + public void A_pdf_comes_out_page_by_page() + { + var text = DocumentText.Extract(Pdf("Invoice 4711", "Total 150.00 EUR"), "invoice.pdf", 10_000); + + text.Ok.ShouldBeTrue(text.Problem); + text.Kind.ShouldBe("PDF"); + text.Text.ShouldContain("## Page 1"); + text.Text.ShouldContain("Invoice 4711"); + text.Text.ShouldContain("## Page 2"); + text.Text.ShouldContain("Total 150.00 EUR"); + } + + [Fact] + public void A_pdf_without_text_is_named_as_probably_scanned() + => DocumentText.Extract(Pdf((string?)null), "scan.pdf", 10_000).Problem!.ShouldContain("no text layer"); }