ClawdDotNet/tests/ClawdDotNet.Core.Tests/Documents/DocumentTextTests.cs

using System.IO.Compression;
using System.Text;
using ClawdDotNet.Core.Documents;
using Shouldly;

namespace ClawdDotNet.Core.Tests.Documents;

/// <summary>
/// #692: The text of what really comes in — a Word order confirmation, a price list as a
/// spreadsheet — without running anything in the file and without being unpacked to death.
/// </summary>
public sealed class DocumentTextTests
{
    private const string W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main";
    private const string S = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";

    public static byte[] Zip(params (string Name, string Content)[] parts)
    {
        using var buffer = new MemoryStream();
        using (var zip = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true))
            foreach (var (name, content) in parts)
            {
                using var stream = zip.CreateEntry(name).Open();
                stream.Write(Encoding.UTF8.GetBytes(content));
            }
        return buffer.ToArray();
    }

    public static byte[] Docx(string body)
        => Zip(("word/document.xml", $"""<?xml version="1.0" encoding="UTF-8"?><w:document xmlns:w="{W}"><w:body>{body}</w:body></w:document>"""));

    [Fact]
    public void A_word_document_comes_out_as_paragraphs_and_table_rows()
    {
        var doc = Docx("""
            <w:p><w:r><w:t>Order confirmation</w:t></w:r></w:p>
            <w:p><w:r><w:t xml:space="preserve">No. </w:t></w:r><w:r><w:t>4711</w:t></w:r><w:r><w:tab/><w:t>due 2026-10-15</w:t></w:r></w:p>
            <w:tbl>
              <w:tr><w:tc><w:p><w:r><w:t>Item</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Qty</w:t></w:r></w:p></w:tc></w:tr>
              <w:tr><w:tc><w:p><w:r><w:t>Paper A4</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>40</w:t></w:r></w:p></w:tc></w:tr>
            </w:tbl>
            <w:p><w:r><w:t>Thank you.</w:t></w:r></w:p>
            """);

        var text = DocumentText.Extract(doc, "confirmation.docx", 10_000);

        text.Ok.ShouldBeTrue(text.Problem);
        text.Kind.ShouldBe("Word document");
        text.Text.ShouldContain("Order confirmation");
        text.Text.ShouldContain("No. 4711\tdue 2026-10-15");
        text.Text.ShouldContain("| Item | Qty |");
        text.Text.ShouldContain("| Paper A4 | 40 |");
        text.Text.ShouldEndWith("Thank you.");
    }

    [Fact]
    public void A_spreadsheet_comes_out_sheet_by_sheet_with_every_cell_at_its_column()
    {
        var xlsx = Zip(
            ("xl/workbook.xml", $"""<workbook xmlns="{S}" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Prices" sheetId="1" r:id="rId1"/><sheet name="Notes" sheetId="2" r:id="rId2"/></sheets></workbook>"""),
            ("xl/_rels/workbook.xml.rels", """<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Target="worksheets/sheet1.xml"/><Relationship Id="rId2" Target="/xl/worksheets/sheet2.xml"/></Relationships>"""),
            ("xl/sharedStrings.xml", $"""<sst xmlns="{S}"><si><t>Item</t></si><si><r><t>Pa</t></r><r><t>per</t></r><rPh><t>ignored</t></rPh></si><si><t>Price</t></si></sst>"""),
            ("xl/worksheets/sheet1.xml", $"""
                <worksheet xmlns="{S}"><sheetData>
                  <row r="1"><c r="A1" t="s"><v>0</v></c><c r="C1" t="s"><v>2</v></c></row>
                  <row r="2"><c r="A2" t="s"><v>1</v></c><c r="B2" t="b"><v>1</v></c><c r="C2"><f>B2*3</f><v>4.5</v></c></row>
                </sheetData></worksheet>
                """),
            ("xl/worksheets/sheet2.xml", $"""<worksheet xmlns="{S}"><sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>Order by Friday</t></is></c></row></sheetData></worksheet>"""));

        var text = DocumentText.Extract(xlsx, "prices.xlsx", 10_000);

        text.Ok.ShouldBeTrue(text.Problem);
        text.Text.ShouldContain("## Sheet: Prices\nItem\t\tPrice\nPaper\tTRUE\t4.5".Replace("\n", Environment.NewLine));
        text.Text.ShouldContain("## Sheet: Notes");
        text.Text.ShouldContain("Order by Friday");
        text.Text.ShouldNotContain("ignored", customMessage: "phonetic hints are no content");
        text.Text.ShouldNotContain("B2*3", customMessage: "the stored value is read, the formula is not");
    }

    [Fact]
    public void A_document_with_a_dtd_is_refused_not_resolved()
    {
        var doc = Zip(("word/document.xml", $"""<?xml version="1.0"?><!DOCTYPE x [<!ENTITY e SYSTEM "file:///etc/passwd">]><w:document xmlns:w="{W}"><w:body><w:p><w:r><w:t>&e;</w:t></w:r></w:p></w:body></w:document>"""));

        var text = DocumentText.Extract(doc, "evil.docx", 10_000);

        text.Ok.ShouldBeFalse();
        text.Problem!.ShouldContain("could not be read");
    }

    [Fact]
    public void A_part_that_unpacks_beyond_the_limit_is_stopped()
    {
        var huge = new string('a', (int)DocumentText.MaxPartBytes + 1024);
        var doc = Docx($"<w:p><w:r><w:t>{huge}</w:t></w:r></w:p>");

        DocumentText.Extract(doc, "bomb.docx", 100).Ok.ShouldBeFalse();
    }

    [Fact]
    public void Text_is_cut_at_the_limit_and_odd_bytes_are_no_error()
    {
        var cut = DocumentText.Extract(Encoding.UTF8.GetBytes(new string('x', 500)), "a.csv", 100);
        cut.Truncated.ShouldBeTrue();
        cut.Text.Length.ShouldBe(100);

        DocumentText.Extract([0x50, 0x72, 0x65, 0x69, 0x73, 0x20, 0xE4], "latin.txt", 100).Text.ShouldBe("Preis ä");
    }

    [Fact]
    public void Broken_and_unknown_files_are_named_not_guessed()
    {
        DocumentText.Extract([1, 2, 3], "invoice.pdf", 100).Problem!.ShouldContain("PDF");
        DocumentText.Extract([1, 2, 3], "image.png", 100).Ok.ShouldBeFalse();
        DocumentText.Extract([1, 2, 3], "broken.docx", 100).Ok.ShouldBeFalse();
    }

    private static byte[] Pdf(params string?[] pages)
    {
        var builder = new UglyToad.PdfPig.Writer.PdfDocumentBuilder();
        var font = builder.AddStandard14Font(UglyToad.PdfPig.Fonts.Standard14Fonts.Standard14Font.Helvetica);
        foreach (var text in pages)
        {
            var page = builder.AddPage(UglyToad.PdfPig.Content.PageSize.A4);
            if (text is not null)
                page.AddText(text, 12, new UglyToad.PdfPig.Core.PdfPoint(40, 700), font);
        }
        return builder.Build();
    }

    [Fact]
    public void A_pdf_comes_out_page_by_page()
    {
        var text = DocumentText.Extract(Pdf("Invoice 4711", "Total 150.00 EUR"), "invoice.pdf", 10_000);

        text.Ok.ShouldBeTrue(text.Problem);
        text.Kind.ShouldBe("PDF");
        text.Text.ShouldContain("## Page 1");
        text.Text.ShouldContain("Invoice 4711");
        text.Text.ShouldContain("## Page 2");
        text.Text.ShouldContain("Total 150.00 EUR");
    }

    [Fact]
    public void A_pdf_without_text_is_named_as_probably_scanned()
        => DocumentText.Extract(Pdf((string?)null), "scan.pdf", 10_000).Problem!.ShouldContain("no text layer");
}