Testfundament aufbauen und Bestandsaufnahme dokumentieren
IChatCompletionClient aus OpenRouterClient extrahiert, damit AgentEngine und ContextCompactor ohne echte API-Aufrufe testbar sind. Neues Testprojekt tests/ClawdDotNet.Core.Tests (xUnit, Shouldly, NSubstitute, FsCheck) mit: - FakeChatClient (programmierbare Antwortfolgen, Deep-Copy der Requests) - ContextInvariants (prueft die API-Regeln fuer tool_call-Paarung) - Conversation-Builder fuer gueltige Testkonversationen - 26 Tests: Compaction, LoopGuard, 2 Property-Tests 10 Tests sind bewusst rot — sie reproduzieren die Bugs B1, B3 und B14 aus der Bestandsaufnahme und werden mit den Fixes gruen. Ausserdem: fehlende Tool-Projekte in slnx ergaenzt, Test-Pakete im packageSourceMapping der NuGet.Config eingetragen. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Richard
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
92e50d3ac4
commit
667cecce25
@@ -0,0 +1,25 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<RootNamespace>ClawdDotNet.Core.Tests</RootNamespace>
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
||||
<PackageReference Include="Shouldly" Version="4.2.1" />
|
||||
<PackageReference Include="NSubstitute" Version="5.3.0" />
|
||||
<PackageReference Include="FsCheck.Xunit" Version="2.16.6" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\ClawdDotNet.Core\ClawdDotNet.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,150 @@
|
||||
using ClawdDotNet.Core.Api.Models;
|
||||
using ClawdDotNet.Core.Config;
|
||||
using ClawdDotNet.Core.Engine;
|
||||
using ClawdDotNet.Core.Tests.Infrastructure;
|
||||
using FsCheck;
|
||||
using FsCheck.Xunit;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Engine;
|
||||
|
||||
/// <summary>
|
||||
/// Property-Based Tests: Statt einzelner Beispiele werden hunderte zufällige,
|
||||
/// aber gültige Konversationen erzeugt und geprüft, dass die Compaction die
|
||||
/// API-Invarianten in JEDEM Fall erhält.
|
||||
///
|
||||
/// Genau diese Grenzfälle — wo die Tail-Grenze zufällig mitten in einen
|
||||
/// Tool-Zyklus fällt — schreibt von Hand niemand auf. Bug B1 fällt in diese Klasse.
|
||||
/// </summary>
|
||||
public sealed class ContextCompactorPropertyTests
|
||||
{
|
||||
private const string Model = "test/model";
|
||||
|
||||
private static readonly LoopGuardConfig AlwaysCompact = new()
|
||||
{
|
||||
MaxContextTokens = 1_000,
|
||||
CompactionThreshold = 0.5
|
||||
};
|
||||
|
||||
[Property(MaxTest = 300, Arbitrary = [typeof(ConversationArbitrary)])]
|
||||
public Property Compaction_erhaelt_immer_eine_gueltige_Nachrichtenfolge(ConversationShape shape)
|
||||
{
|
||||
var messages = shape.ToMessages();
|
||||
|
||||
// Vorbedingung: Der Generator erzeugt nur gültige Ausgangsfolgen.
|
||||
if (!ContextInvariants.IsValid(messages))
|
||||
return false.ToProperty().Label("Generator hat eine ungültige Folge erzeugt");
|
||||
|
||||
var client = new FakeChatClient().AlwaysRespondsWithText("- Zusammenfassung des Verlaufs.");
|
||||
var compactor = new ContextCompactor(client, TestLogging.Factory);
|
||||
|
||||
compactor.CompactIfNeededAsync(messages, 50_000, AlwaysCompact, Model, default)
|
||||
.GetAwaiter().GetResult();
|
||||
|
||||
var valid = ContextInvariants.IsValid(messages);
|
||||
|
||||
return valid.ToProperty()
|
||||
.Label($"Nach Compaction ungültig. Ausgangsform: {shape}\n{ContextInvariants.Describe(messages)}");
|
||||
}
|
||||
|
||||
[Property(MaxTest = 300, Arbitrary = [typeof(ConversationArbitrary)])]
|
||||
public Property Compaction_behaelt_den_SystemPrompt_genau_einmal(ConversationShape shape)
|
||||
{
|
||||
var messages = shape.ToMessages();
|
||||
var hadSystem = messages.Any(m => m.Role == "system");
|
||||
|
||||
var client = new FakeChatClient().AlwaysRespondsWithText("- Zusammenfassung.");
|
||||
var compactor = new ContextCompactor(client, TestLogging.Factory);
|
||||
|
||||
compactor.CompactIfNeededAsync(messages, 50_000, AlwaysCompact, Model, default)
|
||||
.GetAwaiter().GetResult();
|
||||
|
||||
var systemCount = messages.Count(m => m.Role == "system");
|
||||
var expected = hadSystem ? 1 : 0;
|
||||
|
||||
return (systemCount == expected).ToProperty()
|
||||
.Label($"system-Nachrichten: erwartet {expected}, gefunden {systemCount}");
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
// Generator
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
|
||||
/// <summary>
|
||||
/// Beschreibt die Form einer Konversation. FsCheck erzeugt davon zufällige
|
||||
/// Varianten und schrumpft sie im Fehlerfall auf das minimale Gegenbeispiel.
|
||||
/// </summary>
|
||||
public sealed record ConversationShape(bool HasSystemPrompt, IReadOnlyList<TurnShape> Turns)
|
||||
{
|
||||
public List<ChatMessage> ToMessages()
|
||||
{
|
||||
var c = Conversation.Start(HasSystemPrompt ? "Du bist ein Testagent." : null);
|
||||
|
||||
foreach (var turn in Turns)
|
||||
{
|
||||
c.User($"Anfrage mit {turn.ToolCalls} Tool-Aufrufen");
|
||||
|
||||
if (turn.ToolCalls > 0)
|
||||
c.ToolCycle(turn.ToolCalls, turn.ResultLength);
|
||||
|
||||
if (turn.EndsWithText)
|
||||
c.Assistant("Abschließende Antwort des Agenten.");
|
||||
}
|
||||
|
||||
return c.Build();
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
=> $"system={HasSystemPrompt}, turns=[{string.Join(", ", Turns)}]";
|
||||
}
|
||||
|
||||
public sealed record TurnShape(int ToolCalls, int ResultLength, bool EndsWithText)
|
||||
{
|
||||
public override string ToString() => $"{ToolCalls}tc/{ResultLength}b/{(EndsWithText ? "text" : "offen")}";
|
||||
}
|
||||
|
||||
public static class ConversationArbitrary
|
||||
{
|
||||
public static Arbitrary<ConversationShape> Conversations()
|
||||
{
|
||||
var turnGen =
|
||||
from toolCalls in Gen.Choose(0, 4)
|
||||
from resultLength in Gen.Elements(20, 200, 2_500, 9_000)
|
||||
from endsWithText in Gen.Elements(true, false)
|
||||
select new TurnShape(toolCalls, resultLength, endsWithText);
|
||||
|
||||
var gen =
|
||||
from hasSystem in Gen.Elements(true, false)
|
||||
from turnCount in Gen.Choose(1, 12)
|
||||
from turns in Gen.ListOf(turnCount, turnGen)
|
||||
select new ConversationShape(hasSystem, turns.ToList());
|
||||
|
||||
return Arb.From(gen, Shrink);
|
||||
}
|
||||
|
||||
/// <summary>Im Fehlerfall auf das kleinste Gegenbeispiel reduzieren.</summary>
|
||||
private static IEnumerable<ConversationShape> Shrink(ConversationShape shape)
|
||||
{
|
||||
// Weniger Turns
|
||||
for (var i = 0; i < shape.Turns.Count; i++)
|
||||
{
|
||||
var reduced = shape.Turns.Where((_, idx) => idx != i).ToList();
|
||||
if (reduced.Count > 0)
|
||||
yield return shape with { Turns = reduced };
|
||||
}
|
||||
|
||||
// Weniger Tool-Calls pro Turn
|
||||
for (var i = 0; i < shape.Turns.Count; i++)
|
||||
{
|
||||
var turn = shape.Turns[i];
|
||||
if (turn.ToolCalls <= 0) continue;
|
||||
|
||||
var reduced = shape.Turns.ToList();
|
||||
reduced[i] = turn with { ToolCalls = turn.ToolCalls - 1 };
|
||||
yield return shape with { Turns = reduced };
|
||||
}
|
||||
|
||||
if (shape.HasSystemPrompt)
|
||||
yield return shape with { HasSystemPrompt = false };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
using ClawdDotNet.Core.Api.Models;
|
||||
using ClawdDotNet.Core.Config;
|
||||
using ClawdDotNet.Core.Engine;
|
||||
using ClawdDotNet.Core.Tests.Infrastructure;
|
||||
using Shouldly;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Engine;
|
||||
|
||||
/// <summary>
|
||||
/// Regressionstests für Bug B1: Die Compaction behält blind die letzten N Nachrichten.
|
||||
/// Liegt diese Grenze mitten in einer Tool-Sequenz, entsteht eine tool-Antwort ohne
|
||||
/// zugehörigen assistant-tool_call — die API lehnt das mit HTTP 400 ab.
|
||||
/// </summary>
|
||||
public sealed class ContextCompactorTests
|
||||
{
|
||||
private const string Model = "test/model";
|
||||
|
||||
/// <summary>Schwelle so gesetzt, dass Compaction sicher ausgelöst wird.</summary>
|
||||
private static LoopGuardConfig AlwaysCompact => new()
|
||||
{
|
||||
MaxContextTokens = 1_000,
|
||||
CompactionThreshold = 0.5 // Schwelle = 500 Tokens
|
||||
};
|
||||
|
||||
private static LoopGuardConfig NeverCompact => new()
|
||||
{
|
||||
MaxContextTokens = 10_000_000,
|
||||
CompactionThreshold = 0.9
|
||||
};
|
||||
|
||||
private static (ContextCompactor Compactor, FakeChatClient Client) CreateCompactor(
|
||||
string summary = "- Der Agent hat Daten geprüft.\n- Ergebnis war unauffällig.")
|
||||
{
|
||||
var client = new FakeChatClient().AlwaysRespondsWithText(summary);
|
||||
return (new ContextCompactor(client, TestLogging.Factory), client);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// C1/C2 — Die eigentliche Bug-Reproduktion
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
[Theory]
|
||||
[InlineData(1)]
|
||||
[InlineData(2)]
|
||||
[InlineData(3)]
|
||||
[InlineData(4)]
|
||||
[InlineData(5)]
|
||||
[InlineData(6)]
|
||||
[InlineData(7)]
|
||||
[InlineData(8)]
|
||||
public async Task Compaction_erhaelt_gueltige_Sequenz_bei_beliebiger_Toolzyklus_Laenge(int toolCallsPerCycle)
|
||||
{
|
||||
// Je nach Anzahl der Tool-Calls fällt die "letzte 6 Nachrichten"-Grenze
|
||||
// an eine andere Stelle im Zyklus. Mindestens eine Variante trifft mitten hinein.
|
||||
var (compactor, _) = CreateCompactor();
|
||||
var messages = Conversation.Start()
|
||||
.Repeat(times: 4, toolCallsPerCycle: toolCallsPerCycle, resultLength: 200)
|
||||
.Build();
|
||||
|
||||
ContextInvariants.AssertValid(messages); // Vorbedingung: Ausgangslage ist gültig
|
||||
|
||||
await compactor.CompactIfNeededAsync(messages, 5_000, AlwaysCompact, Model, default);
|
||||
|
||||
ContextInvariants.AssertValid(messages);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Compaction_trennt_Toolantwort_nie_von_ihrem_Aufruf()
|
||||
{
|
||||
// Gezielt konstruiert: Der Tail von 6 Nachrichten beginnt genau bei einer tool-Antwort.
|
||||
var (compactor, _) = CreateCompactor();
|
||||
var messages = Conversation.Start()
|
||||
.User("Erste Anfrage")
|
||||
.ToolCycle(count: 3, resultLength: 300)
|
||||
.Assistant("Zwischenergebnis")
|
||||
.User("Zweite Anfrage")
|
||||
.ToolCycle(count: 3, resultLength: 300)
|
||||
.Build();
|
||||
|
||||
await compactor.CompactIfNeededAsync(messages, 5_000, AlwaysCompact, Model, default);
|
||||
|
||||
ContextInvariants.AssertValid(messages);
|
||||
|
||||
// Keine verwaiste tool-Antwort
|
||||
for (var i = 0; i < messages.Count; i++)
|
||||
{
|
||||
if (messages[i].Role != "tool") continue;
|
||||
|
||||
var hasParent = messages
|
||||
.Take(i)
|
||||
.Any(m => m.ToolCalls?.Any(tc => tc.Id == messages[i].ToolCallId) == true);
|
||||
|
||||
hasParent.ShouldBeTrue(
|
||||
$"tool-Antwort an Position {i} (id={messages[i].ToolCallId}) hat keinen Aufruf mehr.\n" +
|
||||
ContextInvariants.Describe(messages));
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// C3/C4 — System-Prompt
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
[Fact]
|
||||
public async Task Compaction_behaelt_den_SystemPrompt_an_Position_0()
|
||||
{
|
||||
const string systemPrompt = "Du bist ein sehr spezifischer Testagent.";
|
||||
var (compactor, _) = CreateCompactor();
|
||||
var messages = Conversation.Start(systemPrompt).Repeat(6).Build();
|
||||
|
||||
await compactor.CompactIfNeededAsync(messages, 5_000, AlwaysCompact, Model, default);
|
||||
|
||||
messages[0].Role.ShouldBe("system");
|
||||
messages[0].Content.ShouldBe(systemPrompt);
|
||||
messages.Count(m => m.Role == "system").ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Compaction_erfindet_keinen_SystemPrompt_wenn_keiner_da_war()
|
||||
{
|
||||
var (compactor, _) = CreateCompactor();
|
||||
var messages = Conversation.Start(systemPrompt: null).Repeat(6).Build();
|
||||
|
||||
await compactor.CompactIfNeededAsync(messages, 5_000, AlwaysCompact, Model, default);
|
||||
|
||||
messages.ShouldNotContain(m => m.Role == "system");
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// C5/C6 — Fehlerfälle der Zusammenfassung
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
[Fact]
|
||||
public async Task Compaction_laesst_Nachrichten_unveraendert_wenn_die_Zusammenfassung_fehlschlaegt()
|
||||
{
|
||||
var client = new FakeChatClient().Throws(new HttpRequestException("API nicht erreichbar"));
|
||||
var compactor = new ContextCompactor(client, TestLogging.Factory);
|
||||
var messages = Conversation.Start().Repeat(6).Build();
|
||||
var before = messages.Count;
|
||||
|
||||
await compactor.CompactIfNeededAsync(messages, 5_000, AlwaysCompact, Model, default);
|
||||
|
||||
ContextInvariants.AssertValid(messages);
|
||||
messages.Count.ShouldBe(before, "bei fehlgeschlagener Zusammenfassung darf nichts verloren gehen");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Compaction_laesst_Nachrichten_unveraendert_wenn_die_Zusammenfassung_leer_ist()
|
||||
{
|
||||
var client = new FakeChatClient().AlwaysRespondsWithText(" ");
|
||||
var compactor = new ContextCompactor(client, TestLogging.Factory);
|
||||
var messages = Conversation.Start().Repeat(6).Build();
|
||||
var before = messages.Count;
|
||||
|
||||
await compactor.CompactIfNeededAsync(messages, 5_000, AlwaysCompact, Model, default);
|
||||
|
||||
ContextInvariants.AssertValid(messages);
|
||||
messages.Count.ShouldBe(before);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// C7/C8 — Tool-Result-Pruning
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
[Fact]
|
||||
public async Task Pruning_kuerzt_lange_Toolergebnisse_ausserhalb_des_geschuetzten_Endes()
|
||||
{
|
||||
var (compactor, _) = CreateCompactor();
|
||||
var messages = Conversation.Start()
|
||||
.Repeat(times: 5, toolCallsPerCycle: 1, resultLength: 8_000)
|
||||
.Build();
|
||||
|
||||
await compactor.CompactIfNeededAsync(messages, 5_000, AlwaysCompact, Model, default);
|
||||
|
||||
ContextInvariants.AssertValid(messages);
|
||||
messages.Where(m => m.Role == "tool")
|
||||
.ShouldAllBe(m => m.Content!.Length <= 8_000,
|
||||
"kein Tool-Ergebnis darf nach dem Pruning gewachsen sein");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pruning_laesst_kurze_Toolergebnisse_unveraendert()
|
||||
{
|
||||
var (compactor, _) = CreateCompactor();
|
||||
var messages = Conversation.Start()
|
||||
.Repeat(times: 5, toolCallsPerCycle: 1, resultLength: 100)
|
||||
.Build();
|
||||
|
||||
await compactor.CompactIfNeededAsync(messages, 5_000, AlwaysCompact, Model, default);
|
||||
|
||||
foreach (var content in messages.Where(m => m.Role == "tool").Select(m => m.Content))
|
||||
(content ?? "").ShouldNotContain("gekürzt", Case.Insensitive);
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// C9/C10 — Schätzung und Schwelle
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
[Fact]
|
||||
public void EstimateTokens_waechst_monoton_mit_dem_Inhalt()
|
||||
{
|
||||
var klein = Conversation.Start().Repeat(1).Build();
|
||||
var mittel = Conversation.Start().Repeat(5).Build();
|
||||
var gross = Conversation.Start().Repeat(20).Build();
|
||||
|
||||
var a = ContextCompactor.EstimateTokens(klein);
|
||||
var b = ContextCompactor.EstimateTokens(mittel);
|
||||
var c = ContextCompactor.EstimateTokens(gross);
|
||||
|
||||
a.ShouldBeLessThan(b);
|
||||
b.ShouldBeLessThan(c);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Unterhalb_der_Schwelle_wird_kein_ApiCall_ausgeloest()
|
||||
{
|
||||
// Wichtig fürs Budget: Compaction darf nicht unnötig ein Modell anwerfen.
|
||||
var (compactor, client) = CreateCompactor();
|
||||
var messages = Conversation.Start().Repeat(2).Build();
|
||||
|
||||
var compacted = await compactor.CompactIfNeededAsync(messages, 10, NeverCompact, Model, default);
|
||||
|
||||
compacted.ShouldBeFalse();
|
||||
client.CallCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Compaction_reduziert_die_Nachrichtenzahl_deutlich()
|
||||
{
|
||||
var (compactor, _) = CreateCompactor();
|
||||
var messages = Conversation.Start().Repeat(15).Build();
|
||||
var before = messages.Count;
|
||||
|
||||
var compacted = await compactor.CompactIfNeededAsync(messages, 50_000, AlwaysCompact, Model, default);
|
||||
|
||||
compacted.ShouldBeTrue();
|
||||
messages.Count.ShouldBeLessThan(before);
|
||||
ContextInvariants.AssertValid(messages);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
using ClawdDotNet.Core.Config;
|
||||
using ClawdDotNet.Core.Engine;
|
||||
using Shouldly;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Engine;
|
||||
|
||||
public sealed class LoopGuardTests
|
||||
{
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// L1 — Schrittzählung
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
[Fact]
|
||||
public void RecordStep_erlaubt_genau_MaxSteps_Schritte()
|
||||
{
|
||||
var guard = new LoopGuard(new LoopGuardConfig { MaxSteps = 5 });
|
||||
|
||||
for (var i = 0; i < 5; i++)
|
||||
guard.RecordStep();
|
||||
|
||||
guard.Steps.ShouldBe(5);
|
||||
Should.Throw<LoopLimitExceededException>(() => guard.RecordStep());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordStep_nennt_das_erreichte_Limit_in_der_Meldung()
|
||||
{
|
||||
var guard = new LoopGuard(new LoopGuardConfig { MaxSteps = 3 });
|
||||
for (var i = 0; i < 3; i++) guard.RecordStep();
|
||||
|
||||
var ex = Should.Throw<LoopLimitExceededException>(() => guard.RecordStep());
|
||||
ex.Message.ShouldContain("3");
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// L2 — Die maxTokens-Semantik (Bug B3)
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
/// <summary>
|
||||
/// Kernproblem B3: RecordTokens summiert die TotalTokens jedes Schritts.
|
||||
/// Da jeder Schritt den kompletten Kontext erneut sendet, wächst diese Summe
|
||||
/// quadratisch — ein völlig normaler Chat bricht dadurch nach wenigen Schritten ab.
|
||||
///
|
||||
/// Dieser Test beschreibt das GEWÜNSCHTE Verhalten: Ein Agent mit einem stabilen
|
||||
/// 20k-Kontext muss 10 Schritte durchhalten können.
|
||||
///
|
||||
/// Er schlägt mit den aktuellen Defaults fehl und wird grün, sobald
|
||||
/// maxCumulativeTokens (Kostenbudget) und maxContextTokens (Kontextgröße)
|
||||
/// getrennt sind.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Ein_stabiler_Kontext_ueberlebt_zehn_Schritte()
|
||||
{
|
||||
var config = new LoopGuardConfig(); // bewusst die Produktiv-Defaults
|
||||
var guard = new LoopGuard(config);
|
||||
|
||||
Should.NotThrow(() =>
|
||||
{
|
||||
for (var step = 0; step < 10; step++)
|
||||
{
|
||||
guard.RecordStep();
|
||||
// Realistisch: 20k Kontext geht rein, ~500 Tokens kommen raus.
|
||||
guard.RecordTokens(20_000 + 500);
|
||||
}
|
||||
}, $"Ein Kontext von 20k über 10 Schritte ist normal und darf nicht am " +
|
||||
$"Limit maxTokens={config.MaxTokens} scheitern.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordTokens_greift_wenn_das_Kostenbudget_wirklich_erschoepft_ist()
|
||||
{
|
||||
var guard = new LoopGuard(new LoopGuardConfig { MaxTokens = 1_000 });
|
||||
|
||||
Should.Throw<LoopLimitExceededException>(() => guard.RecordTokens(1_001));
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
// L3 — Thread-Sicherheit
|
||||
// ═══════════════════════════════════════════════════════════
|
||||
|
||||
[Fact]
|
||||
public void RecordStep_zaehlt_unter_Parallelzugriff_korrekt()
|
||||
{
|
||||
var guard = new LoopGuard(new LoopGuardConfig { MaxSteps = int.MaxValue });
|
||||
|
||||
Parallel.For(0, 1_000, _ => guard.RecordStep());
|
||||
|
||||
guard.Steps.ShouldBe(1_000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordTokens_summiert_unter_Parallelzugriff_korrekt()
|
||||
{
|
||||
var guard = new LoopGuard(new LoopGuardConfig { MaxTokens = int.MaxValue });
|
||||
|
||||
Parallel.For(0, 1_000, _ => guard.RecordTokens(10));
|
||||
|
||||
guard.Tokens.ShouldBe(10_000);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
global using Xunit;
|
||||
@@ -0,0 +1,126 @@
|
||||
using System.Text;
|
||||
using ClawdDotNet.Core.Api.Models;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Prüft, ob eine Nachrichtenfolge von der Chat-Completions-API akzeptiert würde.
|
||||
///
|
||||
/// Die Regeln entsprechen dem, was OpenRouter/Anthropic/OpenAI verlangen:
|
||||
/// Eine tool-Antwort ist nur gültig, wenn ihr eine assistant-Nachricht mit einem
|
||||
/// passenden tool_call vorausgeht — und jeder tool_call braucht seine Antwort.
|
||||
///
|
||||
/// Wird eine Regel verletzt, antwortet die API mit HTTP 400 und der laufende
|
||||
/// Agent bricht ab. Genau das ist Bug B1 aus der Bestandsaufnahme.
|
||||
/// </summary>
|
||||
internal static class ContextInvariants
|
||||
{
|
||||
public static void AssertValid(IReadOnlyList<ChatMessage> messages)
|
||||
{
|
||||
var violations = Validate(messages).ToList();
|
||||
if (violations.Count == 0)
|
||||
return;
|
||||
|
||||
var sb = new StringBuilder();
|
||||
sb.AppendLine($"Ungültige Nachrichtenfolge ({violations.Count} Verstoß/Verstöße):");
|
||||
foreach (var v in violations)
|
||||
sb.AppendLine($" • {v}");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine("Sequenz:");
|
||||
sb.Append(Describe(messages));
|
||||
|
||||
throw new ContextInvariantViolationException(sb.ToString());
|
||||
}
|
||||
|
||||
public static bool IsValid(IReadOnlyList<ChatMessage> messages)
|
||||
=> !Validate(messages).Any();
|
||||
|
||||
private static IEnumerable<string> Validate(IReadOnlyList<ChatMessage> messages)
|
||||
{
|
||||
// ── Regel 1: höchstens eine system-Nachricht, und zwar ganz vorne ──
|
||||
for (var i = 1; i < messages.Count; i++)
|
||||
{
|
||||
if (messages[i].Role == "system")
|
||||
yield return $"[{i}] system-Nachricht steht nicht an Position 0";
|
||||
}
|
||||
|
||||
// ── Regel 2: jede tool-Nachricht braucht eine ToolCallId ──
|
||||
for (var i = 0; i < messages.Count; i++)
|
||||
{
|
||||
if (messages[i].Role == "tool" && string.IsNullOrEmpty(messages[i].ToolCallId))
|
||||
yield return $"[{i}] tool-Nachricht ohne tool_call_id";
|
||||
}
|
||||
|
||||
// ── Regel 3: jede tool-Nachricht gehört zum unmittelbar vorausgehenden
|
||||
// assistant-Block mit passender tool_call-Id ──
|
||||
var openCalls = new HashSet<string>();
|
||||
var answered = new HashSet<string>();
|
||||
|
||||
for (var i = 0; i < messages.Count; i++)
|
||||
{
|
||||
var msg = messages[i];
|
||||
|
||||
if (msg.Role == "assistant" && msg.ToolCalls is { Count: > 0 })
|
||||
{
|
||||
// Vorheriger Block muss vollständig beantwortet sein
|
||||
var unanswered = openCalls.Except(answered).ToList();
|
||||
if (unanswered.Count > 0)
|
||||
yield return $"[{i}] neuer assistant-Block, aber tool_call(s) noch unbeantwortet: {string.Join(", ", unanswered)}";
|
||||
|
||||
openCalls.Clear();
|
||||
answered.Clear();
|
||||
foreach (var tc in msg.ToolCalls)
|
||||
openCalls.Add(tc.Id);
|
||||
}
|
||||
else if (msg.Role == "tool")
|
||||
{
|
||||
var id = msg.ToolCallId ?? "";
|
||||
if (!openCalls.Contains(id))
|
||||
{
|
||||
yield return $"[{i}] tool-Antwort '{id}' ohne vorausgehenden assistant-tool_call " +
|
||||
"(die API lehnt das mit HTTP 400 ab)";
|
||||
}
|
||||
else if (!answered.Add(id))
|
||||
{
|
||||
yield return $"[{i}] tool_call '{id}' wurde doppelt beantwortet";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// user/system/assistant-ohne-tool_calls beenden den Block
|
||||
var unanswered = openCalls.Except(answered).ToList();
|
||||
if (unanswered.Count > 0)
|
||||
yield return $"[{i}] {msg.Role}-Nachricht, aber tool_call(s) unbeantwortet: {string.Join(", ", unanswered)}";
|
||||
|
||||
openCalls.Clear();
|
||||
answered.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
// ── Regel 4: am Ende darf kein tool_call offen sein ──
|
||||
var stillOpen = openCalls.Except(answered).ToList();
|
||||
if (stillOpen.Count > 0)
|
||||
yield return $"[Ende] unbeantwortete tool_call(s): {string.Join(", ", stillOpen)}";
|
||||
}
|
||||
|
||||
/// <summary>Kompakte, lesbare Darstellung für Fehlermeldungen.</summary>
|
||||
public static string Describe(IReadOnlyList<ChatMessage> messages)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
for (var i = 0; i < messages.Count; i++)
|
||||
{
|
||||
var m = messages[i];
|
||||
var detail = m.Role switch
|
||||
{
|
||||
"tool" => $"tool_call_id={m.ToolCallId}, len={m.Content?.Length ?? 0}",
|
||||
"assistant" when m.ToolCalls is { Count: > 0 }
|
||||
=> $"tool_calls=[{string.Join(", ", m.ToolCalls.Select(t => $"{t.Function.Name}#{t.Id}"))}]",
|
||||
_ => $"len={m.Content?.Length ?? 0}"
|
||||
};
|
||||
sb.AppendLine($" [{i,2}] {m.Role,-9} {detail}");
|
||||
}
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class ContextInvariantViolationException(string message) : Exception(message);
|
||||
@@ -0,0 +1,68 @@
|
||||
using ClawdDotNet.Core.Api.Models;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Baut gültige Nachrichtenfolgen für Tests — lesbar und garantiert API-konform.
|
||||
/// </summary>
|
||||
internal sealed class Conversation
|
||||
{
|
||||
private readonly List<ChatMessage> _messages = new();
|
||||
private int _callCounter;
|
||||
|
||||
public static Conversation Start(string? systemPrompt = "Du bist ein Testagent.")
|
||||
{
|
||||
var c = new Conversation();
|
||||
if (!string.IsNullOrEmpty(systemPrompt))
|
||||
c._messages.Add(ChatMessage.System(systemPrompt));
|
||||
return c;
|
||||
}
|
||||
|
||||
public Conversation User(string text)
|
||||
{
|
||||
_messages.Add(ChatMessage.User(text));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Conversation Assistant(string text)
|
||||
{
|
||||
_messages.Add(ChatMessage.Assistant(text));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fügt einen vollständigen Tool-Zyklus hinzu: assistant mit N tool_calls,
|
||||
/// gefolgt von genau N passenden tool-Antworten.
|
||||
/// </summary>
|
||||
public Conversation ToolCycle(int count = 1, int resultLength = 50, string toolName = "TestTool")
|
||||
{
|
||||
var calls = new List<ToolCall>();
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
calls.Add(new ToolCall
|
||||
{
|
||||
Id = $"call_{++_callCounter}",
|
||||
Type = "function",
|
||||
Function = new ToolCallFunction { Name = toolName, Arguments = """{"action":"test"}""" }
|
||||
});
|
||||
}
|
||||
|
||||
_messages.Add(ChatMessage.AssistantWithToolCalls(calls));
|
||||
foreach (var call in calls)
|
||||
_messages.Add(ChatMessage.ToolResponse(call.Id, new string('x', resultLength)));
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>Wiederholt ein Muster aus User-Nachricht und Tool-Zyklus.</summary>
|
||||
public Conversation Repeat(int times, int toolCallsPerCycle = 1, int resultLength = 50)
|
||||
{
|
||||
for (var i = 0; i < times; i++)
|
||||
User($"Anfrage {i}").ToolCycle(toolCallsPerCycle, resultLength).Assistant($"Antwort {i}");
|
||||
return this;
|
||||
}
|
||||
|
||||
public List<ChatMessage> Build() => _messages;
|
||||
|
||||
public static implicit operator List<ChatMessage>(Conversation c) => c._messages;
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
using System.Text.Json;
|
||||
using ClawdDotNet.Core.Api;
|
||||
using ClawdDotNet.Core.Api.Models;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Infrastructure;
|
||||
|
||||
/// <summary>
|
||||
/// Ersetzt den OpenRouterClient in Tests. Liefert eine vorprogrammierte Antwortfolge
|
||||
/// und schreibt jeden empfangenen Request mit.
|
||||
///
|
||||
/// Wichtig: Requests werden tief kopiert. Die Engine reicht dieselbe List<ChatMessage>
|
||||
/// weiter und verändert sie danach — ohne Kopie würden Tests den Endzustand prüfen
|
||||
/// statt dessen, was tatsächlich gesendet wurde.
|
||||
/// </summary>
|
||||
internal sealed class FakeChatClient : IChatCompletionClient
|
||||
{
|
||||
private readonly Queue<Func<ChatRequest, ChatResponse>> _responses = new();
|
||||
private Func<ChatRequest, ChatResponse>? _fallback;
|
||||
|
||||
/// <summary>Alle empfangenen Requests, als tiefe Kopien.</summary>
|
||||
public List<ChatRequest> ReceivedRequests { get; } = new();
|
||||
|
||||
public int CallCount => ReceivedRequests.Count;
|
||||
|
||||
// ─── Programmierung der Antworten ───
|
||||
|
||||
public FakeChatClient RespondsWithText(string text, Usage? usage = null)
|
||||
{
|
||||
_responses.Enqueue(_ => TextResponse(text, usage));
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeChatClient RespondsWithToolCall(string toolName, string argumentsJson = "{}", string? id = null)
|
||||
=> RespondsWithToolCalls((toolName, argumentsJson, id));
|
||||
|
||||
public FakeChatClient RespondsWithToolCalls(params (string Tool, string Args, string? Id)[] calls)
|
||||
{
|
||||
var toolCalls = calls.Select((c, i) => new ToolCall
|
||||
{
|
||||
Id = c.Id ?? $"call_{Guid.NewGuid():N}"[..12],
|
||||
Type = "function",
|
||||
Function = new ToolCallFunction { Name =c.Tool, Arguments = c.Args }
|
||||
}).ToList();
|
||||
|
||||
_responses.Enqueue(_ => new ChatResponse
|
||||
{
|
||||
Id = "resp_" + ReceivedRequests.Count,
|
||||
Choices = [new Choice { Index = 0, Message = new ChatMessage { Role = "assistant", ToolCalls = toolCalls } }],
|
||||
Usage = new Usage { PromptTokens = 100, CompletionTokens = 20, TotalTokens = 120 }
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>Antwortet mit einer bestimmten Prompt-Token-Zahl — für Compaction-Schwellen.</summary>
|
||||
public FakeChatClient RespondsWithTokens(int promptTokens, string text = "fertig")
|
||||
{
|
||||
_responses.Enqueue(_ => TextResponse(text, new Usage
|
||||
{
|
||||
PromptTokens = promptTokens,
|
||||
CompletionTokens = 10,
|
||||
TotalTokens = promptTokens + 10
|
||||
}));
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeChatClient Throws(Exception ex)
|
||||
{
|
||||
_responses.Enqueue(_ => throw ex);
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>Antwort für alle Aufrufe, die über die programmierte Folge hinausgehen.</summary>
|
||||
public FakeChatClient AlwaysRespondsWithText(string text)
|
||||
{
|
||||
_fallback = _ => TextResponse(text);
|
||||
return this;
|
||||
}
|
||||
|
||||
// ─── IChatCompletionClient ───
|
||||
|
||||
public Task<ChatResponse> CompleteAsync(ChatRequest request, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
ReceivedRequests.Add(DeepClone(request));
|
||||
|
||||
if (_responses.Count > 0)
|
||||
return Task.FromResult(_responses.Dequeue()(request));
|
||||
|
||||
if (_fallback is not null)
|
||||
return Task.FromResult(_fallback(request));
|
||||
|
||||
throw new InvalidOperationException(
|
||||
$"FakeChatClient: unerwarteter Aufruf Nr. {ReceivedRequests.Count} — " +
|
||||
"keine Antwort mehr programmiert.");
|
||||
}
|
||||
|
||||
// ─── Helfer ───
|
||||
|
||||
private static ChatResponse TextResponse(string text, Usage? usage = null) => new()
|
||||
{
|
||||
Id = "resp",
|
||||
Choices = [new Choice { Index = 0, Message = new ChatMessage { Role = "assistant", Content = text }, FinishReason = "stop" }],
|
||||
Usage = usage ?? new Usage { PromptTokens = 100, CompletionTokens = 20, TotalTokens = 120 }
|
||||
};
|
||||
|
||||
private static ChatRequest DeepClone(ChatRequest request) => new()
|
||||
{
|
||||
Model = request.Model,
|
||||
Stream = request.Stream,
|
||||
Temperature = request.Temperature,
|
||||
MaxTokens = request.MaxTokens,
|
||||
ToolChoice = request.ToolChoice,
|
||||
Tools = request.Tools?.ToList(),
|
||||
Messages = request.Messages.Select(CloneMessage).ToList()
|
||||
};
|
||||
|
||||
private static ChatMessage CloneMessage(ChatMessage m) => new()
|
||||
{
|
||||
Role = m.Role,
|
||||
Content = m.Content,
|
||||
ToolCallId = m.ToolCallId,
|
||||
ToolCalls = m.ToolCalls?.Select(tc => new ToolCall
|
||||
{
|
||||
Id = tc.Id,
|
||||
Type = tc.Type,
|
||||
Function = new ToolCallFunction { Name =tc.Function.Name, Arguments = tc.Function.Arguments }
|
||||
}).ToList()
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Infrastructure;
|
||||
|
||||
internal static class TestLogging
|
||||
{
|
||||
public static ILoggerFactory Factory { get; } = NullLoggerFactory.Instance;
|
||||
}
|
||||
Reference in New Issue
Block a user