Stufe 4c (#764): Bezeichner auf Englisch - Typen, Methoden, Felder, Parameter, Locals, Testnamen
Mit Roslyn umbenannt (Verweise folgen, Zeichenketten bleiben unberuehrt). Nicht umbenannt, weil gespeichert oder ueber die API gebunden (Stufe 4e): Properties, Record-Parameter, Enum-Member sowie die Parameter der Endpunkt-Handler (Routen-/Query-Bindung ueber den Namen). Die JSON-Felder anonymer Objekte bleiben ausdruecklich benannt (JsonFilter: pfad). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
837a634a22
commit
77062facc9
@@ -0,0 +1,403 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using ClawdDotNet.Core.Api.Models;
|
||||
using ClawdDotNet.Core.Engine;
|
||||
using ClawdDotNet.Core.Tests.Infrastructure;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Engine;
|
||||
|
||||
/// <summary>
|
||||
/// The hardening of the harness against silent loss: #547 to #551.
|
||||
///
|
||||
/// What connects these five findings is not the place in the code, but the
|
||||
/// failure mode — none of them produces an error message. The run finishes,
|
||||
/// the result looks plausible, and only the quality is worse. Exactly
|
||||
/// that's why each needs a test: by hand such a thing is not noticed.
|
||||
/// </summary>
|
||||
public class HarnessHardeningTests
|
||||
{
|
||||
// ─────────────────────────── #547 ───────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task The_text_accompanying_the_tool_call_stays_in_the_history()
|
||||
{
|
||||
var fixture = new EngineFixture().WithTool(FakeTool.Returning("Ergebnis"));
|
||||
var agent = fixture.AddAgent("a1", "TestTool");
|
||||
|
||||
fixture.Client
|
||||
.RespondsWithToolCallAndContent("TestTool", "Ich sehe zuerst in der Konfiguration nach.")
|
||||
.RespondsWithText("Fertig.");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
// The second request carries the history the first one produced.
|
||||
var history = fixture.Client.ReceivedRequests[1].Messages;
|
||||
var withCall = history.Single(m => m.Role == "assistant" && m.ToolCalls is { Count: > 0 });
|
||||
|
||||
Assert.Equal("Ich sehe zuerst in der Konfiguration nach.", withCall.Content);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_accompanying_text_the_message_stays_without_content()
|
||||
{
|
||||
// Counter-check: an empty text must not go along as "". Some providers
|
||||
// reject an assistant message with empty content and tool_calls.
|
||||
var fixture = new EngineFixture().WithTool(FakeTool.Returning("Ergebnis"));
|
||||
var agent = fixture.AddAgent("a1", "TestTool");
|
||||
|
||||
fixture.Client
|
||||
.RespondsWithToolCall("TestTool")
|
||||
.RespondsWithText("Fertig.");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
var withCall = fixture.Client.ReceivedRequests[1].Messages
|
||||
.Single(m => m.Role == "assistant" && m.ToolCalls is { Count: > 0 });
|
||||
|
||||
Assert.Null(withCall.Content);
|
||||
}
|
||||
|
||||
// ─────────────────────────── #548 ───────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData("length")]
|
||||
[InlineData("max_tokens")]
|
||||
[InlineData("LENGTH")]
|
||||
public void An_abort_at_the_output_limit_is_recognised(string reason)
|
||||
=> Assert.True(AgentEngine.IsTruncated(reason));
|
||||
|
||||
[Theory]
|
||||
[InlineData("stop")]
|
||||
[InlineData("tool_calls")]
|
||||
[InlineData("end_turn")]
|
||||
[InlineData(null)]
|
||||
public void A_regular_end_does_not_count_as_an_abort(string? reason)
|
||||
=> Assert.False(AgentEngine.IsTruncated(reason));
|
||||
|
||||
[Fact]
|
||||
public async Task A_truncated_answer_is_not_reported_as_done()
|
||||
{
|
||||
var fixture = new EngineFixture();
|
||||
var agent = fixture.AddAgent("a1");
|
||||
|
||||
fixture.Client.RespondsWithTruncatedText("Der Bericht beginnt mit");
|
||||
|
||||
var result = await fixture.Engine.RunAsync(agent, "Bericht", "inst", CancellationToken.None);
|
||||
|
||||
Assert.Equal(AgentRunStatus.Truncated, result.Status);
|
||||
|
||||
// The partial result goes along anyway — throwing it away would be worse than
|
||||
// passing it on marked.
|
||||
Assert.Equal("Der Bericht beginnt mit", result.FinalMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_complete_answer_stays_Completed()
|
||||
{
|
||||
var fixture = new EngineFixture();
|
||||
var agent = fixture.AddAgent("a1");
|
||||
|
||||
fixture.Client.RespondsWithText("Fertiger Bericht.");
|
||||
|
||||
var result = await fixture.Engine.RunAsync(agent, "Bericht", "inst", CancellationToken.None);
|
||||
|
||||
Assert.Equal(AgentRunStatus.Completed, result.Status);
|
||||
}
|
||||
|
||||
// ─────────────────────────── #549 ───────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task The_reasoning_goes_back_unchanged()
|
||||
{
|
||||
const string details = """
|
||||
[{"type":"reasoning.text","text":"Erst die Konfiguration prüfen.","id":"r1","format":"anthropic-claude-v1","index":0,"signature":"sha256:abc"}]
|
||||
""";
|
||||
|
||||
var fixture = new EngineFixture().WithTool(FakeTool.Returning("Ergebnis"));
|
||||
var agent = fixture.AddAgent("a1", "TestTool");
|
||||
|
||||
fixture.Client
|
||||
.RespondsWithToolCallAndContent("TestTool", "Ich prüfe.", reasoningDetailsJson: details)
|
||||
.RespondsWithText("Fertig.");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
var withCall = fixture.Client.ReceivedRequests[1].Messages
|
||||
.Single(m => m.Role == "assistant" && m.ToolCalls is { Count: > 0 });
|
||||
|
||||
Assert.NotNull(withCall.ReasoningDetails);
|
||||
|
||||
// Signature and order must be preserved — by them the provider recognizes
|
||||
// its own block.
|
||||
var block = withCall.ReasoningDetails!.Value[0];
|
||||
Assert.Equal("reasoning.text", block.GetProperty("type").GetString());
|
||||
Assert.Equal("sha256:abc", block.GetProperty("signature").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_reasoning_survives_serialisation_and_the_way_back()
|
||||
{
|
||||
// The history is written to disk and loaded again at startup.
|
||||
// If the reasoning does not survive that, it is gone after a restart.
|
||||
const string details = """
|
||||
[{"type":"reasoning.encrypted","data":"AAAA","id":"r1","index":0}]
|
||||
""";
|
||||
|
||||
var original = ChatMessage.AssistantWithToolCalls(
|
||||
[new ToolCall { Id = "c1", Type = "function", Function = new ToolCallFunction { Name = "T", Arguments = "{}" } }],
|
||||
content: "Text",
|
||||
reasoningDetails: JsonDocument.Parse(details).RootElement.Clone());
|
||||
|
||||
var json = JsonSerializer.Serialize(original);
|
||||
var back = JsonSerializer.Deserialize<ChatMessage>(json)!;
|
||||
|
||||
Assert.Equal("Text", back.Content);
|
||||
Assert.NotNull(back.ReasoningDetails);
|
||||
Assert.Equal("reasoning.encrypted", back.ReasoningDetails!.Value[0].GetProperty("type").GetString());
|
||||
Assert.Equal("AAAA", back.ReasoningDetails!.Value[0].GetProperty("data").GetString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Without_reasoning_there_is_no_empty_field_in_the_request()
|
||||
{
|
||||
var without = ChatMessage.Assistant("Nur Text");
|
||||
var json = JsonSerializer.Serialize(without);
|
||||
|
||||
Assert.DoesNotContain("reasoning", json);
|
||||
}
|
||||
|
||||
// ─────────────────────────── #550 ───────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task The_request_asks_for_a_provider_that_supports_everything()
|
||||
{
|
||||
var fixture = new EngineFixture();
|
||||
var agent = fixture.AddAgent("a1");
|
||||
fixture.Client.RespondsWithText("ok");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
var request = fixture.Client.ReceivedRequests[0];
|
||||
|
||||
Assert.True(request.Provider?.RequireParameters);
|
||||
Assert.True(request.Usage?.Include);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_provider_binding_can_be_switched_off_per_agent()
|
||||
{
|
||||
var fixture = new EngineFixture();
|
||||
var agent = fixture.AddAgent("a1");
|
||||
agent.RequireProviderParameters = false;
|
||||
fixture.Client.RespondsWithText("ok");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
Assert.Null(fixture.Client.ReceivedRequests[0].Provider);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_reasoning_depth_no_reasoning_field_is_sent()
|
||||
{
|
||||
var fixture = new EngineFixture();
|
||||
var agent = fixture.AddAgent("a1"); // the default is "off"
|
||||
fixture.Client.RespondsWithText("ok");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
Assert.Null(fixture.Client.ReceivedRequests[0].Reasoning);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("high", "high", null)]
|
||||
[InlineData("MAX", "max", null)]
|
||||
[InlineData("4096", null, 4096)]
|
||||
public async Task The_reasoning_depth_is_translated_correctly(string setting, string? effort, int? tokens)
|
||||
{
|
||||
var fixture = new EngineFixture();
|
||||
var agent = fixture.AddAgent("a1");
|
||||
agent.ReasoningEffort = setting;
|
||||
fixture.Client.RespondsWithText("ok");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
var reasoning = fixture.Client.ReceivedRequests[0].Reasoning;
|
||||
Assert.NotNull(reasoning);
|
||||
Assert.Equal(effort, reasoning!.Effort);
|
||||
Assert.Equal(tokens, reasoning.MaxTokens);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_reported_amount_is_taken_instead_of_estimated()
|
||||
{
|
||||
var fixture = new EngineFixture();
|
||||
var agent = fixture.AddAgent("a1");
|
||||
|
||||
fixture.Client.RespondsWithText("ok", new Usage
|
||||
{
|
||||
PromptTokens = 100,
|
||||
CompletionTokens = 20,
|
||||
TotalTokens = 120,
|
||||
Cost = 0.0042m
|
||||
});
|
||||
|
||||
var result = await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
Assert.Equal(0.0042m, result.ReportedCostUsd);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_reported_amount_the_field_stays_empty()
|
||||
{
|
||||
// Important: not 0. A run booked as free would be worse than
|
||||
// no information at all, because it silently falsifies the budget calculation.
|
||||
var fixture = new EngineFixture();
|
||||
var agent = fixture.AddAgent("a1");
|
||||
fixture.Client.RespondsWithText("ok");
|
||||
|
||||
var result = await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
Assert.Null(result.ReportedCostUsd);
|
||||
}
|
||||
|
||||
// ─────────────────────────── #551 ───────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Approved_tools_really_run_concurrently()
|
||||
{
|
||||
var tool = FakeTool.SlowParallelSafe(TimeSpan.FromMilliseconds(300));
|
||||
var fixture = new EngineFixture().WithTool(tool);
|
||||
var agent = fixture.AddAgent("a1", "ParallelTool");
|
||||
|
||||
fixture.Client
|
||||
.RespondsWithToolCalls(
|
||||
("ParallelTool", "{}", "c1"),
|
||||
("ParallelTool", "{}", "c2"),
|
||||
("ParallelTool", "{}", "c3"))
|
||||
.RespondsWithText("Fertig.");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
// The reliable proof: calls overlapped.
|
||||
//
|
||||
// Here there was additionally a wall-clock check (below 750 ms, serially it would be 900).
|
||||
// It failed on 2026-09-24 with 927 ms when the test projects ran in parallel —
|
||||
// the time measures the load of the machine, not the behavior of the engine. The
|
||||
// peak of simultaneous calls proves the same without wobbling.
|
||||
Assert.True(tool.MaxConcurrent > 1,
|
||||
$"Erwartet: überlappende Aufrufe. Höchststand war {tool.MaxConcurrent}.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Non_approved_tools_stay_serial()
|
||||
{
|
||||
var tool = FakeTool.SlowSerial(TimeSpan.FromMilliseconds(80));
|
||||
var fixture = new EngineFixture().WithTool(tool);
|
||||
var agent = fixture.AddAgent("a1", "SerialTool");
|
||||
|
||||
fixture.Client
|
||||
.RespondsWithToolCalls(
|
||||
("SerialTool", "{}", "c1"),
|
||||
("SerialTool", "{}", "c2"),
|
||||
("SerialTool", "{}", "c3"))
|
||||
.RespondsWithText("Fertig.");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, tool.MaxConcurrent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_order_of_results_follows_the_calls()
|
||||
{
|
||||
// The slowest call comes first. If the engine sorted by completion,
|
||||
// it would be at the end — and every tool_call_id would be assigned wrongly.
|
||||
var tool = new DelayedMarkerTool();
|
||||
var fixture = new EngineFixture().WithTool(tool);
|
||||
var agent = fixture.AddAgent("a1", "StaffelTool");
|
||||
|
||||
fixture.Client
|
||||
.RespondsWithToolCalls(
|
||||
("StaffelTool", """{"delay":250,"marke":"erste"}""", "c1"),
|
||||
("StaffelTool", """{"delay":10,"marke":"zweite"}""", "c2"),
|
||||
("StaffelTool", """{"delay":120,"marke":"dritte"}""", "c3"))
|
||||
.RespondsWithText("Fertig.");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
var answers = fixture.Client.ReceivedRequests[1].Messages
|
||||
.Where(m => m.Role == "tool")
|
||||
.ToList();
|
||||
|
||||
Assert.Equal(["c1", "c2", "c3"], answers.Select(m => m.ToolCallId));
|
||||
Assert.Equal(
|
||||
["erste", "zweite", "dritte"],
|
||||
answers.Select(m => m.Content?.ToString()?.Split('\n')[1]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_concurrency_limit_is_respected()
|
||||
{
|
||||
var tool = FakeTool.SlowParallelSafe(TimeSpan.FromMilliseconds(120));
|
||||
var fixture = new EngineFixture().WithTool(tool);
|
||||
var agent = fixture.AddAgent("a1", "ParallelTool");
|
||||
agent.MaxParallelToolCalls = 2;
|
||||
|
||||
fixture.Client
|
||||
.RespondsWithToolCalls(
|
||||
("ParallelTool", "{}", "c1"),
|
||||
("ParallelTool", "{}", "c2"),
|
||||
("ParallelTool", "{}", "c3"),
|
||||
("ParallelTool", "{}", "c4"))
|
||||
.RespondsWithText("Fertig.");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
Assert.True(tool.MaxConcurrent <= 2,
|
||||
$"Grenze war 2, gleichzeitig liefen aber {tool.MaxConcurrent}.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task With_limit_one_everything_stays_serial()
|
||||
{
|
||||
var tool = FakeTool.SlowParallelSafe(TimeSpan.FromMilliseconds(60));
|
||||
var fixture = new EngineFixture().WithTool(tool);
|
||||
var agent = fixture.AddAgent("a1", "ParallelTool");
|
||||
agent.MaxParallelToolCalls = 1;
|
||||
|
||||
fixture.Client
|
||||
.RespondsWithToolCalls(
|
||||
("ParallelTool", "{}", "c1"),
|
||||
("ParallelTool", "{}", "c2"))
|
||||
.RespondsWithText("Fertig.");
|
||||
|
||||
await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, tool.MaxConcurrent);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tool that returns a marker after a time given in the call.
|
||||
/// This allows checking whether the order of the results follows the calls
|
||||
/// and not the completion.
|
||||
/// </summary>
|
||||
internal sealed class DelayedMarkerTool : ClawdDotNet.Core.Tools.IAgentTool
|
||||
{
|
||||
public string Name => "StaffelTool";
|
||||
public string Description => "Wartet und meldet eine Marke.";
|
||||
public bool IsParallelSafe => true;
|
||||
|
||||
public JsonElement InputSchema { get; } = JsonDocument.Parse("""
|
||||
{ "type": "object", "properties": { "delay": { "type": "integer" }, "marke": { "type": "string" } } }
|
||||
""").RootElement.Clone();
|
||||
|
||||
public async Task<ClawdDotNet.Core.Tools.ToolResult> ExecuteAsync(
|
||||
JsonElement input, ClawdDotNet.Core.Tools.AgentToolContext context, CancellationToken ct)
|
||||
{
|
||||
var delay = input.GetProperty("delay").GetInt32();
|
||||
var mark = input.GetProperty("marke").GetString()!;
|
||||
await Task.Delay(delay, ct);
|
||||
return ClawdDotNet.Core.Tools.ToolResult.Ok(mark);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user