RichardandClaude Opus 5.5
68e723a9c0
A lead looks at the runs of the agents it leads - held against the task and against what the engine stamped, never by asking the agent. - RunInspector (Core/Inspection): the mechanical check and first profile of the inspector foundation (#204). Flags failed runs, tool errors, refusals, unfinished calls, tasks answered with bookkeeping alone, and operations the agent names but never called in the run (the 2026-09-14 pattern). - SpotChecks: picks runs per unit (rate, denser for new agents, new models and after a failure, daily cap), deterministic by run id; one task per lead. - Escalation tool: spot_checks, inspect_run, check_done, propose_measure. Measures go to a human; nothing is carried out automatically. A security suspicion is its own case, not a personnel matter. - Organisation view: settings per unit, recent checks, measures to decide. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
354 lines
14 KiB
C#
354 lines
14 KiB
C#
using ClawdDotNet.App.Services;
|
|
using ClawdDotNet.Core.Cases;
|
|
using ClawdDotNet.Core.Config;
|
|
using Shouldly;
|
|
|
|
namespace ClawdDotNet.App.Tests.Services;
|
|
|
|
/// <summary>
|
|
/// #683: Units in the administration — create, lead, move, limit, write unit-wide values.
|
|
/// </summary>
|
|
public sealed class OrgAdministrationTests : IDisposable
|
|
{
|
|
private readonly TestHost _test;
|
|
|
|
public OrgAdministrationTests()
|
|
{
|
|
_test = new TestHost(
|
|
Agent("marketing-lead", "WebFetch", "AgentComm", "Taskboard"),
|
|
Agent("ads", "WebFetch", "Taskboard"),
|
|
Agent("listing", "WebFetch", "Taskboard"),
|
|
Agent("specialist", "WebFetch", "Taskboard", "Database"),
|
|
Agent("bookkeeper", "Database", "Taskboard"));
|
|
}
|
|
|
|
public void Dispose() => _test.Dispose();
|
|
|
|
private static AgentConfig Agent(string id, params string[] tools)
|
|
=> new() { AgentId = id, DisplayName = id, Tools = tools.ToDictionary(t => t, _ => new Dictionary<string, object?>()) };
|
|
|
|
private OrgAdministration Org => _test.Host.Organisation;
|
|
|
|
private AgentConfig Find(string id) => _test.Host.AgentAdmin.Find(id)!;
|
|
|
|
private void BuildMarketing()
|
|
{
|
|
Org.CreateUnit("marketing", "Marketing", null, "Sell more.").Ok.ShouldBeTrue();
|
|
Org.CreateUnit("amazon", "Amazon", "marketing").Ok.ShouldBeTrue();
|
|
Org.MoveAgent("marketing-lead", "marketing", "setup").Ok.ShouldBeTrue();
|
|
Org.SetLeader("marketing", "marketing-lead").Ok.ShouldBeTrue();
|
|
foreach (var id in new[] { "ads", "listing", "specialist" })
|
|
Org.MoveAgent(id, "amazon", "setup").Ok.ShouldBeTrue();
|
|
}
|
|
|
|
// ═══════════════════ units ═══════════════════
|
|
|
|
[Fact]
|
|
public void A_unit_needs_a_valid_id_a_name_and_an_existing_parent()
|
|
{
|
|
Org.CreateUnit("bad id", "X", null).Ok.ShouldBeFalse();
|
|
Org.CreateUnit("x", " ", null).Ok.ShouldBeFalse();
|
|
Org.CreateUnit("x", "X", "nowhere").Ok.ShouldBeFalse();
|
|
Org.CreateUnit("x", "X", null).Ok.ShouldBeTrue();
|
|
Org.CreateUnit("x", "Again", null).Ok.ShouldBeFalse("ids are unique");
|
|
}
|
|
|
|
[Fact]
|
|
public void Only_a_member_can_lead_a_unit()
|
|
{
|
|
Org.CreateUnit("marketing", "Marketing", null);
|
|
|
|
Org.SetLeader("marketing", "ads").Ok.ShouldBeFalse();
|
|
Org.MoveAgent("ads", "marketing", "join");
|
|
Org.SetLeader("marketing", "ads").Ok.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Only_an_empty_unit_can_be_deleted()
|
|
{
|
|
BuildMarketing();
|
|
|
|
Org.DeleteUnit("amazon").Ok.ShouldBeFalse("it has members");
|
|
foreach (var id in new[] { "ads", "listing", "specialist" })
|
|
Org.MoveAgent(id, null, "leave");
|
|
Org.DeleteUnit("amazon").Ok.ShouldBeTrue();
|
|
}
|
|
|
|
// ═══════════════════ moves ═══════════════════
|
|
|
|
[Fact]
|
|
public void A_move_records_since_when_and_where_from()
|
|
{
|
|
BuildMarketing();
|
|
|
|
Org.MoveAgent("ads", "marketing", "promoted").Ok.ShouldBeTrue();
|
|
|
|
var ads = Find("ads");
|
|
ads.UnitId.ShouldBe("marketing");
|
|
ads.PreviousUnitId.ShouldBe("amazon");
|
|
ads.UnitSinceUtc.ShouldNotBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_lead_that_moves_leaves_its_unit_without_lead()
|
|
{
|
|
BuildMarketing();
|
|
|
|
Org.MoveAgent("marketing-lead", "amazon", "reorg").Ok.ShouldBeTrue();
|
|
|
|
Org.Current().LeaderOf("marketing").ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_unit_cannot_be_moved_under_itself()
|
|
{
|
|
BuildMarketing();
|
|
|
|
Org.MoveUnit("marketing", "amazon", "oops").Ok.ShouldBeFalse();
|
|
Org.MoveUnit("marketing", "marketing", "oops").Ok.ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_moved_unit_takes_its_members_along_and_tells_them()
|
|
{
|
|
BuildMarketing();
|
|
Org.CreateUnit("marketplaces", "Marketplaces", "marketing");
|
|
var before = Find("ads").UnitSinceUtc;
|
|
|
|
Org.MoveUnit("amazon", "marketplaces", "a new level in between").Ok.ShouldBeTrue();
|
|
|
|
Find("ads").UnitId.ShouldBe("amazon", "the unit is the same");
|
|
Org.Current().PathOf("amazon").ShouldBe("Marketing / Marketplaces / Amazon");
|
|
Find("ads").UnitSinceUtc.ShouldNotBe(before, "the position changed, so the briefing says so");
|
|
}
|
|
|
|
// ═══════════════════ tool ceiling ═══════════════════
|
|
|
|
[Fact]
|
|
public void A_move_under_a_lower_ceiling_is_refused_until_confirmed()
|
|
{
|
|
BuildMarketing();
|
|
Org.SetToolCeiling("amazon", ["WebFetch", "Taskboard"], "no databases in ads").Ok.ShouldBeFalse(
|
|
"the specialist has Database — that has to be confirmed via the preview first");
|
|
|
|
var preview = Org.PreviewToolCeiling("amazon", ["WebFetch", "Taskboard"]);
|
|
preview.ToolsLost.Keys.ShouldBe(["specialist"]);
|
|
preview.ToolsLost["specialist"].ShouldBe(["Database"]);
|
|
}
|
|
|
|
[Fact]
|
|
public void Lowering_a_ceiling_removes_the_tools_beyond_it_from_the_members()
|
|
{
|
|
BuildMarketing();
|
|
Org.MoveAgent("specialist", null, "out first").Ok.ShouldBeTrue();
|
|
|
|
Org.SetToolCeiling("amazon", ["WebFetch", "Taskboard"], "no databases in ads").Ok.ShouldBeTrue();
|
|
|
|
// And from now on it cannot be granted.
|
|
var refused = _test.Host.AgentAdmin.SetToolEnabled("ads", "Database", true);
|
|
refused.Ok.ShouldBeFalse();
|
|
refused.Message.ShouldContain("ceiling");
|
|
}
|
|
|
|
[Fact]
|
|
public void Moving_into_a_ceiling_needs_the_tools_beyond_it_dropped_explicitly()
|
|
{
|
|
Org.CreateUnit("amazon", "Amazon", null);
|
|
Org.SetToolCeiling("amazon", ["WebFetch", "Taskboard"], "setup").Ok.ShouldBeTrue();
|
|
|
|
var refused = Org.MoveAgent("specialist", "amazon", "join");
|
|
refused.Ok.ShouldBeFalse();
|
|
refused.Message.ShouldContain("Database");
|
|
Find("specialist").UnitId.ShouldBeNull("nothing happened");
|
|
|
|
Org.MoveAgent("specialist", "amazon", "join", dropExcessTools: true).Ok.ShouldBeTrue();
|
|
Find("specialist").Tools.Keys.ShouldNotContain("Database");
|
|
}
|
|
|
|
[Fact]
|
|
public void A_ceiling_higher_up_limits_the_units_below()
|
|
{
|
|
BuildMarketing();
|
|
Org.MoveAgent("specialist", null, "out first");
|
|
Org.SetToolCeiling("marketing", ["WebFetch", "Taskboard", "AgentComm"], "setup").Ok.ShouldBeTrue();
|
|
|
|
_test.Host.AgentAdmin.SetToolEnabled("ads", "Database", true).Ok.ShouldBeFalse(
|
|
"amazon has no ceiling of its own, but marketing above it has");
|
|
}
|
|
|
|
// ═══════════════════ unit-wide values ═══════════════════
|
|
|
|
[Fact]
|
|
public void A_unit_wide_budget_marks_the_specialist_as_an_outlier_and_leaves_it_unselected()
|
|
{
|
|
BuildMarketing();
|
|
Find("ads").Budget.DailyCostUsd = 5;
|
|
Find("listing").Budget.DailyCostUsd = 4;
|
|
Find("specialist").Budget.DailyCostUsd = 40; // deliberately large
|
|
Find("marketing-lead").Budget.DailyCostUsd = 6;
|
|
|
|
var preview = Org.PreviewSetting("marketing", UnitSetting.DailyCostUsd, "10");
|
|
|
|
preview.Count.ShouldBe(4, "the unit and its sub-units");
|
|
var specialist = preview.Single(c => c.AgentId == "specialist");
|
|
specialist.Outlier.ShouldBeTrue();
|
|
specialist.SelectedByDefault.ShouldBeFalse();
|
|
preview.Where(c => c.AgentId != "specialist").ShouldAllBe(c => c.SelectedByDefault && c.Proposed == "10.00");
|
|
}
|
|
|
|
[Fact]
|
|
public void A_distributed_total_keeps_the_proportions()
|
|
{
|
|
BuildMarketing();
|
|
Find("ads").Budget.DailyCostUsd = 10;
|
|
Find("listing").Budget.DailyCostUsd = 10;
|
|
Find("specialist").Budget.DailyCostUsd = 20;
|
|
Find("marketing-lead").Budget.DailyCostUsd = 0;
|
|
|
|
var preview = Org.PreviewSetting("amazon", UnitSetting.DailyCostUsd, "80", UnitSettingMode.Distribute);
|
|
|
|
preview.Single(c => c.AgentId == "specialist").Proposed.ShouldBe("40.00");
|
|
preview.Single(c => c.AgentId == "ads").Proposed.ShouldBe("20.00");
|
|
}
|
|
|
|
[Fact]
|
|
public void Applying_writes_only_the_chosen_agents_and_remembers_the_default()
|
|
{
|
|
BuildMarketing();
|
|
Find("specialist").Budget.DailyCostUsd = 40;
|
|
|
|
var result = Org.ApplySetting("amazon", UnitSetting.DailyCostUsd,
|
|
new Dictionary<string, string> { ["ads"] = "10", ["listing"] = "10" }, "budget round", defaultValue: "10");
|
|
|
|
result.Ok.ShouldBeTrue(result.Message);
|
|
Find("ads").Budget.DailyCostUsd.ShouldBe(10);
|
|
Find("specialist").Budget.DailyCostUsd.ShouldBe(40, "not chosen, not touched");
|
|
Org.Current().Unit("amazon")!.Defaults.DailyCostUsd.ShouldBe(10);
|
|
}
|
|
|
|
[Fact]
|
|
public void An_agent_outside_the_unit_cannot_be_written_through_it()
|
|
{
|
|
BuildMarketing();
|
|
|
|
Org.ApplySetting("amazon", UnitSetting.DailyCostUsd,
|
|
new Dictionary<string, string> { ["bookkeeper"] = "1" }, "sneaky").Ok.ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_new_agent_placed_in_a_unit_starts_with_its_defaults_but_a_moved_one_keeps_its_values()
|
|
{
|
|
BuildMarketing();
|
|
Org.ApplySetting("amazon", UnitSetting.DailyCostUsd, new Dictionary<string, string>(), "defaults", defaultValue: "7");
|
|
Find("bookkeeper").Budget.DailyCostUsd = 30;
|
|
|
|
Org.MoveAgent("bookkeeper", "amazon", "moved").Ok.ShouldBeTrue();
|
|
Find("bookkeeper").Budget.DailyCostUsd.ShouldBe(30, "a move does not overwrite tuned values");
|
|
|
|
var created = _test.Host.AgentAdmin.Add(new NewAgent("newbie", "Newbie", "test/model", UnitId: "amazon"));
|
|
created.Ok.ShouldBeTrue(created.Message);
|
|
Find("newbie").Budget.DailyCostUsd.ShouldBe(7);
|
|
Find("newbie").UnitId.ShouldBe("amazon");
|
|
}
|
|
|
|
// ═══════════════════ approvals by the lead (#687) ═══════════════════
|
|
|
|
private UnitConfig Unit(string id) => _test.Host.Instance.Units.Single(u => u.Id == id);
|
|
|
|
[Fact]
|
|
public void A_unit_grants_its_lead_approvals_but_never_changes_to_agents()
|
|
{
|
|
BuildMarketing();
|
|
|
|
Org.SetApprovals("marketing", ["AgentEditor.update_soul"]).Message.ShouldContain("always stays with a human");
|
|
Org.SetApprovals("marketing", ["AgentEditor"]).Ok.ShouldBeFalse("a whole tool with such an operation reads as a grant for it");
|
|
Org.SetApprovals("marketing", [".x"]).Ok.ShouldBeFalse();
|
|
|
|
var set = Org.SetApprovals("marketing", ["Database.insert", "FileRW.delete"], perDay: 5, hours: 2, reason: "night shift");
|
|
|
|
set.Ok.ShouldBeTrue(set.Message);
|
|
Unit("marketing").Approvals!.Operations.ShouldBe(["Database.insert", "FileRW.delete"]);
|
|
Unit("marketing").Approvals!.PerDay.ShouldBe(5);
|
|
Org.SetApprovals("marketing", []).Ok.ShouldBeTrue();
|
|
Unit("marketing").Approvals.ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_sub_unit_grants_no_more_than_the_units_above_and_shrinks_with_them()
|
|
{
|
|
BuildMarketing();
|
|
Org.SetApprovals("marketing", ["Database"]).Ok.ShouldBeTrue();
|
|
|
|
Org.SetApprovals("amazon", ["Mail.send"]).Message.ShouldContain("Marketing");
|
|
Org.SetApprovals("amazon", ["Database.insert", "Database.update"]).Ok.ShouldBeTrue("'Database' above covers each of its operations");
|
|
|
|
var lowered = Org.SetApprovals("marketing", ["Database.insert"]);
|
|
|
|
lowered.Message.ShouldContain("Amazon");
|
|
Unit("amazon").Approvals!.Operations.ShouldBe(["Database.insert"]);
|
|
}
|
|
|
|
// ═══════════════════ spot checks (#693) ═══════════════════
|
|
|
|
[Fact]
|
|
public void A_human_sets_how_often_the_lead_checks_within_bounds()
|
|
{
|
|
BuildMarketing();
|
|
|
|
Org.SetInspection("marketing", true, rate: 1.5).Ok.ShouldBeFalse("a share lies between 0 and 1");
|
|
Org.SetInspection("marketing", true, perDay: -1).Ok.ShouldBeFalse();
|
|
|
|
var set = Org.SetInspection("marketing", true, rate: 0.2, denseRate: 0.1, perDay: 3, reason: "new team");
|
|
|
|
set.Ok.ShouldBeTrue(set.Message);
|
|
Unit("marketing").Inspection!.Rate.ShouldBe(0.2);
|
|
Unit("marketing").Inspection!.DenseRate.ShouldBe(0.2, "never less dense where it matters than for ordinary runs");
|
|
Unit("marketing").Inspection!.PerDay.ShouldBe(3);
|
|
Org.SetInspection("marketing", false).Ok.ShouldBeTrue();
|
|
Unit("marketing").Inspection.ShouldBeNull();
|
|
}
|
|
|
|
// ═══════════════════ the top (#690) ═══════════════════
|
|
|
|
private void MakeDecider(string id)
|
|
{
|
|
_test.Host.AgentAdmin.SetToolEnabled(id, "WebFetch", false).Ok.ShouldBeTrue();
|
|
_test.Host.AgentAdmin.SetToolEnabled(id, "Database", false).Ok.ShouldBeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Only_an_agent_that_fits_can_take_the_top()
|
|
{
|
|
BuildMarketing();
|
|
|
|
Org.SetTop(TopMode.Agent, null).Ok.ShouldBeFalse("an agent at the top needs a name");
|
|
Org.SetTop(TopMode.Agent, "bookkeeper").Message.ShouldContain("Database");
|
|
Org.SetTop(TopMode.Agent, "marketing-lead").Message.ShouldContain("belongs to a unit");
|
|
|
|
MakeDecider("bookkeeper");
|
|
var set = Org.SetTop(TopMode.HumanFirst, "bookkeeper", 3, "decisions at night");
|
|
|
|
set.Ok.ShouldBeTrue(set.Message);
|
|
var top = _test.Host.Instance.Cases.Top;
|
|
top.Mode.ShouldBe(TopMode.HumanFirst);
|
|
top.AgentId.ShouldBe("bookkeeper");
|
|
top.HumanFirstHours.ShouldBe(3);
|
|
|
|
Org.SetTop(TopMode.Human, null).Ok.ShouldBeTrue();
|
|
_test.Host.Instance.Cases.Top.AgentId.ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void The_deciding_agent_cannot_be_given_outside_tools_nor_moved_into_a_unit()
|
|
{
|
|
BuildMarketing();
|
|
MakeDecider("bookkeeper");
|
|
Org.SetTop(TopMode.Agent, "bookkeeper").Ok.ShouldBeTrue();
|
|
|
|
_test.Host.AgentAdmin.SetToolEnabled("bookkeeper", "Mail", true).Message.ShouldContain("deciding agent");
|
|
_test.Host.AgentAdmin.SetToolEnabled("bookkeeper", "Memory", true).Ok.ShouldBeTrue();
|
|
Org.MoveAgent("bookkeeper", "marketing", "reorg").Ok.ShouldBeFalse();
|
|
Find("bookkeeper").UnitId.ShouldBeNull();
|
|
}
|
|
}
|