Automatisierungsvorschlaege: Agenten schlagen n8n-Workflows vor, der Mensch importiert von Hand (#726)
- Core/Automation/WorkflowReview: free check of an n8n export - every node with its effect and target, code/command/SSH nodes only with a stated reason, embedded secrets refused (by name or by look), a short summary of what the workflow does. - AutomationProposals: measured saving from the replaced tasks' receipts (28 days), at most 3 open per agent, the same purpose not twice, a rejected proposal must be answered; the human gets a task and decides in the approvals view. Accepting records it - importing stays a human's step. - Candidates: recurring tasks whose runs repeat the same calls (>= 75 % of >= 8 runs in 14 days) are put to their agent once a month. - Escalation tool: automation_find, automation_propose. - Approvals view: proposals with nodes, targets, warnings, saving, JSON download, accept/reject. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
64e09b82d5
commit
067ff62317
@@ -0,0 +1,176 @@
|
||||
using ClawdDotNet.Core.Audit;
|
||||
using ClawdDotNet.Core.Automation;
|
||||
using ClawdDotNet.Core.Storage;
|
||||
using ClawdDotNet.Core.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Automation;
|
||||
|
||||
/// <summary>
|
||||
/// #726: Agents propose workflows, measured and checked for free; the human decides and imports
|
||||
/// by hand. No flood, and a rejected proposal does not come back without a new argument.
|
||||
/// </summary>
|
||||
public sealed class AutomationProposalsTests : IDisposable
|
||||
{
|
||||
private readonly string _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", "auto-" + Guid.NewGuid().ToString("N")[..8]);
|
||||
private readonly SqliteAuditRepository _audit;
|
||||
private readonly SqliteTaskRepository _tasks;
|
||||
private readonly TaskboardService _board;
|
||||
private readonly AutomationProposals _proposals;
|
||||
|
||||
private const string Workflow = """
|
||||
{"nodes":[
|
||||
{"name":"Every morning","type":"n8n-nodes-base.scheduleTrigger","parameters":{}},
|
||||
{"name":"Get rates","type":"n8n-nodes-base.httpRequest","parameters":{"url":"https://api.rates.example/eur","headers":{"x":"={{ $env.TOKEN }}"}}},
|
||||
{"name":"Mail","type":"n8n-nodes-base.emailSend","parameters":{"toEmail":"desk@example.org"},"credentials":{"smtp":{"id":"1","name":"Office SMTP"}}}
|
||||
],"connections":{}}
|
||||
""";
|
||||
|
||||
public AutomationProposalsTests()
|
||||
{
|
||||
var storage = new SqliteStorage(Path.Combine(_directory, "state.db"));
|
||||
_audit = new SqliteAuditRepository(storage);
|
||||
_tasks = new SqliteTaskRepository(storage);
|
||||
_board = new TaskboardService(_tasks, Path.Combine(_directory, "tasks"));
|
||||
_proposals = new AutomationProposals(storage, _audit, _tasks, _board, NullLogger.Instance);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
SqliteStorage.ReleasePooledConnections(_directory);
|
||||
try { Directory.Delete(_directory, recursive: true); }
|
||||
catch { /* cleanup is secondary */ }
|
||||
}
|
||||
|
||||
// ─── the review ───
|
||||
|
||||
[Fact]
|
||||
public void The_review_says_what_the_workflow_does_and_where_it_reaches()
|
||||
{
|
||||
var review = WorkflowReview.Review(Workflow, null);
|
||||
|
||||
review.Ok.ShouldBeTrue(string.Join(" ", review.Problems));
|
||||
review.Nodes.Select(n => n.Effect).ShouldBe(["starts the workflow", "calls an address", "sends mail"]);
|
||||
review.Targets.ShouldBe(["api.rates.example", "desk@example.org"]);
|
||||
review.Summary.ShouldContain("Reaches outside: api.rates.example");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Code_needs_a_reason_and_a_secret_is_never_carried()
|
||||
{
|
||||
const string code = """{"nodes":[{"name":"Calc","type":"n8n-nodes-base.code","parameters":{"jsCode":"return 1"}}]}""";
|
||||
|
||||
WorkflowReview.Review(code, null).Problems.ShouldContain(p => p.Contains("Without a reason"));
|
||||
var reasoned = WorkflowReview.Review(code, "Rounds prices the same way the shop does.");
|
||||
reasoned.Ok.ShouldBeTrue();
|
||||
reasoned.Warnings.ShouldContain(w => w.Contains("Rounds prices"));
|
||||
|
||||
WorkflowReview.Review("""{"nodes":[{"name":"Api","type":"n8n-nodes-base.httpRequest","parameters":{"url":"https://x.example","apiKey":"k-123"}}]}""", null)
|
||||
.Problems.ShouldContain(p => p.Contains("parameters") || p.Contains("apiKey"));
|
||||
WorkflowReview.Review("""{"nodes":[{"name":"Api","type":"n8n-nodes-base.httpRequest","parameters":{"headerValue":"Bearer abc"}}]}""", null)
|
||||
.Ok.ShouldBeFalse("a bearer token by its look");
|
||||
WorkflowReview.Review("not json", null).Ok.ShouldBeFalse();
|
||||
WorkflowReview.Review("""{"nodes":[]}""", null).Ok.ShouldBeFalse();
|
||||
}
|
||||
|
||||
// ─── proposing ───
|
||||
|
||||
private async Task<string> RecurringTask(int runs, string agent = "desk", bool same = true)
|
||||
{
|
||||
var task = await _board.CreateAsync(new TaskItem
|
||||
{
|
||||
Title = "Morning exchange rates",
|
||||
Assignee = "@" + agent,
|
||||
When = new TaskWhen { Kind = TaskWhenKind.Cron, Value = "0 7 * * *", TimeZone = "Europe/Berlin" }
|
||||
}, default);
|
||||
for (var i = 0; i < runs; i++)
|
||||
{
|
||||
var run = $"run-{task.Id}-{i}";
|
||||
await _audit.AppendAsync(new AuditEntry { RunId = run, AgentId = agent, Tool = "WebFetch", Arguments = "{\"action\":\"fetch\"}", Status = AuditStatus.Ok, OccurredAt = DateTime.UtcNow.AddDays(-i) }, default);
|
||||
await _audit.AppendAsync(new AuditEntry { RunId = run, AgentId = agent, Tool = same || i % 2 == 0 ? "Mail" : "FileRW", Arguments = "{\"action\":\"send\"}", Status = AuditStatus.Ok, OccurredAt = DateTime.UtcNow.AddDays(-i) }, default);
|
||||
await _audit.RecordReceiptAsync(new RunReceipt { RunId = run, AgentId = agent, TaskId = task.Id, Status = "Completed", CostUsd = 0.05m, CostIsKnown = true, PromptTokens = 1000, OccurredAt = DateTime.UtcNow.AddDays(-i) }, default);
|
||||
}
|
||||
return task.Id;
|
||||
}
|
||||
|
||||
private AutomationRequest Request(string taskId, string purpose = "Send the morning exchange rates", long? answers = null, string? argument = null)
|
||||
=> new("desk", purpose, "every morning at 7", [taskId], ["Office SMTP"], "Sends rates to the desk only.", "Run once by hand, check the mail.", Workflow, null, answers, argument);
|
||||
|
||||
[Fact]
|
||||
public async Task A_proposal_carries_its_measured_saving_and_goes_to_a_human()
|
||||
{
|
||||
var taskId = await RecurringTask(10);
|
||||
|
||||
var outcome = await _proposals.ProposeAsync(Request(taskId), default);
|
||||
|
||||
outcome.Ok.ShouldBeTrue(outcome.Message);
|
||||
var proposal = (await _proposals.ListAsync("proposed", 10, default)).Single();
|
||||
proposal.Savings.Runs.ShouldBe(10);
|
||||
proposal.Savings.CostUsd.ShouldBe(0.50m);
|
||||
proposal.Summary.ShouldContain("api.rates.example");
|
||||
(await _board.ListAsync(new TaskQuery { Assignee = TaskAssignee.Human }, default)).ShouldContain(t => t.Title.Contains($"#{proposal.Id}"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_same_purpose_is_not_proposed_twice_and_a_rejection_must_be_answered()
|
||||
{
|
||||
var taskId = await RecurringTask(3);
|
||||
var first = await _proposals.ProposeAsync(Request(taskId), default);
|
||||
|
||||
(await _proposals.ProposeAsync(Request(taskId, "Send the exchange rates every morning"), default)).Message.ShouldContain($"#{first.Id}");
|
||||
|
||||
(await _proposals.DecideAsync(first.Id!.Value, false, "", default)).Ok.ShouldBeFalse("a rejection needs its reason");
|
||||
(await _proposals.DecideAsync(first.Id!.Value, false, "Rates change format often — the agent copes, a workflow breaks.", default)).Ok.ShouldBeTrue();
|
||||
|
||||
(await _proposals.ProposeAsync(Request(taskId), default)).Message.ShouldContain("must answer");
|
||||
(await _proposals.ProposeAsync(Request(taskId, answers: first.Id, argument: "The rates API now has a versioned, stable format."), default))
|
||||
.Ok.ShouldBeTrue();
|
||||
(await _proposals.FindAsync("exchange rates", default)).Count.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_agent_has_at_most_three_open_proposals()
|
||||
{
|
||||
for (var i = 0; i < 3; i++)
|
||||
(await _proposals.ProposeAsync(Request(await RecurringTask(1), $"Purpose number {i} quite different {Guid.NewGuid():N}"), default)).Ok.ShouldBeTrue();
|
||||
|
||||
(await _proposals.ProposeAsync(Request(await RecurringTask(1), "A fourth unrelated thing entirely"), default)).Message.ShouldContain("Wait for a decision");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_proposal_without_the_required_parts_or_a_broken_workflow_is_refused()
|
||||
{
|
||||
var taskId = await RecurringTask(1);
|
||||
|
||||
(await _proposals.ProposeAsync(Request(taskId) with { TaskIds = [] }, default)).Message.ShouldContain("'tasks'");
|
||||
(await _proposals.ProposeAsync(Request("t-nope"), default)).Message.ShouldContain("no task");
|
||||
(await _proposals.ProposeAsync(Request(taskId) with { WorkflowJson = """{"nodes":[{"name":"x","type":"n8n-nodes-base.executeCommand"}]}""" }, default))
|
||||
.Message.ShouldContain("does not pass the check");
|
||||
}
|
||||
|
||||
// ─── candidates ───
|
||||
|
||||
[Fact]
|
||||
public async Task A_recurring_task_that_does_the_same_each_time_is_put_to_its_agent_once()
|
||||
{
|
||||
var same = await RecurringTask(9, "desk");
|
||||
var varied = await RecurringTask(9, "other", same: false);
|
||||
|
||||
var noted = await _proposals.FindCandidatesAsync(default);
|
||||
|
||||
noted.ShouldBe([same], "half the runs of the other task differ");
|
||||
(await _board.ListAsync(new TaskQuery { Assignee = "@desk" }, default)).ShouldContain(t => t.Title.StartsWith("Automation candidate"));
|
||||
(await _proposals.FindCandidatesAsync(default)).ShouldBeEmpty("once a month, not every day");
|
||||
varied.ShouldNotBe(same);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_signature_folds_repeats_and_leaves_out_bookkeeping()
|
||||
{
|
||||
AuditEntry E(string tool, string op) => new() { Tool = tool, Arguments = $"{{\"action\":\"{op}\"}}", Status = AuditStatus.Ok };
|
||||
|
||||
AutomationProposals.Signature([E("FileRW", "read"), E("FileRW", "read"), E("Taskboard", "complete"), E("Mail", "send")])
|
||||
.ShouldBe("FileRW.read > Mail.send");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user