using System.Diagnostics;
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text;
using Microsoft.Extensions.Logging;
namespace ClawdDotNet.Core.Sandbox;
///
/// #654: The sandbox under Linux, built on bubblewrap.
///
/// What holds, and where it comes from (each with a test that tries to break it):
///
/// - File system: own mount namespace. /usr and the few files of
/// /etc a program needs are mounted read-only; the workspace at /work is the
/// only writable place besides a private /tmp. $HOME, the instance, the key
/// ring and other workspaces are not mounted at all.
/// - Inheritance: the environment of bwrap itself is empty, the sandbox gets only
/// what the request names. .NET opens its files close-on-exec; the sandbox sees fds 0–2.
/// - Network: own network namespace with nothing but loopback.
/// - Limits: wall-clock timeout (the whole namespace dies with bwrap), address
/// space, CPU time, file size, open files — as rlimits; processes as RLIMIT_NPROC, set
/// inside the sandbox's own user namespace, where it counts the sandbox's processes and not
/// the instance's threads. Output is capped while it is read.
/// - Privileges: all capabilities dropped, no_new_privs (bwrap sets it
/// always), no nested user namespaces where bwrap can refuse them, and the
/// .
///
///
public sealed class BwrapSandbox : ISandboxRunner
{
private const string Shell = "/bin/sh";
private readonly string _bwrap;
private readonly BwrapFeatures _features;
private readonly ILogger _logger;
private BwrapSandbox(string bwrap, BwrapFeatures features, ILogger logger)
{
_bwrap = bwrap;
_features = features;
_logger = logger;
}
public bool IsAvailable => true;
public string? UnavailableReason => null;
///
/// #712: The launcher for runs that need network or secrets ().
/// Without it such a run is refused, not run without them.
///
public SandboxHelper? Helper { get; set; }
/// #712: The logger the egress proxy reports to.
public ILoggerFactory? LoggerFactory { get; set; }
///
/// The runner of this machine: bwrap under Linux if it can actually build a sandbox here
/// (tried once — a kernel or AppArmor policy that forbids user namespaces shows only when
/// trying), otherwise with the reason.
///
public static ISandboxRunner Detect(ILogger logger)
{
if (!OperatingSystem.IsLinux())
return new NoSandbox("code runs only under Linux, in a bubblewrap sandbox");
if (!SeccompFilter.Supports(RuntimeInformation.ProcessArchitecture))
return new NoSandbox($"no seccomp filter for {RuntimeInformation.ProcessArchitecture}");
var bwrap = new[] { "/usr/bin/bwrap", "/bin/bwrap", "/usr/local/bin/bwrap" }.FirstOrDefault(File.Exists);
if (bwrap is null)
return new NoSandbox("bubblewrap (bwrap) is not installed — 'apt install bubblewrap'");
var help = RunQuiet(bwrap, ["--help"]);
// The probe mounts the system exactly as a real run does — a missing /lib64 (the
// dynamic loader) looks like "no such file" and would pass for a broken sandbox.
var probeArgs = new List { "--unshare-all", "--die-with-parent", "--ro-bind", "/usr", "/usr" };
foreach (var (path, link) in HostLayout.Read().TopLevel)
probeArgs.AddRange(link is not null ? ["--symlink", link, path] : ["--ro-bind-try", path, path]);
probeArgs.AddRange(["--proc", "/proc", "--dev", "/dev", "/usr/bin/true"]);
var probe = RunQuiet(bwrap, probeArgs);
if (probe.ExitCode != 0)
return new NoSandbox($"bwrap cannot create a sandbox here ({probe.Output.Trim()}) — user namespaces may be disabled");
var features = new BwrapFeatures(
DisableUserns: help.Output.Contains("--disable-userns", StringComparison.Ordinal),
ClearEnv: help.Output.Contains("--clearenv", StringComparison.Ordinal));
logger.LogInformation("Sandbox ready: {Bwrap}{Userns}", bwrap,
features.DisableUserns ? "" : " (older bwrap: nested user namespaces are refused by the seccomp filter only)");
return new BwrapSandbox(bwrap, features, logger);
}
public async Task RunAsync(SandboxRequest request, CancellationToken ct)
{
Validate(request);
if (request.NeedsLauncher && Helper is null)
throw new SandboxUnavailableException("this run needs network or secrets, and the sandbox launcher is not available");
// A directory only this user can enter: the filter, the proxy's socket, the secrets.
var privateDir = Path.Combine(Path.GetTempPath(), $"clawddotnet_sbx_{Guid.NewGuid():N}");
Directory.CreateDirectory(privateDir);
if (!OperatingSystem.IsWindows())
File.SetUnixFileMode(privateDir, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute);
EgressProxy? proxy = null;
try
{
var filterFile = Path.Combine(privateDir, "seccomp.bpf");
await File.WriteAllBytesAsync(filterFile, SeccompFilter.Build(RuntimeInformation.ProcessArchitecture), ct);
var launch = LaunchFiles.None;
if (request.Egress.Count > 0)
{
var socket = Path.Combine(privateDir, "egress.sock");
proxy = EgressProxy.Start(socket, request.Egress,
LoggerFactory?.CreateLogger() ?? (ILogger)Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance);
launch = launch with { EgressSocket = socket };
}
if (request.SecretEnvironment.Count > 0)
{
var secrets = Path.Combine(privateDir, "env.json");
await SandboxLauncher.WriteSecretFileAsync(secrets, request.SecretEnvironment, ct);
launch = launch with { SecretFile = secrets };
}
var start = new ProcessStartInfo(Shell)
{
UseShellExecute = false,
RedirectStandardInput = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
StandardOutputEncoding = Encoding.UTF8,
StandardErrorEncoding = Encoding.UTF8
};
// Nothing of the main process's environment — not even to bwrap.
start.Environment.Clear();
foreach (var argument in CommandLine(_bwrap, filterFile, request, HostLayout.Read(), _features, Helper, launch))
start.ArgumentList.Add(argument);
return await RunProcessAsync(start, request, ct);
}
finally
{
proxy?.Dispose();
try { Directory.Delete(privateDir, recursive: true); }
catch (IOException) { /* in /tmp, harmless */ }
}
}
///
/// The whole command line: /bin/sh -c WRAPPER sh FILTER LIMITS… bwrap ARGS… -- sh -c INNER sh NPROC PROGRAM ARGS….
/// Pure — the tests read it on any platform.
///
public static IReadOnlyList CommandLine(
string bwrap, string filterFile, SandboxRequest request, HostLayout host, BwrapFeatures features,
SandboxHelper? helper = null, LaunchFiles? launch = null)
{
launch ??= LaunchFiles.None;
if (request.NeedsLauncher && helper is null)
throw new ArgumentException("This run needs the sandbox launcher.");
var limits = request.Limits;
var args = new List
{
"-c",
// Outer: rlimits that bind the sandbox as a whole, then bwrap with the filter on fd 3.
"""f=$1; ulimit -v "$2" && ulimit -t "$3" && ulimit -f "$4" && ulimit -n "$5" || exit 126; shift 5; exec "$@" 3<"$f" """,
"sh",
filterFile,
limits.MaxMemoryBytes > 0 ? Kilobytes(limits.MaxMemoryBytes) : "unlimited",
((long)Math.Ceiling(limits.Timeout.TotalSeconds) + 5).ToString(CultureInfo.InvariantCulture),
Blocks(limits.MaxFileBytes),
limits.MaxOpenFiles.ToString(CultureInfo.InvariantCulture),
bwrap,
"--unshare-all",
"--unshare-user",
"--die-with-parent",
"--new-session",
"--cap-drop", "ALL",
"--hostname", "sandbox"
};
// bwrap's own environment is already empty; where it can, it says so once more.
if (features.ClearEnv)
args.Add("--clearenv");
if (features.DisableUserns)
args.Add("--disable-userns");
// The system, read-only. Merged /usr: /bin, /lib … are links into it.
args.AddRange(["--ro-bind", "/usr", "/usr"]);
foreach (var (path, link) in host.TopLevel)
{
if (link is not null)
args.AddRange(["--symlink", link, path]);
else
args.AddRange(["--ro-bind-try", path, path]);
}
foreach (var etc in EtcFiles)
args.AddRange(["--ro-bind-try", etc, etc]);
args.AddRange(["--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", "/run"]);
// The workspace, and inside it what must stay as it was checked.
var workspace = Path.GetFullPath(request.WorkspacePath);
args.AddRange(["--bind", workspace, SandboxPaths.Work]);
foreach (var relative in request.ProtectedPaths)
{
var hostPath = Path.GetFullPath(Path.Combine(workspace, relative));
if (SandboxPaths.InSandbox(workspace, hostPath) is { } inside && inside != SandboxPaths.Work)
args.AddRange(["--ro-bind-try", hostPath, inside]);
}
foreach (var path in request.ReadOnlyPaths)
args.AddRange(["--ro-bind", Path.GetFullPath(path), Path.GetFullPath(path)]);
// #712: a runtime's home, the MCP socket — each at its own place, never inside /work,
// where it would stand among the files that come back as the run's result.
foreach (var mount in request.Mounts)
{
if (!mount.Target.StartsWith('/') || mount.Target == SandboxPaths.Work
|| mount.Target.StartsWith(SandboxPaths.Work + "/", StringComparison.Ordinal)
|| mount.Target.Split('/').Contains(".."))
throw new ArgumentException($"'{mount.Target}' is not a place for a mount.");
args.AddRange([mount.Writable ? "--bind" : "--ro-bind", Path.GetFullPath(mount.HostPath), mount.Target]);
}
// #712: what the launcher needs — the helper itself, the proxy's socket, the secrets.
if (request.NeedsLauncher && helper is not null)
{
foreach (var path in helper.ReadOnlyPaths.Except(request.ReadOnlyPaths.Select(Path.GetFullPath)))
args.AddRange(["--ro-bind", path, path]);
if (launch.EgressSocket is { } socket)
args.AddRange(["--bind", socket, SandboxPaths.EgressSocket]);
if (launch.SecretFile is { } secrets)
args.AddRange(["--ro-bind", secrets, SandboxPaths.SecretFile]);
}
// The root bwrap builds is a tmpfs of its own — writable unless it is remounted. Only
// /work and /tmp, each a mount of its own, stay writable.
args.AddRange(["--remount-ro", "/", "--chdir", SandboxPaths.Work]);
var environment = new Dictionary(StringComparer.Ordinal)
{
["PATH"] = "/usr/local/bin:/usr/bin:/bin",
["HOME"] = "/tmp",
["LANG"] = "C.UTF-8",
["TMPDIR"] = "/tmp"
};
if (request.NeedsLauncher && helper is not null)
{
foreach (var (key, value) in helper.Environment)
environment[key] = value;
if (request.Egress.Count > 0)
{
// Everything goes to the proxy; the sandbox's loopback has nothing else.
var proxy = $"http://127.0.0.1:{SandboxPaths.ProxyPort}";
environment["HTTPS_PROXY"] = environment["https_proxy"] = proxy;
environment["HTTP_PROXY"] = environment["http_proxy"] = proxy;
}
}
foreach (var (key, value) in request.Environment)
environment[key] = value;
foreach (var (key, value) in environment.OrderBy(e => e.Key, StringComparer.Ordinal))
args.AddRange(["--setenv", key, value]);
args.AddRange(["--seccomp", "3", "--"]);
// Inner: fd 3 — the filter bwrap read — is closed; bwrap hands it on otherwise. Then
// the process limit, set inside the sandbox's own user namespace. bash calls it -u,
// dash (Debian's and Ubuntu's sh) -p; if neither takes, nothing runs.
args.AddRange([Shell, "-c", """exec 3<&-; ulimit -u "$1" 2>/dev/null || ulimit -p "$1" || exit 126; shift; exec "$@" """, "sh",
limits.MaxProcesses.ToString(CultureInfo.InvariantCulture)]);
if (request.NeedsLauncher && helper is not null)
{
args.AddRange([helper.Executable, SandboxLauncher.Argument]);
if (request.Egress.Count > 0)
args.AddRange([SandboxLauncher.EgressOption, SandboxPaths.EgressSocket]);
if (request.SecretEnvironment.Count > 0)
args.AddRange([SandboxLauncher.SecretsOption, SandboxPaths.SecretFile]);
args.Add("--");
}
args.Add(request.Executable);
args.AddRange(request.Arguments);
return args;
}
/// What a program commonly needs of /etc — and nothing that says who uses this machine.
private static readonly string[] EtcFiles =
[
"/etc/ld.so.cache", "/etc/ld.so.conf", "/etc/ld.so.conf.d", "/etc/alternatives",
"/etc/localtime", "/etc/ssl/certs", "/etc/ca-certificates", "/etc/fonts", "/etc/mime.types"
];
/// The workspace must exist and must not be a place that contains everything else.
private static void Validate(SandboxRequest request)
{
var workspace = Path.GetFullPath(request.WorkspacePath).TrimEnd('/');
if (!Directory.Exists(workspace))
throw new ArgumentException($"The workspace '{workspace}' does not exist.");
var home = System.Environment.GetFolderPath(System.Environment.SpecialFolder.UserProfile).TrimEnd('/');
if (workspace.Length == 0 || workspace == home || workspace.Count(c => c == '/') < 2)
throw new ArgumentException($"'{workspace}' is not a workspace — it would expose far more than one agent's files.");
if (string.IsNullOrWhiteSpace(request.Executable))
throw new ArgumentException("No program given.");
}
private async Task RunProcessAsync(ProcessStartInfo start, SandboxRequest request, CancellationToken ct)
{
using var process = await Spawner.StartAsync(start);
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
timeout.CancelAfter(request.Limits.Timeout);
var output = new CappedOutput(request.Limits.MaxOutputBytes);
var stdout = PumpAsync(process.StandardOutput, output, isError: false, process);
var stderr = PumpAsync(process.StandardError, output, isError: true, process);
try
{
if (request.StandardInput is { } input)
await process.StandardInput.WriteAsync(input.AsMemory(), timeout.Token);
process.StandardInput.Close();
}
catch (IOException)
{
// The program did not read its input — that is its business.
}
var timedOut = false;
try
{
await process.WaitForExitAsync(timeout.Token);
}
catch (OperationCanceledException)
{
timedOut = !ct.IsCancellationRequested;
Kill(process);
if (!timedOut)
throw;
}
// After a stop, the streams end with the processes — but a reader must never hang on a
// pipe something still holds. Bounded, then the result is what arrived.
var pumps = Task.WhenAll(stdout, stderr);
if (timedOut || output.Truncated)
await Task.WhenAny(pumps, Task.Delay(TimeSpan.FromSeconds(5), CancellationToken.None));
else
await pumps;
if (timedOut)
_logger.LogWarning("Sandbox run of {Program} stopped after {Timeout}", request.Executable, request.Limits.Timeout);
return new SandboxResult
{
ExitCode = process.HasExited ? process.ExitCode : -1,
StandardOutput = output.Out,
StandardError = output.Err,
TimedOut = timedOut,
OutputTruncated = output.Truncated
};
}
/// Reads one stream into the shared cap; past the cap the run is stopped.
private static async Task PumpAsync(StreamReader reader, CappedOutput output, bool isError, Process process)
{
var buffer = new char[8192];
int read;
while ((read = await reader.ReadAsync(buffer.AsMemory())) > 0)
{
if (!output.Append(buffer.AsSpan(0, read), isError))
{
Kill(process);
return;
}
}
}
///
/// Stops the run: everything below bwrap is collected first, then all of it is killed.
///
/// Not trusting --die-with-parent. The kernel clears the parent-death
/// signal when a process changes its credentials — and bwrap does exactly that while it
/// builds the user namespace. Seen on the test machine: bwrap's monitor was gone, the
/// sandbox lived on under PID 1 and kept the output pipe open. Killing the sandbox's own
/// PID 1 (the init of its PID namespace) takes every process inside with it.
///
private static void Kill(Process process)
{
int root;
try
{
root = process.Id;
}
catch (InvalidOperationException)
{
return;
}
foreach (var pid in Descendants(root).Append(root))
{
try
{
using var victim = Process.GetProcessById(pid);
victim.Kill();
}
catch (ArgumentException) { } // already gone
catch (InvalidOperationException) { }
catch (System.ComponentModel.Win32Exception) { }
}
}
/// All processes below , from /proc — collected before anything is killed.
private static IEnumerable Descendants(int root)
{
var children = new Dictionary>();
foreach (var dir in Directory.EnumerateDirectories("/proc"))
{
if (!int.TryParse(Path.GetFileName(dir), out var pid))
continue;
try
{
// /proc//stat: "pid (comm) state ppid …" — comm may hold spaces and parentheses.
var stat = File.ReadAllText(Path.Combine(dir, "stat"));
var fields = stat[(stat.LastIndexOf(')') + 2)..].Split(' ');
if (int.TryParse(fields[1], out var parent))
{
if (!children.TryGetValue(parent, out var list))
children[parent] = list = [];
list.Add(pid);
}
}
catch (IOException) { } // it ended meanwhile
catch (UnauthorizedAccessException) { }
}
var result = new List();
var queue = new Queue([root]);
while (queue.Count > 0 && result.Count < 10_000)
{
foreach (var child in children.GetValueOrDefault(queue.Dequeue()) ?? [])
{
result.Add(child);
queue.Enqueue(child);
}
}
return result;
}
private static string Kilobytes(long bytes) => Math.Max(1, bytes / 1024).ToString(CultureInfo.InvariantCulture);
/// ulimit -f counts 512-byte blocks in POSIX sh.
private static string Blocks(long bytes) => Math.Max(1, bytes / 512).ToString(CultureInfo.InvariantCulture);
private static (int ExitCode, string Output) RunQuiet(string program, IReadOnlyList arguments)
{
try
{
var start = new ProcessStartInfo(program)
{
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true
};
start.Environment.Clear();
foreach (var argument in arguments)
start.ArgumentList.Add(argument);
using var process = Process.Start(start)!;
var text = process.StandardOutput.ReadToEnd() + process.StandardError.ReadToEnd();
return process.WaitForExit(5000) ? (process.ExitCode, text) : (-1, "timed out");
}
catch (Exception ex) when (ex is System.ComponentModel.Win32Exception or IOException)
{
return (-1, ex.Message);
}
}
private sealed class CappedOutput(int max)
{
private readonly StringBuilder _out = new();
private readonly StringBuilder _err = new();
private int _total;
public bool Truncated { get; private set; }
public string Out { get { lock (this) return _out.ToString(); } }
public string Err { get { lock (this) return _err.ToString(); } }
public bool Append(ReadOnlySpan text, bool isError)
{
lock (this)
{
var room = max - _total;
var take = Math.Min(room, text.Length);
(isError ? _err : _out).Append(text[..Math.Max(0, take)]);
_total += Math.Max(0, take);
if (take < text.Length)
Truncated = true;
return !Truncated;
}
}
}
}
///
/// #654: Starts every sandbox from one thread that lives as long as the process.
///
/// Why. --die-with-parent uses the kernel's parent-death signal, and that
/// is bound to the thread that forked, not to the process. .NET forks on the calling
/// thread — a thread-pool thread, and the pool retires idle threads. Seen on the test machine
/// under load: the pool dropped a thread, bwrap got SIGKILL (exit 137), and when it happened
/// while bwrap was still building the sandbox, the inside lived on as an orphan holding the
/// output pipes. Started from a thread that never ends, the signal fires only when the
/// instance itself dies — which is what it is for.
///
internal static class Spawner
{
private static readonly System.Collections.Concurrent.BlockingCollection<(ProcessStartInfo Start, TaskCompletionSource Done)> Queue = new();
static Spawner()
{
var thread = new Thread(() =>
{
foreach (var (start, done) in Queue.GetConsumingEnumerable())
{
try
{
done.SetResult(Process.Start(start) ?? throw new InvalidOperationException("The sandbox did not start."));
}
catch (Exception ex)
{
done.SetException(ex);
}
}
})
{
IsBackground = true,
Name = "ClawdDotNet sandbox spawner"
};
thread.Start();
}
public static Task StartAsync(ProcessStartInfo start)
{
var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
Queue.Add((start, done));
return done.Task;
}
}
/// #712: Files the runner prepared for the launcher, on the host.
public sealed record LaunchFiles(string? EgressSocket, string? SecretFile)
{
public static LaunchFiles None { get; } = new(null, null);
}
/// #654: What the installed bwrap can do beyond the basics.
public sealed record BwrapFeatures(bool DisableUserns, bool ClearEnv)
{
public static BwrapFeatures All { get; } = new(true, true);
}
///
/// #654: How the top level of this machine looks — which of /bin, /lib … are
/// links into /usr (merged /usr) and which are directories. Read once per run; a
/// record, so the tests can describe any machine.
///
public sealed record HostLayout(IReadOnlyList<(string Path, string? Link)> TopLevel)
{
private static readonly string[] Candidates = ["/bin", "/sbin", "/lib", "/lib64", "/lib32", "/libx32"];
public static HostLayout Read()
{
var entries = new List<(string, string?)>();
foreach (var path in Candidates)
{
var info = new FileInfo(path);
if (info.LinkTarget is { } target)
entries.Add((path, target.TrimStart('/')));
else if (Directory.Exists(path))
entries.Add((path, null));
}
return new HostLayout(entries);
}
/// A merged-/usr system, as Debian and Ubuntu are today.
public static HostLayout MergedUsr { get; } = new([("/bin", "usr/bin"), ("/sbin", "usr/sbin"), ("/lib", "usr/lib"), ("/lib64", "usr/lib64")]);
}