using System.Diagnostics;
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text;
using Microsoft.Extensions.Logging;
namespace ClawdDotNet.Core.Sandbox;
///
/// #654: The sandbox under Linux, built on bubblewrap.
///
/// What holds, and where it comes from (each with a test that tries to break it):
///
/// - File system: own mount namespace. /usr and the few files of
/// /etc a program needs are mounted read-only; the workspace at /work is the
/// only writable place besides a private /tmp. $HOME, the instance, the key
/// ring and other workspaces are not mounted at all.
/// - Inheritance: the environment of bwrap itself is empty, the sandbox gets only
/// what the request names. .NET opens its files close-on-exec; the sandbox sees fds 0–2.
/// - Network: own network namespace with nothing but loopback.
/// - Limits: wall-clock timeout (the whole namespace dies with bwrap), address
/// space, CPU time, file size, open files — as rlimits; processes as RLIMIT_NPROC, set
/// inside the sandbox's own user namespace, where it counts the sandbox's processes and not
/// the instance's threads. Output is capped while it is read.
/// - Privileges: all capabilities dropped, no_new_privs (bwrap sets it
/// always), no nested user namespaces where bwrap can refuse them, and the
/// .
///
///
public sealed class BwrapSandbox : ISandboxRunner
{
private const string Shell = "/bin/sh";
private readonly string _bwrap;
private readonly BwrapFeatures _features;
private readonly ILogger _logger;
private BwrapSandbox(string bwrap, BwrapFeatures features, ILogger logger)
{
_bwrap = bwrap;
_features = features;
_logger = logger;
}
public bool IsAvailable => true;
public string? UnavailableReason => null;
///
/// The runner of this machine: bwrap under Linux if it can actually build a sandbox here
/// (tried once — a kernel or AppArmor policy that forbids user namespaces shows only when
/// trying), otherwise with the reason.
///
public static ISandboxRunner Detect(ILogger logger)
{
if (!OperatingSystem.IsLinux())
return new NoSandbox("code runs only under Linux, in a bubblewrap sandbox");
if (!SeccompFilter.Supports(RuntimeInformation.ProcessArchitecture))
return new NoSandbox($"no seccomp filter for {RuntimeInformation.ProcessArchitecture}");
var bwrap = new[] { "/usr/bin/bwrap", "/bin/bwrap", "/usr/local/bin/bwrap" }.FirstOrDefault(File.Exists);
if (bwrap is null)
return new NoSandbox("bubblewrap (bwrap) is not installed — 'apt install bubblewrap'");
var help = RunQuiet(bwrap, ["--help"]);
var probe = RunQuiet(bwrap, ["--unshare-all", "--die-with-parent", "--ro-bind", "/usr", "/usr",
"--symlink", "usr/bin", "/bin", "--symlink", "usr/lib", "/lib", "--proc", "/proc", "--dev", "/dev", "/bin/true"]);
if (probe.ExitCode != 0)
return new NoSandbox($"bwrap cannot create a sandbox here ({probe.Output.Trim()}) — user namespaces may be disabled");
var features = new BwrapFeatures(
DisableUserns: help.Output.Contains("--disable-userns", StringComparison.Ordinal),
ClearEnv: help.Output.Contains("--clearenv", StringComparison.Ordinal));
logger.LogInformation("Sandbox ready: {Bwrap}{Userns}", bwrap,
features.DisableUserns ? "" : " (older bwrap: nested user namespaces are refused by the seccomp filter only)");
return new BwrapSandbox(bwrap, features, logger);
}
public async Task RunAsync(SandboxRequest request, CancellationToken ct)
{
Validate(request);
var filterFile = Path.Combine(Path.GetTempPath(), $"clawddotnet_seccomp_{Guid.NewGuid():N}.bpf");
await File.WriteAllBytesAsync(filterFile, SeccompFilter.Build(RuntimeInformation.ProcessArchitecture), ct);
try
{
var start = new ProcessStartInfo(Shell)
{
UseShellExecute = false,
RedirectStandardInput = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
StandardOutputEncoding = Encoding.UTF8,
StandardErrorEncoding = Encoding.UTF8
};
// Nothing of the main process's environment — not even to bwrap.
start.Environment.Clear();
foreach (var argument in CommandLine(_bwrap, filterFile, request, HostLayout.Read(), _features))
start.ArgumentList.Add(argument);
return await RunProcessAsync(start, request, ct);
}
finally
{
try { File.Delete(filterFile); }
catch (IOException) { /* in /tmp, harmless */ }
}
}
///
/// The whole command line: /bin/sh -c WRAPPER sh FILTER LIMITS… bwrap ARGS… -- sh -c INNER sh NPROC PROGRAM ARGS….
/// Pure — the tests read it on any platform.
///
public static IReadOnlyList CommandLine(
string bwrap, string filterFile, SandboxRequest request, HostLayout host, BwrapFeatures features)
{
var limits = request.Limits;
var args = new List
{
"-c",
// Outer: rlimits that bind the sandbox as a whole, then bwrap with the filter on fd 3.
"""f=$1; ulimit -v "$2" && ulimit -t "$3" && ulimit -f "$4" && ulimit -n "$5" || exit 126; shift 5; exec "$@" 3<"$f" """,
"sh",
filterFile,
Kilobytes(limits.MaxMemoryBytes),
((long)Math.Ceiling(limits.Timeout.TotalSeconds) + 5).ToString(CultureInfo.InvariantCulture),
Blocks(limits.MaxFileBytes),
limits.MaxOpenFiles.ToString(CultureInfo.InvariantCulture),
bwrap,
"--unshare-all",
"--unshare-user",
"--die-with-parent",
"--new-session",
"--cap-drop", "ALL",
"--hostname", "sandbox"
};
// bwrap's own environment is already empty; where it can, it says so once more.
if (features.ClearEnv)
args.Add("--clearenv");
if (features.DisableUserns)
args.Add("--disable-userns");
// The system, read-only. Merged /usr: /bin, /lib … are links into it.
args.AddRange(["--ro-bind", "/usr", "/usr"]);
foreach (var (path, link) in host.TopLevel)
{
if (link is not null)
args.AddRange(["--symlink", link, path]);
else
args.AddRange(["--ro-bind-try", path, path]);
}
foreach (var etc in EtcFiles)
args.AddRange(["--ro-bind-try", etc, etc]);
args.AddRange(["--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", "/run"]);
// The workspace, and inside it what must stay as it was checked.
var workspace = Path.GetFullPath(request.WorkspacePath);
args.AddRange(["--bind", workspace, SandboxPaths.Work]);
foreach (var relative in request.ProtectedPaths)
{
var hostPath = Path.GetFullPath(Path.Combine(workspace, relative));
if (SandboxPaths.InSandbox(workspace, hostPath) is { } inside && inside != SandboxPaths.Work)
args.AddRange(["--ro-bind-try", hostPath, inside]);
}
foreach (var path in request.ReadOnlyPaths)
args.AddRange(["--ro-bind", Path.GetFullPath(path), Path.GetFullPath(path)]);
args.AddRange(["--chdir", SandboxPaths.Work]);
var environment = new Dictionary(StringComparer.Ordinal)
{
["PATH"] = "/usr/local/bin:/usr/bin:/bin",
["HOME"] = "/tmp",
["LANG"] = "C.UTF-8",
["TMPDIR"] = "/tmp"
};
foreach (var (key, value) in request.Environment)
environment[key] = value;
foreach (var (key, value) in environment.OrderBy(e => e.Key, StringComparer.Ordinal))
args.AddRange(["--setenv", key, value]);
args.AddRange(["--seccomp", "3", "--"]);
// Inner: the process limit, set inside the sandbox's own user namespace.
args.AddRange([Shell, "-c", """ulimit -u "$1" || exit 126; shift; exec "$@" """, "sh",
limits.MaxProcesses.ToString(CultureInfo.InvariantCulture), request.Executable]);
args.AddRange(request.Arguments);
return args;
}
/// What a program commonly needs of /etc — and nothing that says who uses this machine.
private static readonly string[] EtcFiles =
[
"/etc/ld.so.cache", "/etc/ld.so.conf", "/etc/ld.so.conf.d", "/etc/alternatives",
"/etc/localtime", "/etc/ssl/certs", "/etc/ca-certificates", "/etc/fonts", "/etc/mime.types"
];
/// The workspace must exist and must not be a place that contains everything else.
private static void Validate(SandboxRequest request)
{
var workspace = Path.GetFullPath(request.WorkspacePath).TrimEnd('/');
if (!Directory.Exists(workspace))
throw new ArgumentException($"The workspace '{workspace}' does not exist.");
var home = System.Environment.GetFolderPath(System.Environment.SpecialFolder.UserProfile).TrimEnd('/');
if (workspace.Length == 0 || workspace == home || workspace.Count(c => c == '/') < 2)
throw new ArgumentException($"'{workspace}' is not a workspace — it would expose far more than one agent's files.");
if (string.IsNullOrWhiteSpace(request.Executable))
throw new ArgumentException("No program given.");
}
private async Task RunProcessAsync(ProcessStartInfo start, SandboxRequest request, CancellationToken ct)
{
using var process = Process.Start(start) ?? throw new InvalidOperationException("The sandbox did not start.");
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
timeout.CancelAfter(request.Limits.Timeout);
var output = new CappedOutput(request.Limits.MaxOutputBytes);
var stdout = PumpAsync(process.StandardOutput, output, isError: false, process);
var stderr = PumpAsync(process.StandardError, output, isError: true, process);
try
{
if (request.StandardInput is { } input)
await process.StandardInput.WriteAsync(input.AsMemory(), timeout.Token);
process.StandardInput.Close();
}
catch (IOException)
{
// The program did not read its input — that is its business.
}
var timedOut = false;
try
{
await process.WaitForExitAsync(timeout.Token);
}
catch (OperationCanceledException)
{
timedOut = !ct.IsCancellationRequested;
Kill(process);
if (!timedOut)
throw;
}
await Task.WhenAll(stdout, stderr);
if (timedOut)
_logger.LogWarning("Sandbox run of {Program} stopped after {Timeout}", request.Executable, request.Limits.Timeout);
return new SandboxResult
{
ExitCode = process.HasExited ? process.ExitCode : -1,
StandardOutput = output.Out,
StandardError = output.Err,
TimedOut = timedOut,
OutputTruncated = output.Truncated
};
}
/// Reads one stream into the shared cap; past the cap the run is stopped.
private static async Task PumpAsync(StreamReader reader, CappedOutput output, bool isError, Process process)
{
var buffer = new char[8192];
int read;
while ((read = await reader.ReadAsync(buffer.AsMemory())) > 0)
{
if (!output.Append(buffer.AsSpan(0, read), isError))
{
Kill(process);
return;
}
}
}
/// bwrap dies, and with it — die-with-parent, own PID namespace — everything inside.
private static void Kill(Process process)
{
try
{
if (!process.HasExited)
process.Kill(entireProcessTree: true);
}
catch (InvalidOperationException) { }
catch (System.ComponentModel.Win32Exception) { }
}
private static string Kilobytes(long bytes) => Math.Max(1, bytes / 1024).ToString(CultureInfo.InvariantCulture);
/// ulimit -f counts 512-byte blocks in POSIX sh.
private static string Blocks(long bytes) => Math.Max(1, bytes / 512).ToString(CultureInfo.InvariantCulture);
private static (int ExitCode, string Output) RunQuiet(string program, IReadOnlyList arguments)
{
try
{
var start = new ProcessStartInfo(program)
{
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true
};
start.Environment.Clear();
foreach (var argument in arguments)
start.ArgumentList.Add(argument);
using var process = Process.Start(start)!;
var text = process.StandardOutput.ReadToEnd() + process.StandardError.ReadToEnd();
return process.WaitForExit(5000) ? (process.ExitCode, text) : (-1, "timed out");
}
catch (Exception ex) when (ex is System.ComponentModel.Win32Exception or IOException)
{
return (-1, ex.Message);
}
}
private sealed class CappedOutput(int max)
{
private readonly StringBuilder _out = new();
private readonly StringBuilder _err = new();
private int _total;
public bool Truncated { get; private set; }
public string Out { get { lock (this) return _out.ToString(); } }
public string Err { get { lock (this) return _err.ToString(); } }
public bool Append(ReadOnlySpan text, bool isError)
{
lock (this)
{
var room = max - _total;
var take = Math.Min(room, text.Length);
(isError ? _err : _out).Append(text[..Math.Max(0, take)]);
_total += Math.Max(0, take);
if (take < text.Length)
Truncated = true;
return !Truncated;
}
}
}
}
/// #654: What the installed bwrap can do beyond the basics.
public sealed record BwrapFeatures(bool DisableUserns, bool ClearEnv)
{
public static BwrapFeatures All { get; } = new(true, true);
}
///
/// #654: How the top level of this machine looks — which of /bin, /lib … are
/// links into /usr (merged /usr) and which are directories. Read once per run; a
/// record, so the tests can describe any machine.
///
public sealed record HostLayout(IReadOnlyList<(string Path, string? Link)> TopLevel)
{
private static readonly string[] Candidates = ["/bin", "/sbin", "/lib", "/lib64", "/lib32", "/libx32"];
public static HostLayout Read()
{
var entries = new List<(string, string?)>();
foreach (var path in Candidates)
{
var info = new FileInfo(path);
if (info.LinkTarget is { } target)
entries.Add((path, target.TrimStart('/')));
else if (Directory.Exists(path))
entries.Add((path, null));
}
return new HostLayout(entries);
}
/// A merged-/usr system, as Debian and Ubuntu are today.
public static HostLayout MergedUsr { get; } = new([("/bin", "usr/bin"), ("/sbin", "usr/sbin"), ("/lib", "usr/lib"), ("/lib64", "usr/lib64")]);
}