More Agents, Smaller Gains: What Agensh, MAST and the Google Study Say About Your Orchestrator

PANDOC, MARGINAL GAIN PER ADDED AGENT (approximate, from the paper's curve)
---------------------------------------------------------------------------
1 -> 128 agents ~17 points / 127 agents ~0.134 points per agent
128 -> 1,024 agents ~4 points / 896 agents ~0.004 points per agent
---------------------------------------------------------------------------
The last 896 agents are worth roughly 30x less each than the first 127.

+--------------------+-------------+--------------------------+------------------------------+
| SOURCE / TASK | AGENTS | PASS RATE / CHANGE | COST REPORTED? |
+--------------------+-------------+--------------------------+------------------------------+
| Agensh ProgramBench| 1 | 19.31% | No |
| | 128 | 28.78% (+9.5 pts) | No |
+--------------------+-------------+--------------------------+------------------------------+
| Agensh Pandoc | 1 | 33.89% | No |
| | ~128 | ~51% (+17 pts, approx.) | No |
| | 1,024 | 55.06% (+21 pts) | No |
+--------------------+-------------+--------------------------+------------------------------+
| Google Finance- | 1 vs | +80.9% relative for | Token budget reportedly |
| Agent | centralized | centralized | matched across architectures |
+--------------------+-------------+--------------------------+------------------------------+
| Google PlanCraft | 1 vs any | -39% to -70% relative | Same as above |
| | multi-agent | | |
+--------------------+-------------+--------------------------+------------------------------+
| MAST (per | varies | 41% to 86.7% FAILURE, | No, and different benchmark |
| framework) | | each on its own benchmark| per framework |
+--------------------+-------------+--------------------------+------------------------------+

STRATEGY AGENTS CALLS PER TASK WHAT IT DOES
------------------------------------------------------------------------
single 1 1 Solve, return JSON
single+check 1 2 Solve, then recompute and fix
planner+2 3 4 Plan into 2 subtasks, 2 workers
                                       solve, planner synthesizes

# Option A: native install (macOS: brew install ollama, Linux: see ollama.com)
ollama serve &
# Let the two workers actually run at the same time
export OLLAMA_NUM_PARALLEL=2

ollama pull qwen2.5:7b
ollama pull llama3.2:3b

# Option B: Docker
docker run -d --name ollama -p 11434:11434 \
  -e OLLAMA_NUM_PARALLEL=2 \
  -v ollama:/root/.ollama ollama/ollama
docker exec ollama ollama pull qwen2.5:7b
docker exec ollama ollama pull llama3.2:3b

dotnet new console -n AgentScaleBench
cd AgentScaleBench
# replace Program.cs with the file below, then:
dotnet run -c Release -- --model qwen2.5:7b --runs 3 --tasks 20
dotnet run -c Release -- --model llama3.2:3b --runs 3 --tasks 20

# optional: illustrative pricing per million tokens (input, output)
dotnet run -c Release -- --model qwen2.5:7b --runs 3 --price-in 0.15 --price-out 0.60

// AgentScaleBench: one agent vs. one agent + self-check vs. planner + two workers,
// against a local Ollama model, on a small set of tasks with exact-match checkers.
// No NuGet packages. .NET 8+.
//
// Usage:
// dotnet run -c Release -- --model qwen2.5:7b --runs 3
// dotnet run -c Release -- --model llama3.2:3b --runs 3 --tasks 20 --price-in 0.15 --price-out 0.60
//
// Writes results.csv (one row per attempt) and prints an ASCII summary table.

using System.Diagnostics;
using System.Globalization;
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
var cfg = Config.Parse(args);
var ollama = new Ollama(cfg);
var tasks = TaskSet.Generate(cfg.TaskCount, seed: 42);
Console.WriteLine($"Model: {cfg.Model} | tasks: {tasks.Count} | runs: {cfg.Runs} | endpoint: {cfg.Endpoint}");
var strategies = new IStrategy[] { new SingleAgent(), new SingleWithSelfCheck(), new PlannerTwoWorkers() };
var attempts = new List<Attempt>();
foreach (var strategy in strategies)
{
    for (var run = 0; run < cfg.Runs; run++)
    {
        foreach (var task in tasks)
        {
            var ledger = new Ledger();
            var sw = Stopwatch.StartNew();
            string? final = null;
            string? error = null;
            try { final = await strategy.SolveAsync(task, ollama, ledger, cfg.Seed + run); }
            catch (Exception ex) { error = ex.GetType().Name + ": " + ex.Message; }
            sw.Stop();
            var correct = final is not null && Checker.IsCorrect(task, final);
            attempts.Add(new Attempt(strategy.Name, strategy.Agents, task.Id, task.Kind, run, correct,
                ledger.Calls, ledger.PromptTokens, ledger.OutputTokens, sw.Elapsed.TotalSeconds, error));
            Console.Write(correct ? "." : "x");
        }
    }
    Console.WriteLine($" {strategy.Name}");
}
Report.WriteCsv(attempts, "results.csv");
Report.PrintTable(attempts, cfg);
// ---------------------------------------------------------------------------
// Strategies
// ---------------------------------------------------------------------------
interface IStrategy
{
    string Name { get; }
    int Agents { get; }
    Task<string?> SolveAsync(EvalTask task, Ollama llm, Ledger ledger, int seed);
}
sealed class SingleAgent : IStrategy
{
    public string Name => "single";
    public int Agents => 1;
    public Task<string?> SolveAsync(EvalTask task, Ollama llm, Ledger ledger, int seed) =>
        llm.ChatJsonAsync(Prompts.Solver, task.Prompt, ledger, seed)!;
}
// Same agent, second pass. This is the "spend the extra budget on one agent" baseline.
sealed class SingleWithSelfCheck : IStrategy
{
    public string Name => "single+check";
    public int Agents => 1;
    public async Task<string?> SolveAsync(EvalTask task, Ollama llm, Ledger ledger, int seed)
    {
        var draft = await llm.ChatJsonAsync(Prompts.Solver, task.Prompt, ledger, seed);
        var review = $"TASK:\n{task.Prompt}\n\nPROPOSED ANSWER:\n{draft}\n\n" +
                     "Recompute every step yourself. If the proposed answer is wrong, fix it. " +
                     "Return only the final JSON in the exact format the task asks for.";
        return await llm.ChatJsonAsync(Prompts.Solver, review, ledger, seed + 1);
    }
}
// Planner splits into exactly two subtasks, two workers solve them,
// planner synthesizes. Sequential plans pass worker 1's output to worker 2.
sealed class PlannerTwoWorkers : IStrategy
{
    public string Name => "planner+2";
    public int Agents => 3;
    public async Task<string?> SolveAsync(EvalTask task, Ollama llm, Ledger ledger, int seed)
    {
        var planJson = await llm.ChatJsonAsync(Prompts.Planner, task.Prompt, ledger, seed);
        var plan = JsonNode.Parse(planJson ?? "{}");
        var mode = plan?["mode"]?.GetValue<string>() ?? "parallel";
        var subtasks = plan?["subtasks"]?.AsArray().Select(n => n?.ToString() ?? "").ToList() ?? new();
        while (subtasks.Count < 2) subtasks.Add("Solve the whole task: " + task.Prompt);
        string? r1, r2;
        if (mode == "sequential")
        {
            r1 = await llm.ChatJsonAsync(Prompts.Worker, subtasks[0], ledger, seed + 1);
            r2 = await llm.ChatJsonAsync(Prompts.Worker,
                $"{subtasks[1]}\n\nRESULT OF THE PREVIOUS STEP:\n{r1}", ledger, seed + 2);
        }
        else
        {
            var t1 = llm.ChatJsonAsync(Prompts.Worker, subtasks[0], ledger, seed + 1);
            var t2 = llm.ChatJsonAsync(Prompts.Worker, subtasks[1], ledger, seed + 2);
            await Task.WhenAll(t1, t2);
            (r1, r2) = (t1.Result, t2.Result);
        }
        var synth = $"ORIGINAL TASK:\n{task.Prompt}\n\nWORKER 1 ({subtasks[0]}):\n{r1}\n\n" +
                    $"WORKER 2 ({subtasks[1]}):\n{r2}\n\n" +
                    "Combine the worker results into the final answer. " +
                    "Return only the final JSON in the exact format the original task asks for.";
        return await llm.ChatJsonAsync(Prompts.Planner, synth, ledger, seed + 3);
    }
}
static class Prompts
{
    public const string Solver =
        "You solve small exact tasks. Think carefully, then reply with JSON only, in the exact format requested.";
    public const string Planner =
        "You are a planner. When given a task, split it into exactly two subtasks for two workers. " +
        "Reply with JSON only: {\"mode\": \"parallel\" | \"sequential\", \"subtasks\": [\"...\", \"...\"]}. " +
        "Use \"sequential\" if the second subtask needs the first one's result. " +
        "Each subtask must be self-contained: copy every number and word it needs. " +
        "When asked to combine worker results, reply with the final JSON the original task asks for.";
    public const string Worker =
        "You are a worker. Solve exactly the subtask you are given. " +
        "Reply with JSON only: {\"result\": <your result>}.";
}
// ---------------------------------------------------------------------------
// Ollama client with token accounting
// ---------------------------------------------------------------------------
sealed class Ledger
{
    private int _calls, _prompt, _output;
    public int Calls => _calls;
    public int PromptTokens => _prompt;
    public int OutputTokens => _output;
    public void Add(int prompt, int output)
    {
        Interlocked.Increment(ref _calls);
        Interlocked.Add(ref _prompt, prompt);
        Interlocked.Add(ref _output, output);
    }
}
sealed class Ollama(Config cfg)
{
    private readonly HttpClient _http = new() { BaseAddress = new Uri(cfg.Endpoint), Timeout = TimeSpan.FromMinutes(5) };
    public async Task<string?> ChatJsonAsync(string system, string user, Ledger ledger, int seed)
    {
        var body = new
        {
            model = cfg.Model,
            stream = false,
            format = "json",
            messages = new[]
            {
                new { role = "system", content = system },
                new { role = "user", content = user }
            },
            options = new { temperature = cfg.Temperature, seed, num_ctx = 4096 }
        };
        var payload = JsonSerializer.Serialize(body);
        HttpResponseMessage resp;
        try
        {
            resp = await _http.PostAsync("/api/chat", new StringContent(payload, Encoding.UTF8, "application/json"));
        }
        catch (HttpRequestException)
        {
            // One retry for dropped keep-alive connections. Not counted as an extra model call.
            await Task.Delay(500);
            resp = await _http.PostAsync("/api/chat", new StringContent(payload, Encoding.UTF8, "application/json"));
        }
        using var _ = resp;
        resp.EnsureSuccessStatusCode();
        var json = await resp.Content.ReadFromJsonAsync<JsonElement>();
        var prompt = json.TryGetProperty("prompt_eval_count", out var p) ? p.GetInt32() : 0;
        var output = json.TryGetProperty("eval_count", out var o) ? o.GetInt32() : 0;
        ledger.Add(prompt, output);
        return json.GetProperty("message").GetProperty("content").GetString();
    }
}
// ---------------------------------------------------------------------------
// Tasks and checking
// ---------------------------------------------------------------------------
sealed record EvalTask(string Id, string Kind, string Prompt, Dictionary<string, string> Expected);
static class TaskSet
{
    // Two shapes on purpose:
    // "split": two independent parts, the case where decomposition should help.
    // "chain": every step depends on the previous one, the case where it should hurt.
    public static List<EvalTask> Generate(int count, int seed)
    {
        var rng = new Random(seed);
        var list = new List<EvalTask>();
        for (var i = 0; i < count; i++)
            list.Add(i % 2 == 0 ? Split(rng, i) : Chain(rng, i));
        return list;
    }
    private static EvalTask Split(Random rng, int i)
    {
        int a = rng.Next(12, 99), b = rng.Next(12, 99);
        var nums = Enumerable.Range(0, 6).Select(_ => rng.Next(1, 200)).Distinct().ToList();
        var sorted = string.Join(",", nums.OrderBy(n => n));
        var prompt =
            $"Answer two independent questions.\n" +
            $"Part 1: What is {a} * {b}?\n" +
            $"Part 2: Sort these numbers in ascending order: {string.Join(", ", nums)}.\n" +
            "Reply with JSON: {\"p1\": <integer>, \"p2\": \"<comma-separated numbers, no spaces>\"}";
        return new EvalTask($"split-{i:00}", "split", prompt,
            new() { ["p1"] = (a * b).ToString(), ["p2"] = sorted });
    }
    private static EvalTask Chain(Random rng, int i)
    {
        var x = rng.Next(5, 40);
        var start = x;
        var steps = new List<string>();
        for (var s = 0; s < 5; s++)
        {
            switch (rng.Next(3))
            {
                case 0: { var k = rng.Next(3, 30); x += k; steps.Add($"add {k}"); break; }
                case 1: { var k = rng.Next(2, 15); x -= k; steps.Add($"subtract {k}"); break; }
                default: { var k = rng.Next(2, 4); x *= k; steps.Add($"multiply by {k}"); break; }
            }
        }
        var prompt =
            $"Start with {start}. Then, in order: {string.Join(", then ", steps)}.\n" +
            "Reply with JSON: {\"answer\": <integer>}";
        return new EvalTask($"chain-{i:00}", "chain", prompt, new() { ["answer"] = x.ToString() });
    }
}
static class Checker
{
    public static bool IsCorrect(EvalTask task, string modelJson)
    {
        try
        {
            var node = JsonNode.Parse(modelJson)?.AsObject();
            if (node is null) return false;
            foreach (var (key, expected) in task.Expected)
            {
                if (!node.TryGetPropertyValue(key, out var v) || v is null) return false;
                if (Normalize(v.ToString()) != Normalize(expected)) return false;
            }
            return true;
        }
        catch { return false; }
    }
    private static string Normalize(string s)
    {
        s = s.Trim().Trim('"').Replace(" ", "").Replace("[", "").Replace("]", "");
        return double.TryParse(s, NumberStyles.Float, CultureInfo.InvariantCulture, out var d) && !s.Contains(',')
            ? d.ToString(CultureInfo.InvariantCulture)
            : s;
    }
}
// ---------------------------------------------------------------------------
// Reporting
// ---------------------------------------------------------------------------
sealed record Attempt(string Strategy, int Agents, string TaskId, string Kind, int Run, bool Correct,
    int Calls, int PromptTokens, int OutputTokens, double Seconds, string? Error);
static class Report
{
    public static void WriteCsv(List<Attempt> attempts, string path)
    {
        var sb = new StringBuilder("strategy,agents,task,kind,run,correct,calls,prompt_tokens,output_tokens,seconds,error\n");
        foreach (var a in attempts)
            sb.AppendLine(string.Join(",", a.Strategy, a.Agents, a.TaskId, a.Kind, a.Run, a.Correct, a.Calls,
                a.PromptTokens, a.OutputTokens, a.Seconds.ToString("F2", CultureInfo.InvariantCulture),
                "\"" + (a.Error ?? "").Replace("\"", "'") + "\""));
        File.WriteAllText(path, sb.ToString());
    }
    public static void PrintTable(List<Attempt> attempts, Config cfg)
    {
        var pricing = cfg.PriceInPerM > 0 || cfg.PriceOutPerM > 0;
        var rows = new List<string[]>();
        var header = new List<string> { "STRATEGY", "AGENTS", "SLICE", "PASS", "PASS %", "CALLS/TASK", "TOKENS/TASK", "TOKENS/CORRECT", "SEC/CORRECT" };
        if (pricing) header.Add("$/CORRECT (illustr.)");
        foreach (var group in attempts.GroupBy(a => a.Strategy))
        {
            foreach (var slice in new[] { "all", "split", "chain" })
            {
                var g = group.Where(a => slice == "all" || a.Kind == slice).ToList();
                if (g.Count == 0) continue;
                var correct = g.Count(a => a.Correct);
                var tokens = g.Sum(a => (long)a.PromptTokens + a.OutputTokens);
                var seconds = g.Sum(a => a.Seconds);
                var row = new List<string>
                {
                    group.Key, g[0].Agents.ToString(), slice,
                    $"{correct}/{g.Count}",
                    (100.0 * correct / g.Count).ToString("F1", CultureInfo.InvariantCulture),
                    (g.Average(a => a.Calls)).ToString("F1", CultureInfo.InvariantCulture),
                    (tokens / (double)g.Count).ToString("F0", CultureInfo.InvariantCulture),
                    correct == 0 ? "n/a" : (tokens / (double)correct).ToString("F0", CultureInfo.InvariantCulture),
                    correct == 0 ? "n/a" : (seconds / correct).ToString("F1", CultureInfo.InvariantCulture)
                };
                if (pricing)
                {
                    var dollars = g.Sum(a => a.PromptTokens * cfg.PriceInPerM / 1e6 + a.OutputTokens * cfg.PriceOutPerM / 1e6);
                    row.Add(correct == 0 ? "n/a" : (dollars / correct).ToString("F6", CultureInfo.InvariantCulture));
                }
                rows.Add(row.ToArray());
            }
        }
        var all = new List<string[]> { header.ToArray() };
        all.AddRange(rows);
        var widths = Enumerable.Range(0, header.Count).Select(c => all.Max(r => r[c].Length)).ToArray();
        string Line() => "+" + string.Join("+", widths.Select(w => new string('-', w + 2))) + "+";
        string Fmt(string[] r) => "| " + string.Join(" | ", r.Select((v, c) => v.PadRight(widths[c]))) + " |";
        Console.WriteLine();
        Console.WriteLine(Line());
        Console.WriteLine(Fmt(header.ToArray()));
        Console.WriteLine(Line());
        string? last = null;
        foreach (var r in rows)
        {
            if (last is not null && r[0] != last) Console.WriteLine(Line());
            Console.WriteLine(Fmt(r));
            last = r[0];
        }
        Console.WriteLine(Line());
        var errors = attempts.Count(a => a.Error is not null);
        if (errors > 0) Console.WriteLine($"{errors} attempts threw errors (counted as failures). See results.csv.");
    }
}
sealed record Config(string Endpoint, string Model, int Runs, int TaskCount, double Temperature, int Seed,
    double PriceInPerM, double PriceOutPerM)
{
    public static Config Parse(string[] args)
    {
        string Get(string name, string fallback)
        {
            var i = Array.IndexOf(args, name);
            return i >= 0 && i + 1 < args.Length ? args[i + 1] : fallback;
        }
        double D(string name, string fb) => double.Parse(Get(name, fb), CultureInfo.InvariantCulture);
        return new Config(
            Get("--endpoint", Environment.GetEnvironmentVariable("OLLAMA_URL") ?? "http://localhost:11434"),
            Get("--model", "qwen2.5:7b"),
            int.Parse(Get("--runs", "3")),
            int.Parse(Get("--tasks", "20")),
            D("--temperature", "0.2"),
            int.Parse(Get("--seed", "7")),
            D("--price-in", "0"),
            D("--price-out", "0"));
    }
}