Tool Enumeration Benchmark for MCP Servers - Parallel vs Sequential comparison

Program.cs

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Diagnostics;
using Microsoft.Agents.A365.Tooling.Models;
using Microsoft.Agents.A365.Tooling.Services;
using Microsoft.Agents.Builder;
using Microsoft.Agents.Core.Models;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Moq;

namespace ToolEnumerationBenchmark;

/// <summary>
/// Performance benchmark for MCP tool enumeration parallelization.
/// </summary>
public static class Program
{
    private const int DefaultIterations = 5;

    public static async Task Main(string[] args)
    {
        Console.WriteLine("Tool Enumeration Performance Benchmark");
        Console.WriteLine("======================================");
        Console.WriteLine();

        // Parse command line arguments
        var iterations = args.Length > 0 && int.TryParse(args[0], out var n) ? n : DefaultIterations;

        // Prompt for authentication token
        Console.Write("Enter authentication token: ");
        var authToken = Console.ReadLine()?.Trim();

        if (string.IsNullOrWhiteSpace(authToken))
        {
            Console.WriteLine("Error: Authentication token is required.");
            Environment.Exit(1);
            return;
        }

        Console.WriteLine();

        // Setup DI and services
        var services = ConfigureServices();
        var serviceProvider = services.BuildServiceProvider();

        var configService = serviceProvider.GetRequiredService<IMcpToolServerConfigurationService>();

        // Create mock turn context using Moq
        var turnContext = CreateMockTurnContext();
        var toolOptions = new ToolOptions();
        var agentInstanceId = "benchmark-agent";

        // List servers first to show what we're working with
        Console.WriteLine("Loading servers from ToolingManifest.json...");
        try
        {
            var servers = await configService.ListToolServersAsync(agentInstanceId, authToken, toolOptions);
            Console.WriteLine($"Found {servers.Count} MCP servers.");
            Console.WriteLine();
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error listing servers: {ex.Message}");
            Environment.Exit(1);
            return;
        }

        // Warm-up runs for both modes
        Console.WriteLine("Warm-up runs...");

        var parallelOptions = new ToolOptions { EnumerationMode = ToolEnumerationMode.Parallel };
        var sequentialOptions = new ToolOptions { EnumerationMode = ToolEnumerationMode.Sequential };

        var warmupParallel = await RunBenchmarkIteration(configService, agentInstanceId, authToken, turnContext, parallelOptions);
        Console.WriteLine($"  Parallel:   {warmupParallel.ElapsedMs}ms ({warmupParallel.TotalTools} tools from {warmupParallel.ServersWithTools}/{warmupParallel.TotalServers} servers)");

        var warmupSequential = await RunBenchmarkIteration(configService, agentInstanceId, authToken, turnContext, sequentialOptions);
        Console.WriteLine($"  Sequential: {warmupSequential.ElapsedMs}ms ({warmupSequential.TotalTools} tools from {warmupSequential.ServersWithTools}/{warmupSequential.TotalServers} servers)");
        Console.WriteLine();

        // Benchmark runs - Parallel mode
        Console.WriteLine($"Running {iterations} PARALLEL benchmark iterations...");
        var parallelResults = new List<BenchmarkResult>();

        for (int i = 1; i <= iterations; i++)
        {
            var result = await RunBenchmarkIteration(configService, agentInstanceId, authToken, turnContext, parallelOptions);
            parallelResults.Add(result);
            Console.WriteLine($"  Run {i}: {result.ElapsedMs}ms - {result.TotalTools} tools from {result.ServersWithTools} servers");
        }

        Console.WriteLine();

        // Benchmark runs - Sequential mode
        Console.WriteLine($"Running {iterations} SEQUENTIAL benchmark iterations...");
        var sequentialResults = new List<BenchmarkResult>();

        for (int i = 1; i <= iterations; i++)
        {
            var result = await RunBenchmarkIteration(configService, agentInstanceId, authToken, turnContext, sequentialOptions);
            sequentialResults.Add(result);
            Console.WriteLine($"  Run {i}: {result.ElapsedMs}ms - {result.TotalTools} tools from {result.ServersWithTools} servers");
        }

        // Print comparison statistics
        PrintComparisonStatistics(parallelResults, sequentialResults);
    }

    private static ServiceCollection ConfigureServices()
    {
        var services = new ServiceCollection();

        // Set environment to Development to trigger manifest loading
        Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Development");

        // Configuration
        var configuration = new ConfigurationBuilder()
            .AddEnvironmentVariables()
            .AddInMemoryCollection(new Dictionary<string, string?>
            {
                ["DOTNET_ENVIRONMENT"] = "Development"
            })
            .Build();

        services.AddSingleton<IConfiguration>(configuration);

        // Logging
        services.AddLogging(builder =>
        {
            builder.AddConsole();
            builder.SetMinimumLevel(LogLevel.Warning); // Reduce noise during benchmark
        });

        // HTTP client factory
        services.AddHttpClient();

        // Register the MCP tool server configuration service
        services.AddSingleton<IMcpToolServerConfigurationService, McpToolServerConfigurationService>();

        return services;
    }

    private static ITurnContext CreateMockTurnContext()
    {
        var mockTurnContext = new Mock<ITurnContext>();

        // Setup StackState - required by HttpContextHeadersHandler
        mockTurnContext.Setup(tc => tc.StackState).Returns(new TurnContextStateCollection());

        // Setup Activity with minimal structure needed by the handler
        var mockActivity = new Mock<IActivity>();
        mockActivity.Setup(a => a.Conversation).Returns(new ConversationAccount { Id = "benchmark-conversation-id" });
        mockActivity.Setup(a => a.ChannelId).Returns(new ChannelId("benchmark"));
        mockActivity.Setup(a => a.Text).Returns("benchmark request");

        mockTurnContext.Setup(tc => tc.Activity).Returns(mockActivity.Object);

        return mockTurnContext.Object;
    }

    private static async Task<BenchmarkResult> RunBenchmarkIteration(
        IMcpToolServerConfigurationService configService,
        string agentInstanceId,
        string authToken,
        ITurnContext turnContext,
        ToolOptions toolOptions)
    {
        var stopwatch = Stopwatch.StartNew();

        var (servers, toolsByServer) = await configService.EnumerateToolsFromServersAsync(
            agentInstanceId,
            authToken,
            turnContext,
            toolOptions);

        stopwatch.Stop();

        var totalTools = toolsByServer.Values.Sum(tools => tools.Count);

        return new BenchmarkResult
        {
            ElapsedMs = stopwatch.ElapsedMilliseconds,
            TotalServers = servers.Count,
            ServersWithTools = toolsByServer.Count,
            TotalTools = totalTools
        };
    }

    private static void PrintComparisonStatistics(List<BenchmarkResult> parallelResults, List<BenchmarkResult> sequentialResults)
    {
        Console.WriteLine();
        Console.WriteLine("==============================================");
        Console.WriteLine("         COMPARISON RESULTS SUMMARY           ");
        Console.WriteLine("==============================================");
        Console.WriteLine();

        var parallelStats = CalculateStats(parallelResults);
        var sequentialStats = CalculateStats(sequentialResults);

        // Side-by-side comparison
        Console.WriteLine("                    PARALLEL     SEQUENTIAL");
        Console.WriteLine("----------------------------------------------");
        Console.WriteLine($"Average time:       {parallelStats.Average,8:F0}ms    {sequentialStats.Average,8:F0}ms");
        Console.WriteLine($"Min time:           {parallelStats.Min,8}ms    {sequentialStats.Min,8}ms");
        Console.WriteLine($"Max time:           {parallelStats.Max,8}ms    {sequentialStats.Max,8}ms");
        Console.WriteLine($"Std Dev:            {parallelStats.StdDev,8:F0}ms    {sequentialStats.StdDev,8:F0}ms");

        if (parallelResults.Count > 0)
        {
            var lastResult = parallelResults.Last();
            Console.WriteLine($"Servers responding: {lastResult.ServersWithTools,8}/{lastResult.TotalServers}      {lastResult.ServersWithTools,8}/{lastResult.TotalServers}");
            Console.WriteLine($"Total tools:        {lastResult.TotalTools,8}        {lastResult.TotalTools,8}");
        }

        Console.WriteLine();
        Console.WriteLine("----------------------------------------------");
        Console.WriteLine("               PERFORMANCE ANALYSIS           ");
        Console.WriteLine("----------------------------------------------");

        if (parallelStats.Average > 0 && sequentialStats.Average > 0)
        {
            var speedupFactor = sequentialStats.Average / parallelStats.Average;
            var timeSaved = sequentialStats.Average - parallelStats.Average;
            var percentageFaster = ((sequentialStats.Average - parallelStats.Average) / sequentialStats.Average) * 100;

            Console.WriteLine($"Speedup factor:     {speedupFactor:F2}x");
            Console.WriteLine($"Time saved:         {timeSaved:F0}ms per enumeration");
            Console.WriteLine($"Parallel is:        {percentageFaster:F1}% faster than sequential");

            Console.WriteLine();

            if (speedupFactor > 1)
            {
                Console.WriteLine($"Conclusion: Parallel mode is {speedupFactor:F2}x faster than sequential mode.");
            }
            else if (speedupFactor < 1)
            {
                Console.WriteLine($"Conclusion: Sequential mode is {1/speedupFactor:F2}x faster than parallel mode.");
            }
            else
            {
                Console.WriteLine("Conclusion: Both modes perform similarly.");
            }
        }

        Console.WriteLine();
    }

    private static (double Average, long Min, long Max, double StdDev) CalculateStats(List<BenchmarkResult> results)
    {
        var times = results.Select(r => r.ElapsedMs).ToList();
        var avgTime = times.Average();
        var minTime = times.Min();
        var maxTime = times.Max();

        var variance = times.Select(t => Math.Pow(t - avgTime, 2)).Average();
        var stdDev = Math.Sqrt(variance);

        return (avgTime, minTime, maxTime, stdDev);
    }

    private sealed record BenchmarkResult
    {
        public long ElapsedMs { get; init; }
        public int TotalServers { get; init; }
        public int ServersWithTools { get; init; }
        public int TotalTools { get; init; }
    }
}

tool-enumeration-mode.patch

diff --git a/src/Tooling/Core/Models/ToolEnumerationMode.cs b/src/Tooling/Core/Models/ToolEnumerationMode.cs
new file mode 100644
index 0000000..f731073
--- /dev/null
+++ b/src/Tooling/Core/Models/ToolEnumerationMode.cs
@@ -0,0 +1,23 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace Microsoft.Agents.A365.Tooling.Models
+{
+    /// <summary>
+    /// Specifies the execution mode for tool enumeration from MCP servers.
+    /// </summary>
+    public enum ToolEnumerationMode
+    {
+        /// <summary>
+        /// Enumerates tools from all servers in parallel using Task.WhenAll.
+        /// This is the default mode and provides better performance when multiple servers are configured.
+        /// </summary>
+        Parallel,
+
+        /// <summary>
+        /// Enumerates tools from servers sequentially using a foreach loop with await.
+        /// Use this mode when you need predictable ordering or want to reduce concurrent connections.
+        /// </summary>
+        Sequential
+    }
+}
diff --git a/src/Tooling/Core/Models/ToolOptions.cs b/src/Tooling/Core/Models/ToolOptions.cs
index 3a63f21..ec1e3b6 100644
--- a/src/Tooling/Core/Models/ToolOptions.cs
+++ b/src/Tooling/Core/Models/ToolOptions.cs
@@ -14,5 +14,11 @@ namespace Microsoft.Agents.A365.Tooling.Models
         /// Gets or sets the user agent configuration for this orchestrator.
         /// </summary>
         public IUserAgentConfiguration? UserAgentConfiguration { get; set; }
+
+        /// <summary>
+        /// Gets or sets the execution mode for tool enumeration from MCP servers.
+        /// Defaults to <see cref="ToolEnumerationMode.Parallel"/>.
+        /// </summary>
+        public ToolEnumerationMode EnumerationMode { get; set; } = ToolEnumerationMode.Parallel;
     }
 }
diff --git a/src/Tooling/Core/Services/McpToolServerConfigurationService.ToolEnumeration.cs b/src/Tooling/Core/Services/McpToolServerConfigurationService.ToolEnumeration.cs
index 57c94d9..cd2acb7 100644
--- a/src/Tooling/Core/Services/McpToolServerConfigurationService.ToolEnumeration.cs
+++ b/src/Tooling/Core/Services/McpToolServerConfigurationService.ToolEnumeration.cs
@@ -59,43 +59,78 @@ namespace Microsoft.Agents.A365.Tooling.Services
                 return true;
             }).ToList();
 
-            // Enumerate tools from all servers in parallel
-            var tasks = validServers.Select(async server =>
+            // Enumerate tools from servers using the configured mode
+            if (toolOptions.EnumerationMode == ToolEnumerationMode.Sequential)
             {
-                try
+                // Sequential mode: enumerate servers one at a time
+                foreach (var server in validServers)
                 {
-                    var mcpTools = await GetMcpClientToolsAsync(
-                        turnContext,
-                        server,
-                        authToken,
-                        toolOptions).ConfigureAwait(false);
-
-                    _logger.LogInformation(
-                        "Successfully loaded {ToolCount} tools from MCP server '{ServerName}'",
-                        mcpTools.Count,
-                        server.mcpServerName);
-
-                    return (ServerName: server.mcpServerName, Tools: mcpTools, Success: true);
+                    try
+                    {
+                        var mcpTools = await GetMcpClientToolsAsync(
+                            turnContext,
+                            server,
+                            authToken,
+                            toolOptions).ConfigureAwait(false);
+
+                        _logger.LogInformation(
+                            "Successfully loaded {ToolCount} tools from MCP server '{ServerName}'",
+                            mcpTools.Count,
+                            server.mcpServerName);
+
+                        toolsByServer[server.mcpServerName] = mcpTools;
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(
+                            ex,
+                            "Failed to load tools from MCP server '{ServerName}' at '{Url}': {Error}",
+                            server.mcpServerName,
+                            server.url,
+                            ex.Message);
+                    }
                 }
-                catch (Exception ex)
+            }
+            else
+            {
+                // Parallel mode (default): enumerate all servers concurrently
+                var tasks = validServers.Select(async server =>
                 {
-                    _logger.LogError(
-                        ex,
-                        "Failed to load tools from MCP server '{ServerName}' at '{Url}': {Error}",
-                        server.mcpServerName,
-                        server.url,
-                        ex.Message);
-
-                    return (ServerName: server.mcpServerName, Tools: (IList<McpClientTool>)Array.Empty<McpClientTool>(), Success: false);
+                    try
+                    {
+                        var mcpTools = await GetMcpClientToolsAsync(
+                            turnContext,
+                            server,
+                            authToken,
+                            toolOptions).ConfigureAwait(false);
+
+                        _logger.LogInformation(
+                            "Successfully loaded {ToolCount} tools from MCP server '{ServerName}'",
+                            mcpTools.Count,
+                            server.mcpServerName);
+
+                        return (ServerName: server.mcpServerName, Tools: mcpTools, Success: true);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(
+                            ex,
+                            "Failed to load tools from MCP server '{ServerName}' at '{Url}': {Error}",
+                            server.mcpServerName,
+                            server.url,
+                            ex.Message);
+
+                        return (ServerName: server.mcpServerName, Tools: (IList<McpClientTool>)Array.Empty<McpClientTool>(), Success: false);
+                    }
+                });
+
+                var results = await Task.WhenAll(tasks).ConfigureAwait(false);
+
+                // Populate the dictionary with successful results
+                foreach (var result in results.Where(r => r.Success))
+                {
+                    toolsByServer[result.ServerName] = result.Tools;
                 }
-            });
-
-            var results = await Task.WhenAll(tasks).ConfigureAwait(false);
-
-            // Populate the dictionary with successful results
-            foreach (var result in results.Where(r => r.Success))
-            {
-                toolsByServer[result.ServerName] = result.Tools;
             }
 
             _logger.LogInformation(

ToolEnumerationBenchmark.csproj

<Project Sdk="Microsoft.NET.Sdk">
  <PropertyGroup>
    <OutputType>Exe</OutputType>
    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
    <IsPackable>false</IsPackable>
    <!-- Disable XML doc generation for benchmark project -->
    <GenerateDocumentationFile>false</GenerateDocumentationFile>
  </PropertyGroup>
  <ItemGroup>
    <ProjectReference Include="..\..\Tooling\Core\Microsoft.Agents.A365.Tooling.csproj" />
  </ItemGroup>
  <ItemGroup>
    <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
    <PackageReference Include="Microsoft.Extensions.Logging.Console" />
    <PackageReference Include="Microsoft.Extensions.Http" />
    <PackageReference Include="Microsoft.Extensions.Configuration" />
    <PackageReference Include="Microsoft.Agents.Builder" />
    <PackageReference Include="Moq" />
  </ItemGroup>
  <ItemGroup>
    <None Update="ToolingManifest.json">
      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
    </None>
  </ItemGroup>
</Project>

ToolingManifest.json

{
  "mcpServers": [
    {
      "mcpServerName": "mcp_SharePointListsTools",
      "mcpServerUniqueName": "mcp_SharePointListsTools",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_SharePointListsTools",
      "scope": "McpServers.SharepointLists.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_ODSPRemoteServer",
      "mcpServerUniqueName": "mcp_ODSPRemoteServer",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_ODSPRemoteServer",
      "scope": "McpServers.OneDriveSharepoint.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_MeServer",
      "mcpServerUniqueName": "mcp_MeServer",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_MeServer",
      "scope": "McpServers.Me.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_TeamsServer",
      "mcpServerUniqueName": "mcp_TeamsServer",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_TeamsServer",
      "scope": "McpServers.Teams.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_Admin365_GraphTools",
      "mcpServerUniqueName": "mcp_Admin365_GraphTools",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_Admin365_GraphTools",
      "scope": "McpServers.Admin365Graph.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_KnowledgeTools",
      "mcpServerUniqueName": "mcp_KnowledgeTools",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_KnowledgeTools",
      "scope": "McpServers.Knowledge.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_CalendarTools",
      "mcpServerUniqueName": "mcp_CalendarTools",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_CalendarTools",
      "scope": "McpServers.Calendar.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_DASearch",
      "mcpServerUniqueName": "mcp_DASearch",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_DASearch",
      "scope": "McpServers.DASearch.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_MailTools",
      "mcpServerUniqueName": "mcp_MailTools",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_MailTools",
      "scope": "McpServers.Mail.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_AdminTools",
      "mcpServerUniqueName": "mcp_AdminTools",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_AdminTools",
      "scope": "McpServers.M365Admin.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_WordServer",
      "mcpServerUniqueName": "mcp_WordServer",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_WordServer",
      "scope": "McpServers.Word.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    },
    {
      "mcpServerName": "mcp_M365Copilot",
      "mcpServerUniqueName": "mcp_M365Copilot",
      "url": "https://agent365.svc.cloud.microsoft/agents/servers/mcp_M365Copilot",
      "scope": "McpServers.CopilotMCP.All",
      "audience": "ea9ffc3e-8a23-4a7d-836d-234d7c7565c1"
    }
  ]
}