Files
oai-swift/oAITests/MCPTransportTests.swift
T
rune 57b3477903 Fix External MCP server bugs; add npx/Node.js detection and install help
Root-caused two real issues Rune hit with Obsidian/Homepage external MCP
servers:

1. Toggling a server's enable switch silently wiped transportKind/env/
   url/bearerToken/headers back to stdio defaults (only id/name/command/
   args/isEnabled/timeout/createdAt were preserved) — almost certainly
   how Obsidian's config got corrupted into an empty-command stdio entry
   despite never being edited directly. Fixed via
   ExternalMCPServer.withEnabledToggled(), which flips only isEnabled.

2. npx (installed via Homebrew) was invisible to Confab because GUI apps
   only inherit launchd's minimal PATH, not the Terminal PATH. Tried
   spawning the user's login shell to ask for its real PATH — this
   caused two real hangs in one session (first an -ilc pipe deadlock,
   then a waitUntilExit()/CFRunLoop reentrancy issue even after fixing
   that) and was abandoned entirely in favor of LoginShellEnvironment:
   deterministic, subprocess-free directory probing (Homebrew, MacPorts,
   Volta, nvm's alias file) that can't hang by construction.

Also added:
- Edit capability for existing External MCP servers (previously only
  Add/Toggle/Delete) — the second thing Rune explicitly asked for, and
  the way to fix a corrupted entry like Obsidian's without deleting it.
- MCPClientError.commandNotFound: a stdio server's command is checked
  against PATH up front in StdioMCPTransport.prepare() and fails
  immediately with a clear reason instead of cycling through 3 rounds of
  crash/restart backoff (5s/15s/30s) for a permanently-missing binary.
- A "Get Node.js" button appears when this happens, opening a sheet with
  a copyable `brew install node`, a one-click install (via
  NodeInstallHelper, using the terminationHandler/readabilityHandler
  pattern already proven safe elsewhere in this file — deliberately not
  waitUntilExit()), or a nodejs.org link if Homebrew isn't present.
- ExternalMCPManager.retryClient(id:) to manually retry after fixing the
  underlying cause.
- Help book: new "Servers That Use npx" section, updated Server Status
  section, updated Settings blurb.

37 new/changed tests covering the toggle fix, PATH probing, the
commandNotFound fast-fail path, and missing-command detection — full
suite (374 tests) passes clean.
2026-08-26 14:02:17 +02:00

289 lines
13 KiB
Swift

//
// MCPTransportTests.swift
// ConfabTests
//
// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
// Copyright (C) 2026 Rune Olsen
import Testing
import Foundation
@testable import Confab
@Suite("MCPTransportSupport.extractResult")
struct MCPTransportSupportTests {
@Test("Extracts the result payload from a successful JSON-RPC response")
func extractsResult() throws {
let json: [String: Any] = ["jsonrpc": "2.0", "id": 1, "result": ["tools": []]]
let data = try MCPTransportSupport.extractResult(from: json)
let decoded = try JSONSerialization.jsonObject(with: data) as? [String: Any]
#expect(decoded?["tools"] != nil)
}
@Test("Throws with the server's message when the response is a JSON-RPC error")
func throwsOnJSONRPCError() {
let json: [String: Any] = ["jsonrpc": "2.0", "id": 1, "error": ["code": -32601, "message": "Method not found"]]
do {
_ = try MCPTransportSupport.extractResult(from: json)
Issue.record("Expected extractResult to throw")
} catch let error as MCPClientError {
switch error {
case .invalidResponse(let message): #expect(message == "Method not found")
default: Issue.record("Expected .invalidResponse, got \(error)")
}
} catch {
Issue.record("Expected MCPClientError, got \(error)")
}
}
@Test("Throws when the response has neither result nor error")
func throwsOnMissingResult() {
let json: [String: Any] = ["jsonrpc": "2.0", "id": 1]
do {
_ = try MCPTransportSupport.extractResult(from: json)
Issue.record("Expected extractResult to throw")
} catch let error as MCPClientError {
switch error {
case .invalidResponse(let message): #expect(message == "Missing result field")
default: Issue.record("Expected .invalidResponse, got \(error)")
}
} catch {
Issue.record("Expected MCPClientError, got \(error)")
}
}
}
@Suite("HTTPMCPTransport.parseResponseBody")
struct HTTPMCPTransportParseTests {
@Test("Parses a plain application/json response body directly")
func parsesPlainJSON() throws {
let body = Data(#"{"jsonrpc":"2.0","id":7,"result":{"ok":true}}"#.utf8)
let json = try HTTPMCPTransport.parseResponseBody(body, contentType: "application/json", expectedId: 7)
#expect(json["id"] as? Int == 7)
}
@Test("Parses a text/event-stream body, finding the data: line matching the expected id")
func parsesSSEMatchingId() throws {
let sse = """
event: message
data: {"jsonrpc":"2.0","id":7,"result":{"ok":true}}
"""
let json = try HTTPMCPTransport.parseResponseBody(
Data(sse.utf8), contentType: "text/event-stream", expectedId: 7
)
#expect(json["id"] as? Int == 7)
}
@Test("Skips unrelated server-sent messages before the matching response, per the Streamable HTTP spec")
func skipsUnrelatedMessagesInSSE() throws {
// Spec: "The server MAY send JSON-RPC requests and notifications before sending the
// JSON-RPC response." Simulated here as an unrelated id=99 message before the real id=7 one.
let sse = """
data: {"jsonrpc":"2.0","id":99,"method":"unrelated/notification"}
data: {"jsonrpc":"2.0","id":7,"result":{"ok":true}}
"""
let json = try HTTPMCPTransport.parseResponseBody(
Data(sse.utf8), contentType: "text/event-stream", expectedId: 7
)
#expect(json["id"] as? Int == 7)
}
@Test("Throws when no SSE data: line matches the expected id")
func throwsWhenNoMatchInSSE() {
let sse = """
data: {"jsonrpc":"2.0","id":99,"result":{}}
"""
do {
_ = try HTTPMCPTransport.parseResponseBody(
Data(sse.utf8), contentType: "text/event-stream", expectedId: 7
)
Issue.record("Expected parseResponseBody to throw")
} catch let error as MCPClientError {
switch error {
case .invalidResponse: break
default: Issue.record("Expected .invalidResponse, got \(error)")
}
} catch {
Issue.record("Expected MCPClientError, got \(error)")
}
}
@Test("Throws on malformed JSON in a plain application/json body")
func throwsOnMalformedJSON() {
let body = Data("not json".utf8)
do {
_ = try HTTPMCPTransport.parseResponseBody(body, contentType: "application/json", expectedId: 1)
Issue.record("Expected parseResponseBody to throw")
} catch let error as MCPClientError {
switch error {
case .invalidResponse: break
default: Issue.record("Expected .invalidResponse, got \(error)")
}
} catch {
Issue.record("Expected MCPClientError, got \(error)")
}
}
}
@Suite("LoginShellEnvironment (deterministic, no-subprocess PATH probing)")
struct LoginShellEnvironmentTests {
@Test("candidateDirectories includes the fixed Homebrew/MacPorts/Volta locations")
func candidateDirectoriesIncludesFixedLocations() {
let dirs = LoginShellEnvironment.candidateDirectories(home: "/Users/testuser")
#expect(dirs.contains("/opt/homebrew/bin"))
#expect(dirs.contains("/usr/local/bin"))
#expect(dirs.contains("/opt/local/bin"))
#expect(dirs.contains("/Users/testuser/.volta/bin"))
}
@Test("nvmDefaultNodeBinDirectory returns nil when the alias file doesn't exist")
func nvmReturnsNilWithoutAliasFile() {
#expect(LoginShellEnvironment.nvmDefaultNodeBinDirectory(home: "/nonexistent-\(UUID().uuidString)") == nil)
}
@Test("candidateDirectories appends the nvm default dir when present")
func candidateDirectoriesIncludesNvmWhenPresent() throws {
let tmpHome = NSTemporaryDirectory() + "confab-test-nvm-\(UUID().uuidString)"
let aliasDir = tmpHome + "/.nvm/alias"
try FileManager.default.createDirectory(atPath: aliasDir, withIntermediateDirectories: true)
try "v20.11.0\n".write(toFile: aliasDir + "/default", atomically: true, encoding: .utf8)
defer { try? FileManager.default.removeItem(atPath: tmpHome) }
let nvmDir = LoginShellEnvironment.nvmDefaultNodeBinDirectory(home: tmpHome)
#expect(nvmDir == "\(tmpHome)/.nvm/versions/node/v20.11.0/bin")
#expect(LoginShellEnvironment.candidateDirectories(home: tmpHome).contains(nvmDir ?? ""))
}
@Test("nvmDefaultNodeBinDirectory adds a 'v' prefix if the alias file lacks one")
func nvmAddsVPrefixIfMissing() throws {
let tmpHome = NSTemporaryDirectory() + "confab-test-nvm-\(UUID().uuidString)"
let aliasDir = tmpHome + "/.nvm/alias"
try FileManager.default.createDirectory(atPath: aliasDir, withIntermediateDirectories: true)
try "20.11.0".write(toFile: aliasDir + "/default", atomically: true, encoding: .utf8)
defer { try? FileManager.default.removeItem(atPath: tmpHome) }
#expect(LoginShellEnvironment.nvmDefaultNodeBinDirectory(home: tmpHome) == "\(tmpHome)/.nvm/versions/node/v20.11.0/bin")
}
@Test("augmentedPath prepends only directories that actually exist")
func augmentedPathOnlyAddsExistingDirs() {
// /usr/bin always exists on macOS; a random UUID-named dir never will.
let result = LoginShellEnvironment.augmentedPath(
basePath: "/usr/bin:/bin", home: "/nonexistent-\(UUID().uuidString)"
)
// None of the fixed candidates exist under a bogus home + this sandboxed test environment
// is unlikely to have /opt/homebrew, /usr/local, or /opt/local — but if it does (real dev
// machine running the suite), that's fine too: just confirm the base path is preserved.
#expect(result.hasSuffix("/usr/bin:/bin"))
}
@Test("augmentedPath doesn't duplicate a directory already present in basePath")
func augmentedPathAvoidsDuplicates() {
let result = LoginShellEnvironment.augmentedPath(basePath: "/opt/homebrew/bin:/usr/bin:/bin")
let components = result.split(separator: ":").map(String.init)
#expect(components.filter { $0 == "/opt/homebrew/bin" }.count == 1)
}
@Test("findExecutable finds a real executable on a real search path")
func findExecutableFindsRealBinary() {
// /bin/ls exists and is executable on every macOS install.
#expect(LoginShellEnvironment.findExecutable(named: "ls", in: "/usr/bin:/bin") == "/bin/ls")
}
@Test("findExecutable returns nil for a name that doesn't exist anywhere on the path")
func findExecutableReturnsNilForMissingBinary() {
#expect(LoginShellEnvironment.findExecutable(named: "definitely-not-a-real-binary-\(UUID().uuidString)", in: "/usr/bin:/bin") == nil)
}
@Test("findExecutable checks directories in order and returns the first match")
func findExecutableRespectsOrder() throws {
let tmpDir1 = NSTemporaryDirectory() + "confab-test-bin1-\(UUID().uuidString)"
let tmpDir2 = NSTemporaryDirectory() + "confab-test-bin2-\(UUID().uuidString)"
try FileManager.default.createDirectory(atPath: tmpDir1, withIntermediateDirectories: true)
try FileManager.default.createDirectory(atPath: tmpDir2, withIntermediateDirectories: true)
defer {
try? FileManager.default.removeItem(atPath: tmpDir1)
try? FileManager.default.removeItem(atPath: tmpDir2)
}
let toolName = "confab-test-tool-\(UUID().uuidString)"
for dir in [tmpDir1, tmpDir2] {
let path = "\(dir)/\(toolName)"
FileManager.default.createFile(atPath: path, contents: Data("#!/bin/sh\n".utf8))
try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: path)
}
#expect(LoginShellEnvironment.findExecutable(named: toolName, in: "\(tmpDir1):\(tmpDir2)") == "\(tmpDir1)/\(toolName)")
}
@Test("homebrewPrefix returns nil when neither known brew binary exists")
func homebrewPrefixNilWhenAbsent() {
// Can't easily fake a FileManager that reports both real paths as absent without a full
// protocol seam here, but we can confirm the function doesn't crash and returns a sensible
// type; the presence/absence branches are exercised implicitly by whichever machine runs
// this (either is a valid, non-crashing outcome).
let result = LoginShellEnvironment.homebrewPrefix()
#expect(result == nil || result == "/opt/homebrew" || result == "/usr/local")
}
}
@Suite("StdioMCPTransport.prepare() fails fast on a missing command")
@MainActor
struct StdioMCPTransportPrepareTests {
@Test("Throws commandNotFound immediately for a command that doesn't exist anywhere on PATH — never spawns a process")
func throwsForMissingRelativeCommand() async {
let server = ExternalMCPServer(
name: "Bogus",
command: "definitely-not-a-real-command-\(UUID().uuidString)",
args: []
)
let transport = StdioMCPTransport(server: server)
do {
try await transport.prepare()
Issue.record("Expected prepare() to throw")
} catch let error as MCPClientError {
switch error {
case .commandNotFound(let cmd): #expect(cmd == server.command)
default: Issue.record("Expected .commandNotFound, got \(error)")
}
} catch {
Issue.record("Expected MCPClientError, got \(error)")
}
}
@Test("Throws commandNotFound for an absolute path that doesn't exist")
func throwsForMissingAbsoluteCommand() async {
let server = ExternalMCPServer(
name: "Bogus",
command: "/nonexistent/\(UUID().uuidString)/binary",
args: []
)
let transport = StdioMCPTransport(server: server)
do {
try await transport.prepare()
Issue.record("Expected prepare() to throw")
} catch let error as MCPClientError {
switch error {
case .commandNotFound(let cmd): #expect(cmd == server.command)
default: Issue.record("Expected .commandNotFound, got \(error)")
}
} catch {
Issue.record("Expected MCPClientError, got \(error)")
}
}
@Test("Succeeds in resolving a real, always-present absolute command")
func doesNotThrowForRealAbsoluteCommand() async throws {
// /bin/echo exists on every macOS install and exits immediately — safe to actually launch.
let server = ExternalMCPServer(name: "Echo", command: "/bin/echo", args: ["hi"])
let transport = StdioMCPTransport(server: server)
try await transport.prepare()
transport.stop()
}
}