From 1631f589abf9887dcd712da5f6668941aed55b1f Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 2 May 2026 18:29:16 +0100 Subject: [PATCH 1/6] fix(test): Stream simulator test execution progress Emit a test-running stage when two-phase simulator execution moves from build-for-testing to test-without-building so the CLI does not leave stale build status visible while UI tests are running. Keep static discovery observational, parse destination-suffixed XCTest result lines, and handle multiline Swift Testing arguments so discovery and final counts stay project-agnostic across Calculator and Weather. Fixes #384 Co-Authored-By: OpenAI Codex --- CHANGELOG.md | 3 + .../xcshareddata/xcschemes/Weather.xcscheme | 101 +++++++++ .../simulator-test-execution.test.ts | 94 ++++++++ .../__tests__/swift-test-discovery.test.ts | 54 +++++ src/utils/__tests__/test-common.test.ts | 112 +++++++++- .../__tests__/xcodebuild-event-parser.test.ts | 202 ++++++++++++++++++ .../__tests__/cli-text-renderer.test.ts | 60 ++++++ src/utils/renderers/cli-text-renderer.ts | 15 +- src/utils/renderers/event-formatting.ts | 6 + src/utils/simulator-test-execution.ts | 9 +- src/utils/swift-test-discovery.ts | 42 +++- src/utils/test-common.ts | 8 + src/utils/xcodebuild-event-parser.ts | 35 ++- src/utils/xcodebuild-line-parsers.ts | 4 +- 14 files changed, 727 insertions(+), 18 deletions(-) create mode 100644 example_projects/Weather/Weather.xcodeproj/xcshareddata/xcschemes/Weather.xcscheme create mode 100644 src/utils/__tests__/simulator-test-execution.test.ts create mode 100644 src/utils/__tests__/swift-test-discovery.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a1278934..29d0da2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,12 @@ ### Fixed +- Fixed simulator test JSONL accuracy by keeping preflight discovery observational, preserving only user-supplied test selectors, discovering multiline parameterized Swift Testing tests, and parsing destination-suffixed xcodebuild test result lines. - Removed stale physical-device log session status and shutdown cleanup for deprecated standalone device log capture, and corrected the device build-and-run tool description. +- Fixed mixed Swift Testing and XCTest summaries so simulator test text output no longer overcounts parameterized Swift Testing results or issue lines. - Fixed CLI test summaries showing false-positive compiler errors from xcodebuild NSError dump lines, and added compiler-error snapshot coverage for simulator, device, and macOS build-style flows ([#383](https://github.com/getsentry/XcodeBuildMCP/issues/383)). - Fixed simulator OSLog helper cleanup so server and daemon startup reconcile same-workspace orphaned log streams without stopping helpers owned by live sessions in other workspaces ([#382](https://github.com/getsentry/XcodeBuildMCP/issues/382)). +- Fixed Weather example test discovery and made CLI test progress visible while tests are running instead of leaving the last build phase displayed. ## [2.5.0-beta.1] diff --git a/example_projects/Weather/Weather.xcodeproj/xcshareddata/xcschemes/Weather.xcscheme b/example_projects/Weather/Weather.xcodeproj/xcshareddata/xcschemes/Weather.xcscheme new file mode 100644 index 00000000..55830b38 --- /dev/null +++ b/example_projects/Weather/Weather.xcodeproj/xcshareddata/xcschemes/Weather.xcscheme @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/utils/__tests__/simulator-test-execution.test.ts b/src/utils/__tests__/simulator-test-execution.test.ts new file mode 100644 index 00000000..6c39fd15 --- /dev/null +++ b/src/utils/__tests__/simulator-test-execution.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from 'vitest'; +import { createSimulatorTwoPhaseExecutionPlan } from '../simulator-test-execution.ts'; +import type { TestPreflightResult } from '../test-preflight.ts'; + +function createPreflight(): TestPreflightResult { + return { + scheme: 'CalculatorApp', + configuration: 'Debug', + projectPath: '/tmp/CalculatorApp.xcodeproj', + destinationName: 'iPhone 17 Pro', + selectors: { onlyTesting: [], skipTesting: [] }, + warnings: [], + completeness: 'complete', + totalTests: 2, + targets: [ + { + name: 'CalculatorAppTests', + warnings: [], + files: [ + { + path: '/tmp/CalculatorAppTests.swift', + tests: [ + { + framework: 'xctest', + targetName: 'CalculatorAppTests', + typeName: 'CalculatorAppTests', + methodName: 'testAddition', + displayName: 'CalculatorAppTests/CalculatorAppTests/testAddition', + line: 10, + parameterized: false, + }, + { + framework: 'swift-testing', + targetName: 'CalculatorAppTests', + typeName: 'ExpressionSuite', + methodName: 'evaluatesExpression', + displayName: 'CalculatorAppTests/ExpressionSuite/evaluatesExpression', + line: 20, + parameterized: true, + }, + ], + }, + ], + }, + ], + }; +} + +describe('createSimulatorTwoPhaseExecutionPlan', () => { + it('keeps preflight discovery observational instead of synthesizing only-testing selectors', () => { + const plan = createSimulatorTwoPhaseExecutionPlan({ + extraArgs: ['-parallel-testing-enabled', 'YES'], + preflight: createPreflight(), + resultBundlePath: '/tmp/Calculator.xcresult', + }); + + expect(plan.buildArgs).toEqual(['-parallel-testing-enabled', 'YES']); + expect(plan.testArgs).toEqual([ + '-parallel-testing-enabled', + 'YES', + '-resultBundlePath', + '/tmp/Calculator.xcresult', + ]); + expect(plan.usesExactSelectors).toBe(false); + }); + + it('preserves user-supplied selector arguments in both simulator test phases', () => { + const plan = createSimulatorTwoPhaseExecutionPlan({ + extraArgs: [ + '-only-testing:CalculatorAppTests/CalculatorAppTests/testAddition', + '-skip-testing', + 'CalculatorAppTests/ExpressionSuite/evaluatesExpression', + ], + preflight: createPreflight(), + }); + + expect(plan.buildArgs).toEqual([ + '-only-testing:CalculatorAppTests/CalculatorAppTests/testAddition', + '-skip-testing', + 'CalculatorAppTests/ExpressionSuite/evaluatesExpression', + ]); + expect(plan.testArgs).toEqual(plan.buildArgs); + expect(plan.usesExactSelectors).toBe(true); + }); + + it('keeps resultBundlePath out of build-for-testing args and includes it for test-without-building', () => { + const plan = createSimulatorTwoPhaseExecutionPlan({ + extraArgs: ['-resultBundlePath', '/tmp/UserProvided.xcresult'], + }); + + expect(plan.buildArgs).toEqual([]); + expect(plan.testArgs).toEqual(['-resultBundlePath', '/tmp/UserProvided.xcresult']); + }); +}); diff --git a/src/utils/__tests__/swift-test-discovery.test.ts b/src/utils/__tests__/swift-test-discovery.test.ts new file mode 100644 index 00000000..580eefa5 --- /dev/null +++ b/src/utils/__tests__/swift-test-discovery.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { createMockFileSystemExecutor } from '../../test-utils/mock-executors.ts'; +import { discoverSwiftTestsInFiles } from '../swift-test-discovery.ts'; + +describe('discoverSwiftTestsInFiles', () => { + it('discovers Swift Testing functions with multiline parameterized Test attributes', async () => { + const filePath = '/tmp/CalculatorServiceTests.swift'; + const fileSystemExecutor = createMockFileSystemExecutor({ + readFile: async () => ` +import Testing + +struct CalculatorServiceTests { + @Test( + "evaluates decimal operations", + arguments: [ + ("1 + 1", "2"), + ("4 / 2", "2"), + ] + ) + func evaluatesDecimalOperations(expression: String, expected: String) async throws {} + + @Test(arguments: ["+", "-", "×"]) + func evaluatesOperators(symbol: String) async throws {} +} +`, + }); + + const files = await discoverSwiftTestsInFiles( + 'CalculatorAppFeatureTests', + [filePath], + fileSystemExecutor, + ); + + expect(files).toHaveLength(1); + expect(files[0].tests).toMatchObject([ + { + framework: 'swift-testing', + targetName: 'CalculatorAppFeatureTests', + typeName: 'CalculatorServiceTests', + methodName: 'evaluatesDecimalOperations', + displayName: 'CalculatorAppFeatureTests/CalculatorServiceTests/evaluatesDecimalOperations', + parameterized: true, + }, + { + framework: 'swift-testing', + targetName: 'CalculatorAppFeatureTests', + typeName: 'CalculatorServiceTests', + methodName: 'evaluatesOperators', + displayName: 'CalculatorAppFeatureTests/CalculatorServiceTests/evaluatesOperators', + parameterized: true, + }, + ]); + }); +}); diff --git a/src/utils/__tests__/test-common.test.ts b/src/utils/__tests__/test-common.test.ts index e994d1f5..9006bca1 100644 --- a/src/utils/__tests__/test-common.test.ts +++ b/src/utils/__tests__/test-common.test.ts @@ -1,5 +1,58 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { resolveTestProgressEnabled } from '../test-common.ts'; +import type { ChildProcess } from 'node:child_process'; +import { createTestExecutor, resolveTestProgressEnabled } from '../test-common.ts'; +import type { CommandExecutor, CommandResponse } from '../command.ts'; +import { DefaultStreamingExecutionContext } from '../execution/index.ts'; +import type { AnyFragment } from '../../types/domain-fragments.ts'; +import type { TestPreflightResult } from '../test-preflight.ts'; +import { XcodePlatform } from '../xcode.ts'; + +function createSuccessfulCommandResponse(): CommandResponse { + return { + success: true, + output: '', + process: { pid: 12345 } as ChildProcess, + exitCode: 0, + }; +} + +function createPreflight(): TestPreflightResult { + return { + scheme: 'Weather', + configuration: 'Debug', + projectPath: 'Weather.xcodeproj', + destinationName: 'iPhone 17 Pro', + selectors: { + onlyTesting: [], + skipTesting: [], + }, + targets: [ + { + name: 'WeatherTests', + files: [ + { + path: 'WeatherTests/WeatherTests.swift', + tests: [ + { + framework: 'swift-testing', + targetName: 'WeatherTests', + typeName: 'WeatherTests', + methodName: 'emptySearchReturnsNoResults', + displayName: 'WeatherTests/WeatherTests/emptySearchReturnsNoResults', + line: 12, + parameterized: false, + }, + ], + }, + ], + warnings: [], + }, + ], + warnings: [], + totalTests: 1, + completeness: 'complete', + }; +} describe('resolveTestProgressEnabled', () => { const originalRuntime = process.env.XCODEBUILDMCP_RUNTIME; @@ -39,3 +92,60 @@ describe('resolveTestProgressEnabled', () => { expect(resolveTestProgressEnabled(false)).toBe(false); }); }); + +describe('createTestExecutor', () => { + it('emits RUN_TESTS before test-without-building starts in two-phase simulator execution', async () => { + const emitted: AnyFragment[] = []; + const actions: string[] = []; + const executor: CommandExecutor = async (command, _logPrefix, _useShell, opts) => { + const action = command.at(-1); + if (action) { + actions.push(action); + } + + if (action === 'build-for-testing') { + opts?.onStdout?.('Ld /tmp/Weather.build/Weather normal arm64\n'); + } + + return createSuccessfulCommandResponse(); + }; + + const executeTest = createTestExecutor(executor, { + preflight: createPreflight(), + toolName: 'test_sim', + target: 'simulator', + request: { + scheme: 'Weather', + projectPath: 'Weather.xcodeproj', + configuration: 'Debug', + platform: XcodePlatform.iOSSimulator, + }, + }); + + await executeTest( + { + projectPath: 'Weather.xcodeproj', + scheme: 'Weather', + configuration: 'Debug', + simulatorId: 'A2C64636-37E9-4B68-B872-E7F0A82A5670', + platform: XcodePlatform.iOSSimulator, + }, + new DefaultStreamingExecutionContext({ + onFragment: (fragment) => emitted.push(fragment), + }), + ); + + expect(actions).toEqual(['build-for-testing', 'test-without-building']); + + const stageEvents = emitted.filter((event) => event.fragment === 'build-stage'); + expect(stageEvents.map((event) => event.stage)).toEqual(['LINKING', 'RUN_TESTS']); + + const runTestsIndex = emitted.findIndex( + (event) => event.fragment === 'build-stage' && event.stage === 'RUN_TESTS', + ); + const finalSummaryIndex = emitted.findIndex((event) => event.fragment === 'build-summary'); + + expect(runTestsIndex).toBeGreaterThan(-1); + expect(finalSummaryIndex).toBeGreaterThan(runTestsIndex); + }); +}); diff --git a/src/utils/__tests__/xcodebuild-event-parser.test.ts b/src/utils/__tests__/xcodebuild-event-parser.test.ts index a49ed332..122270e4 100644 --- a/src/utils/__tests__/xcodebuild-event-parser.test.ts +++ b/src/utils/__tests__/xcodebuild-event-parser.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { createXcodebuildEventParser } from '../xcodebuild-event-parser.ts'; +import { createXcodebuildRunState } from '../xcodebuild-run-state.ts'; import type { DomainFragment } from '../../types/domain-fragments.ts'; function collectEvents( @@ -24,6 +25,43 @@ function collectEvents( return events; } +function collectRunStateEvents( + lines: { source: 'stdout' | 'stderr'; text: string }[], +): DomainFragment[] { + const events: DomainFragment[] = []; + const runState = createXcodebuildRunState({ + operation: 'TEST', + onEvent: (event) => events.push(event), + }); + const parser = createXcodebuildEventParser({ + operation: 'TEST', + onEvent: (event) => { + switch (event.fragment) { + case 'build-stage': + case 'compiler-diagnostic': + case 'test-discovery': + case 'test-failure': + case 'test-progress': + case 'test-case-result': + runState.push(event); + break; + } + }, + }); + + for (const { source, text } of lines) { + if (source === 'stdout') { + parser.onStdout(text); + } else { + parser.onStderr(text); + } + } + + parser.flush(); + runState.finalize(false); + return events; +} + describe('xcodebuild-event-parser', () => { it('emits status events for package resolution', () => { const events = collectEvents('TEST', [{ source: 'stdout', text: 'Resolve Package Graph\n' }]); @@ -75,6 +113,31 @@ describe('xcodebuild-event-parser', () => { }); }); + it('emits running-tests status for test suite and case start lines', () => { + const events = collectEvents('TEST', [ + { + source: 'stdout', + text: "Test suite 'WeatherUITests' started on 'Clone 1 of iPhone 17 Pro - WeatherUITests-Runner (12147)'\n", + }, + { + source: 'stdout', + text: "Test case 'WeatherTests/emptySearchReturnsNoResults()' started on 'Clone 2 of iPhone 17 Pro - Weather (12472)'\n", + }, + { + source: 'stdout', + text: '◇ Test "Calculator initializes with correct default values" started.\n', + }, + ]); + + const stages = events.filter((event) => event.fragment === 'build-stage'); + expect(stages).toHaveLength(3); + expect(stages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ fragment: 'build-stage', stage: 'RUN_TESTS' }), + ]), + ); + }); + it('emits test-progress events with cumulative counts', () => { const events = collectEvents('TEST', [ { source: 'stdout', text: "Test Case '-[Suite testA]' passed (0.001 seconds)\n" }, @@ -115,6 +178,50 @@ describe('xcodebuild-event-parser', () => { }); }); + it('parses modern xcodebuild Test Case lines with destinations', () => { + const events = collectEvents('TEST', [ + { + source: 'stdout', + text: "Test Case '-[WeatherTests testForecast]' passed on 'iPhone 16 Pro' (0.010 seconds)\n", + }, + { + source: 'stdout', + text: "Test case 'WeatherUITests/testSearch()' failed on 'Clone 1 of iPhone 16 Pro' (0.250 seconds)\n", + }, + { + source: 'stdout', + text: "Test case 'WeatherUITests/testOfflineMode()' skipped on 'Clone 2 of iPhone 16 Pro' (0.001 seconds)\n", + }, + ]); + + const cases = events.filter((e) => e.fragment === 'test-case-result'); + expect(cases).toHaveLength(3); + expect(cases[0]).toMatchObject({ + suite: 'WeatherTests', + test: 'testForecast', + status: 'passed', + durationMs: 10, + }); + expect(cases[1]).toMatchObject({ + suite: 'WeatherUITests', + test: 'testSearch()', + status: 'failed', + durationMs: 250, + }); + expect(cases[2]).toMatchObject({ + suite: 'WeatherUITests', + test: 'testOfflineMode()', + status: 'skipped', + durationMs: 1, + }); + + const progressEvents = events.filter((e) => e.fragment === 'test-progress'); + expect(progressEvents).toHaveLength(3); + expect(progressEvents[0]).toMatchObject({ completed: 1, failed: 0, skipped: 0 }); + expect(progressEvents[1]).toMatchObject({ completed: 2, failed: 1, skipped: 0 }); + expect(progressEvents[2]).toMatchObject({ completed: 3, failed: 1, skipped: 1 }); + }); + it('emits test-case-result events for Swift Testing passed/failed lines', () => { const events = collectEvents('TEST', [ { source: 'stdout', text: '✔ Test "passingTest()" passed after 0.005 seconds.\n' }, @@ -398,6 +505,101 @@ describe('xcodebuild-event-parser', () => { }); }); + it('uses Swift Testing and XCTest summaries once for mixed Calculator test output', () => { + const xctestPassedLines = Array.from({ length: 21 }, (_, index) => ({ + source: 'stdout' as const, + text: `Test Case '-[CalculatorAppTests.CalculatorAppTests testPassing${index + 1}]' passed (0.001 seconds).\n`, + })); + const events = collectRunStateEvents([ + { + source: 'stdout', + text: '✔ Test "Adding single digit numbers" passed after 0.016 seconds.\n', + }, + { + source: 'stdout', + text: '\u200B✔ Test "Adding decimal numbers" passed after 0.012 seconds.\n', + }, + { + source: 'stdout', + text: '✔ Test "Addition operation" with 4 test cases passed after 0.005 seconds.\n', + }, + { + source: 'stdout', + text: '✘ Test "This test should fail to verify error reporting" recorded an issue at CalculatorServiceTests.swift:37:9: Expectation failed: (calculator.display → "0") == "999"\n', + }, + { + source: 'stdout', + text: '✘ Test "This test should fail to verify error reporting" failed after 0.029 seconds with 1 issue.\n', + }, + { + source: 'stdout', + text: '✘ Test run with 34 tests in 9 suites failed after 0.047 seconds with 1 issue.\n', + }, + ...xctestPassedLines, + { + source: 'stderr', + text: '/Volumes/Developer/XcodeBuildMCP/example_projects/iOS_Calculator/CalculatorAppTests/CalculatorAppTests.swift:52: error: -[CalculatorAppTests.CalculatorAppTests testCalculatorServiceFailure] : XCTAssertEqual failed: ("0") is not equal to ("999") - This test should fail - display should be 0, not 999\n', + }, + { + source: 'stdout', + text: "Test Case '-[CalculatorAppTests.CalculatorAppTests testCalculatorServiceFailure]' failed (0.004 seconds).\n", + }, + { + source: 'stderr', + text: '/Volumes/Developer/XcodeBuildMCP/example_projects/iOS_Calculator/CalculatorAppTests/CalculatorAppTests.swift:286: error: -[CalculatorAppTests.IntentionalFailureTests test] : XCTAssertTrue failed - This test should fail to verify error reporting\n', + }, + { + source: 'stdout', + text: "Test Case '-[CalculatorAppTests.IntentionalFailureTests test]' failed (0.003 seconds).\n", + }, + { + source: 'stdout', + text: '\t Executed 23 tests, with 2 failures (0 unexpected) in 0.654 (0.665) seconds\n', + }, + ]); + + const summary = events.filter((event) => event.fragment === 'build-summary').at(-1); + expect(summary).toMatchObject({ + fragment: 'build-summary', + operation: 'TEST', + totalTests: 57, + passedTests: 54, + failedTests: 3, + skippedTests: 0, + }); + }); + + it('reconciles separate Swift Testing run summaries independently', () => { + const events = collectRunStateEvents([ + { + source: 'stdout', + text: '✔ Test "First target test" passed after 0.001 seconds.\n', + }, + { + source: 'stdout', + text: '✔ Test run with 1 test in 1 suite passed after 0.001 seconds.\n', + }, + { + source: 'stdout', + text: '✔ Test "Second target parameterized test" with 4 test cases passed after 0.002 seconds.\n', + }, + { + source: 'stdout', + text: '✔ Test run with 1 test in 1 suite passed after 0.002 seconds.\n', + }, + ]); + + const summary = events.filter((event) => event.fragment === 'build-summary').at(-1); + expect(summary).toMatchObject({ + fragment: 'build-summary', + operation: 'TEST', + totalTests: 2, + passedTests: 2, + failedTests: 0, + skippedTests: 0, + }); + }); + it('processes full test lifecycle', () => { const events = collectEvents('TEST', [ { source: 'stdout', text: 'Resolve Package Graph\n' }, diff --git a/src/utils/renderers/__tests__/cli-text-renderer.test.ts b/src/utils/renderers/__tests__/cli-text-renderer.test.ts index d112d7ea..2a645b0b 100644 --- a/src/utils/renderers/__tests__/cli-text-renderer.test.ts +++ b/src/utils/renderers/__tests__/cli-text-renderer.test.ts @@ -145,6 +145,66 @@ describe('cli-text-renderer', () => { expect(output).toContain('\u{2705} Resolving app path\n'); }); + it('replaces interactive build-stage updates with test progress updates', () => { + const renderer = createCliTextRenderer({ interactive: true }); + + renderer.onFragment({ + kind: 'test-result', + fragment: 'build-stage', + operation: 'TEST', + stage: 'LINKING', + message: 'Linking', + }); + + renderer.onFragment({ + kind: 'test-result', + fragment: 'test-progress', + operation: 'TEST', + completed: 4, + failed: 0, + skipped: 0, + }); + + expect(reporter.update).toHaveBeenCalledWith('Linking...'); + expect(reporter.update).toHaveBeenCalledWith( + 'Running tests (4 completed, 0 failures, 0 skipped)', + ); + }); + + it('renders non-interactive test progress durably and deduplicates repeated counts', () => { + const output = renderCliTextTranscript({ + items: [ + { + kind: 'test-result', + fragment: 'test-progress', + operation: 'TEST', + completed: 1, + failed: 0, + skipped: 0, + }, + { + kind: 'test-result', + fragment: 'test-progress', + operation: 'TEST', + completed: 1, + failed: 0, + skipped: 0, + }, + { + kind: 'test-result', + fragment: 'test-progress', + operation: 'TEST', + completed: 2, + failed: 0, + skipped: 0, + }, + ], + }); + + expect(output.match(/Running tests \(1 completed, 0 failures, 0 skipped\)/g)).toHaveLength(1); + expect(output).toContain('Running tests (2 completed, 0 failures, 0 skipped)'); + }); + it('renders grouped sad-path diagnostics before the failed summary', () => { const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); const renderer = createCliTextRenderer({ interactive: false }); diff --git a/src/utils/renderers/cli-text-renderer.ts b/src/utils/renderers/cli-text-renderer.ts index 897df0cc..9cbc0969 100644 --- a/src/utils/renderers/cli-text-renderer.ts +++ b/src/utils/renderers/cli-text-renderer.ts @@ -39,6 +39,7 @@ import { formatNextStepsEvent, formatTestCaseResults, formatTestDiscoveryEvent, + formatTestProgressEvent, } from './event-formatting.ts'; import { createXcodebuildEventParser, @@ -112,6 +113,7 @@ function createCliTextProcessor(options: CliTextProcessorOptions): TranscriptRen let nextSteps: readonly NextStep[] = []; let nextStepsRuntime: 'cli' | 'daemon' | 'mcp' | undefined; let sawProgressNextSteps = false; + let lastRenderedTestProgressKey: string | null = null; function writeDurable(text: string): void { sink.clearTransient(); @@ -261,10 +263,16 @@ function createCliTextProcessor(options: CliTextProcessorOptions): TranscriptRen } case 'test-progress': { + const renderedProgress = formatTestProgressEvent(item); + const progressKey = `${item.completed}:${item.failed}:${item.skipped}`; + pendingTransientRuntimeLine = null; if (interactive) { - const failWord = item.failed === 1 ? 'failure' : 'failures'; - pendingTransientRuntimeLine = null; - sink.updateTransient(`Running tests (${item.completed}, ${item.failed} ${failWord})`); + sink.updateTransient(renderedProgress); + } else if (progressKey !== lastRenderedTestProgressKey) { + writeDurable(renderedProgress); + lastRenderedTestProgressKey = progressKey; + lastVisibleEventType = 'test-progress'; + lastStatusLineLevel = null; } break; } @@ -442,6 +450,7 @@ function createCliTextProcessor(options: CliTextProcessorOptions): TranscriptRen parserStates.clear(); sawProgressNextSteps = false; collectedTestCaseResults.length = 0; + lastRenderedTestProgressKey = null; }, }; } diff --git a/src/utils/renderers/event-formatting.ts b/src/utils/renderers/event-formatting.ts index 76b55622..3c4e3b48 100644 --- a/src/utils/renderers/event-formatting.ts +++ b/src/utils/renderers/event-formatting.ts @@ -15,6 +15,7 @@ import type { TestCaseResultRenderItem, TestDiscoveryRenderItem, TestFailureRenderItem, + TestProgressRenderItem, } from '../../rendering/render-items.ts'; import type { DetailTreeTextBlock, @@ -547,6 +548,11 @@ export function formatTestDiscoveryEvent(event: TestDiscoveryRenderItem): string return lines.join('\n'); } +export function formatTestProgressEvent(event: TestProgressRenderItem): string { + const failWord = event.failed === 1 ? 'failure' : 'failures'; + return `Running tests (${event.completed} completed, ${event.failed} ${failWord}, ${event.skipped} skipped)`; +} + export function formatNextStepsEvent(event: NextStepsTextBlock, runtime: 'cli' | 'mcp'): string { return renderNextStepsSection(event.steps, runtime); } diff --git a/src/utils/simulator-test-execution.ts b/src/utils/simulator-test-execution.ts index 5d2b08b1..c23e1bf3 100644 --- a/src/utils/simulator-test-execution.ts +++ b/src/utils/simulator-test-execution.ts @@ -1,4 +1,4 @@ -import { collectResolvedTestSelectors, type TestPreflightResult } from './test-preflight.ts'; +import type { TestPreflightResult } from './test-preflight.ts'; function parseTestSelectorArgs(extraArgs: string[] | undefined): { remainingArgs: string[]; @@ -55,11 +55,8 @@ export function createSimulatorTwoPhaseExecutionPlan(params: { usesExactSelectors: boolean; } { const parsedArgs = parseTestSelectorArgs(params.extraArgs); - const resolvedSelectors = params.preflight ? collectResolvedTestSelectors(params.preflight) : []; - const exactSelectorArgs = resolvedSelectors.flatMap((selector) => [`-only-testing:${selector}`]); - const usesExactSelectors = exactSelectorArgs.length > 0; - - const selectedTestArgs = usesExactSelectors ? exactSelectorArgs : parsedArgs.selectorArgs; + const selectedTestArgs = parsedArgs.selectorArgs; + const usesExactSelectors = selectedTestArgs.length > 0; const resultBundlePath = params.resultBundlePath ?? parsedArgs.resultBundlePath; return { diff --git a/src/utils/swift-test-discovery.ts b/src/utils/swift-test-discovery.ts index 6830795a..8394e59d 100644 --- a/src/utils/swift-test-discovery.ts +++ b/src/utils/swift-test-discovery.ts @@ -94,6 +94,18 @@ function countBraces(line: string): number { return delta; } +function countParentheses(line: string): number { + let delta = 0; + for (const character of line) { + if (character === '(') { + delta += 1; + } else if (character === ')') { + delta -= 1; + } + } + return delta; +} + function collectXCTestTypes(lines: string[]): Set { const xctestTypes = new Set(); @@ -138,6 +150,8 @@ function discoverTestsInFileContent( const scopeStack: Array<{ typeName?: string; xctestContext: boolean; depth: number }> = []; let braceDepth = 0; let pendingAttributes: string[] = []; + let pendingMultilineAttributeIndex: number | null = null; + let pendingAttributeParenthesisDepth = 0; sanitizedLines.forEach((sanitizedLine, index) => { const lineNumber = index + 1; @@ -147,8 +161,27 @@ function discoverTestsInFileContent( scopeStack.pop(); } + let consumedAttributeContinuation = false; + if (line.startsWith('@')) { pendingAttributes.push(line); + const parenthesisDepth = countParentheses(line); + if (parenthesisDepth > 0) { + pendingMultilineAttributeIndex = pendingAttributes.length - 1; + pendingAttributeParenthesisDepth = parenthesisDepth; + } else { + pendingMultilineAttributeIndex = null; + pendingAttributeParenthesisDepth = 0; + } + } else if (pendingMultilineAttributeIndex !== null) { + pendingAttributes[pendingMultilineAttributeIndex] = + `${pendingAttributes[pendingMultilineAttributeIndex]} ${line}`; + pendingAttributeParenthesisDepth += countParentheses(line); + consumedAttributeContinuation = true; + if (pendingAttributeParenthesisDepth <= 0) { + pendingMultilineAttributeIndex = null; + pendingAttributeParenthesisDepth = 0; + } } const typeMatch = line.match( @@ -203,7 +236,14 @@ function discoverTestsInFileContent( } pendingAttributes = []; - } else if (line.length > 0 && !line.startsWith('@')) { + pendingMultilineAttributeIndex = null; + pendingAttributeParenthesisDepth = 0; + } else if ( + line.length > 0 && + !line.startsWith('@') && + !consumedAttributeContinuation && + pendingMultilineAttributeIndex === null + ) { pendingAttributes = []; } diff --git a/src/utils/test-common.ts b/src/utils/test-common.ts index 759ce669..e22fe6b7 100644 --- a/src/utils/test-common.ts +++ b/src/utils/test-common.ts @@ -153,6 +153,14 @@ export function createTestExecutor( }); } + started.pipeline.emitFragment({ + kind: 'test-result', + fragment: 'build-stage', + operation: 'TEST', + stage: 'RUN_TESTS', + message: 'Running tests', + }); + const testWithoutBuildingResult = await executeXcodeBuildCommand( { ...params, extraArgs: executionPlan.testArgs }, platformOptions, diff --git a/src/utils/xcodebuild-event-parser.ts b/src/utils/xcodebuild-event-parser.ts index e32f5f39..17c5d736 100644 --- a/src/utils/xcodebuild-event-parser.ts +++ b/src/utils/xcodebuild-event-parser.ts @@ -33,7 +33,10 @@ function resolveStageFromLine(line: string): XcodebuildStage | null { if ( /^Testing started$/u.test(line) || /^Test [Ss]uite .+ started/u.test(line) || - /^[◇] Test run started/u.test(line) + /^Test [Cc]ase .+ started/u.test(line) || + /^[◇] Test run started/u.test(line) || + /^[◇] Test .+ started/u.test(line) || + /^[◇] Test case .+ started/u.test(line) ) { return 'RUN_TESTS'; } @@ -88,6 +91,10 @@ function isIgnoredNoiseLine(line: string): boolean { return IGNORED_NOISE_PATTERNS.some((pattern) => pattern.test(line)); } +function normalizeEventLine(rawLine: string): string { + return rawLine.trim().replace(/^(?:\u200B|\u200C|\u200D|\uFEFF)+/u, ''); +} + export interface EventParserOptions { operation: XcodebuildOperation; kind?: BuildLikeKind; @@ -112,6 +119,8 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb let completedCount = 0; let failedCount = 0; let skippedCount = 0; + let swiftTestingCompletedSinceSummary = 0; + let swiftTestingFailedSinceSummary = 0; let detectedXcresultPath: string | null = null; let pendingError: { @@ -224,7 +233,10 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb }); } - function recordTestCaseResult(testCase: ParsedTestCase): void { + function recordTestCaseResult( + testCase: ParsedTestCase, + source: 'xcodebuild' | 'swift-testing' = 'xcodebuild', + ): void { const increment = testCase.caseCount ?? 1; completedCount += increment; const durationMs = parseDurationMs(testCase.durationText); @@ -236,6 +248,13 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb skippedCount += increment; } + if (source === 'swift-testing') { + swiftTestingCompletedSinceSummary += increment; + if (testCase.status === 'failed') { + swiftTestingFailedSinceSummary += increment; + } + } + if (operation === 'TEST') { onEvent({ kind: 'test-result', @@ -267,7 +286,7 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb } function processLine(rawLine: string): void { - const line = rawLine.trim(); + const line = normalizeEventLine(rawLine); if (!line) { flushPendingError(); return; @@ -324,14 +343,18 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb const stResult = parseSwiftTestingResultLine(line); if (stResult) { - recordTestCaseResult(stResult); + recordTestCaseResult(stResult, 'swift-testing'); return; } const stSummary = parseSwiftTestingRunSummary(line); if (stSummary) { - completedCount = Math.max(completedCount, stSummary.executed); - failedCount = Math.max(failedCount, stSummary.failed); + const failedFromSummary = + swiftTestingFailedSinceSummary > 0 ? swiftTestingFailedSinceSummary : stSummary.failed; + completedCount += stSummary.executed - swiftTestingCompletedSinceSummary; + failedCount += failedFromSummary - swiftTestingFailedSinceSummary; + swiftTestingCompletedSinceSummary = 0; + swiftTestingFailedSinceSummary = 0; emitTestProgress(); return; } diff --git a/src/utils/xcodebuild-line-parsers.ts b/src/utils/xcodebuild-line-parsers.ts index 3e81fb40..99072d6c 100644 --- a/src/utils/xcodebuild-line-parsers.ts +++ b/src/utils/xcodebuild-line-parsers.ts @@ -88,7 +88,9 @@ export function parseRawTestName(rawName: string): { suiteName?: string; testNam } export function parseTestCaseLine(line: string): ParsedTestCase | null { - const match = line.match(/^Test Case '(.+)' (passed|failed|skipped) \(([^)]+)\)/u); + const match = line.match( + /^Test [Cc]ase '(.+)' (passed|failed|skipped)(?: on '.+')? \(([^)]+)\)/u, + ); if (!match) { return null; } From 3b14b4026595e8536f79a623dce7dde8df182521 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 2 May 2026 17:41:01 +0000 Subject: [PATCH 2/6] Remove unreachable parseXcodebuildSwiftTestingLine dead code The updated parseTestCaseLine regex now matches both 'Test Case' and 'Test case' with an optional 'on' clause, making it strictly more permissive than parseXcodebuildSwiftTestingLine. Since parseTestCaseLine is checked first in processLine, parseXcodebuildSwiftTestingLine became unreachable dead code. --- .../swift-testing-line-parsers.test.ts | 39 ------------------- src/utils/swift-testing-line-parsers.ts | 26 ------------- src/utils/xcodebuild-event-parser.ts | 7 ---- 3 files changed, 72 deletions(-) diff --git a/src/utils/__tests__/swift-testing-line-parsers.test.ts b/src/utils/__tests__/swift-testing-line-parsers.test.ts index 4b49258c..c810cba1 100644 --- a/src/utils/__tests__/swift-testing-line-parsers.test.ts +++ b/src/utils/__tests__/swift-testing-line-parsers.test.ts @@ -4,7 +4,6 @@ import { parseSwiftTestingIssueLine, parseSwiftTestingRunSummary, parseSwiftTestingContinuationLine, - parseXcodebuildSwiftTestingLine, } from '../swift-testing-line-parsers.ts'; describe('Swift Testing line parsers', () => { @@ -259,42 +258,4 @@ describe('Swift Testing line parsers', () => { expect(parseSwiftTestingContinuationLine('regular line')).toBeNull(); }); }); - - describe('parseXcodebuildSwiftTestingLine', () => { - it('should parse a passed test case', () => { - const result = parseXcodebuildSwiftTestingLine( - "Test case 'MCPTestTests/appNameIsCorrect()' passed on 'My Mac - MCPTest (78757)' (0.000 seconds)", - ); - expect(result).toEqual({ - status: 'passed', - rawName: 'MCPTestTests/appNameIsCorrect()', - suiteName: 'MCPTestTests', - testName: 'appNameIsCorrect()', - durationText: '0.000s', - }); - }); - - it('should parse a failed test case', () => { - const result = parseXcodebuildSwiftTestingLine( - "Test case 'MCPTestTests/deliberateFailure()' failed on 'My Mac - MCPTest (78757)' (0.000 seconds)", - ); - expect(result).toEqual({ - status: 'failed', - rawName: 'MCPTestTests/deliberateFailure()', - suiteName: 'MCPTestTests', - testName: 'deliberateFailure()', - durationText: '0.000s', - }); - }); - - it('should return null for XCTest format lines', () => { - expect( - parseXcodebuildSwiftTestingLine("Test Case '-[Suite test]' passed (0.001 seconds)."), - ).toBeNull(); - }); - - it('should return null for non-matching lines', () => { - expect(parseXcodebuildSwiftTestingLine('random text')).toBeNull(); - }); - }); }); diff --git a/src/utils/swift-testing-line-parsers.ts b/src/utils/swift-testing-line-parsers.ts index 1ee463e2..dddf20ef 100644 --- a/src/utils/swift-testing-line-parsers.ts +++ b/src/utils/swift-testing-line-parsers.ts @@ -176,29 +176,3 @@ export function parseSwiftTestingContinuationLine(line: string): string | null { const match = line.match(/^↳ (.+)$/u); return match ? match[1] : null; } - -/** - * Parse xcodebuild's Swift Testing format. - * - * Matches: - * Test case 'Suite/testName()' passed on 'My Mac - App (12345)' (0.001 seconds) - * Test case 'Suite/testName()' failed on 'My Mac - App (12345)' (0.001 seconds) - */ -export function parseXcodebuildSwiftTestingLine(line: string): ParsedTestCase | null { - const match = line.match( - /^Test case '(.+)' (passed|failed|skipped) on '.+' \(([^)]+) seconds?\)$/u, - ); - if (!match) { - return null; - } - const [, rawName, status, duration] = match; - const { suiteName, testName } = parseRawTestName(rawName); - - return { - status: status as 'passed' | 'failed' | 'skipped', - rawName, - suiteName, - testName, - durationText: `${duration}s`, - }; -} diff --git a/src/utils/xcodebuild-event-parser.ts b/src/utils/xcodebuild-event-parser.ts index 17c5d736..045342f6 100644 --- a/src/utils/xcodebuild-event-parser.ts +++ b/src/utils/xcodebuild-event-parser.ts @@ -13,7 +13,6 @@ import { type ParsedTestCase, } from './xcodebuild-line-parsers.ts'; import { - parseXcodebuildSwiftTestingLine, parseSwiftTestingIssueLine, parseSwiftTestingResultLine, parseSwiftTestingRunSummary, @@ -329,12 +328,6 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb return; } - const xcodebuildST = parseXcodebuildSwiftTestingLine(line); - if (xcodebuildST) { - recordTestCaseResult(xcodebuildST); - return; - } - const stIssue = parseSwiftTestingIssueLine(line); if (stIssue) { queueFailureDiagnostic(stIssue); From 5b14997a19f185fa2704d71368430b668d547c13 Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 2 May 2026 18:46:53 +0100 Subject: [PATCH 3/6] fix(test): Reconcile Swift Testing summaries monotonically Keep Swift Testing summary reconciliation from decrementing progress or re-counting xcodebuild-formatted per-test lines when summaries arrive later. Count summary-only failures as additional observed failures. Remove the now-unreachable xcodebuild Swift Testing parser path because the generic test-case parser handles destination-suffixed xcodebuild lines. Refs #384 Co-Authored-By: OpenAI Codex --- .../__tests__/xcodebuild-event-parser.test.ts | 66 ++++++++++++++++++- src/utils/xcodebuild-event-parser.ts | 25 ++++--- 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/src/utils/__tests__/xcodebuild-event-parser.test.ts b/src/utils/__tests__/xcodebuild-event-parser.test.ts index 122270e4..eeeb9cea 100644 --- a/src/utils/__tests__/xcodebuild-event-parser.test.ts +++ b/src/utils/__tests__/xcodebuild-event-parser.test.ts @@ -593,13 +593,75 @@ describe('xcodebuild-event-parser', () => { expect(summary).toMatchObject({ fragment: 'build-summary', operation: 'TEST', - totalTests: 2, - passedTests: 2, + totalTests: 5, + passedTests: 5, failedTests: 0, skippedTests: 0, }); }); + it('keeps Swift Testing summary progress monotonic when per-test lines exceed the summary', () => { + const events = collectEvents('TEST', [ + { + source: 'stdout', + text: '✔ Test "First observed case" passed after 0.001 seconds.\n', + }, + { + source: 'stdout', + text: '✔ Test "Second observed case" passed after 0.001 seconds.\n', + }, + { + source: 'stdout', + text: '✔ Test run with 1 test in 1 suite passed after 0.001 seconds.\n', + }, + ]); + + const progress = events.filter((event) => event.fragment === 'test-progress'); + expect(progress).toEqual([ + expect.objectContaining({ completed: 1, failed: 0, skipped: 0 }), + expect.objectContaining({ completed: 2, failed: 0, skipped: 0 }), + expect.objectContaining({ completed: 2, failed: 0, skipped: 0 }), + ]); + }); + + it('does not double-count xcodebuild-formatted test lines when a Swift Testing summary follows', () => { + const events = collectEvents('TEST', [ + { + source: 'stdout', + text: "Test case 'WeatherTests/emptySearchReturnsNoResults()' passed on 'Clone 1' (0.001 seconds)\n", + }, + { + source: 'stdout', + text: '✔ Test run with 1 test in 1 suite passed after 0.001 seconds.\n', + }, + ]); + + const progress = events.filter((event) => event.fragment === 'test-progress'); + expect(progress).toEqual([ + expect.objectContaining({ completed: 1, failed: 0, skipped: 0 }), + expect.objectContaining({ completed: 1, failed: 0, skipped: 0 }), + ]); + }); + + it('counts additional failures reported only by a Swift Testing summary', () => { + const events = collectEvents('TEST', [ + { + source: 'stdout', + text: '✘ Test "Individually reported failure" failed after 0.001 seconds with 1 issue.\n', + }, + { + source: 'stdout', + text: '✘ Test run with 2 tests in 1 suite failed after 0.001 seconds with 2 issues.\n', + }, + ]); + + const progress = events.filter((event) => event.fragment === 'test-progress'); + expect(progress).toEqual([ + expect.objectContaining({ completed: 1, failed: 1, skipped: 0 }), + expect.objectContaining({ completed: 2, failed: 2, skipped: 0 }), + ]); + }); + it('processes full test lifecycle', () => { const events = collectEvents('TEST', [ { source: 'stdout', text: 'Resolve Package Graph\n' }, diff --git a/src/utils/xcodebuild-event-parser.ts b/src/utils/xcodebuild-event-parser.ts index 045342f6..e4cc1625 100644 --- a/src/utils/xcodebuild-event-parser.ts +++ b/src/utils/xcodebuild-event-parser.ts @@ -118,8 +118,8 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb let completedCount = 0; let failedCount = 0; let skippedCount = 0; - let swiftTestingCompletedSinceSummary = 0; - let swiftTestingFailedSinceSummary = 0; + let testCasesCompletedSinceSwiftTestingSummary = 0; + let testCasesFailedSinceSwiftTestingSummary = 0; let detectedXcresultPath: string | null = null; let pendingError: { @@ -247,11 +247,9 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb skippedCount += increment; } - if (source === 'swift-testing') { - swiftTestingCompletedSinceSummary += increment; - if (testCase.status === 'failed') { - swiftTestingFailedSinceSummary += increment; - } + testCasesCompletedSinceSwiftTestingSummary += increment; + if (testCase.status === 'failed') { + testCasesFailedSinceSwiftTestingSummary += increment; } if (operation === 'TEST') { @@ -342,12 +340,13 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb const stSummary = parseSwiftTestingRunSummary(line); if (stSummary) { - const failedFromSummary = - swiftTestingFailedSinceSummary > 0 ? swiftTestingFailedSinceSummary : stSummary.failed; - completedCount += stSummary.executed - swiftTestingCompletedSinceSummary; - failedCount += failedFromSummary - swiftTestingFailedSinceSummary; - swiftTestingCompletedSinceSummary = 0; - swiftTestingFailedSinceSummary = 0; + completedCount += Math.max( + 0, + stSummary.executed - testCasesCompletedSinceSwiftTestingSummary, + ); + failedCount += Math.max(0, stSummary.failed - testCasesFailedSinceSwiftTestingSummary); + testCasesCompletedSinceSwiftTestingSummary = 0; + testCasesFailedSinceSwiftTestingSummary = 0; emitTestProgress(); return; } From 24a295a637fdd2a2be4455686fab8baca453d88d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 2 May 2026 18:01:18 +0000 Subject: [PATCH 4/6] Fix XCTest/Swift Testing miscount by gating counter increments with source parameter The recordTestCaseResult function now only increments testCasesCompletedSinceSwiftTestingSummary and testCasesFailedSinceSwiftTestingSummary when source is 'swift-testing', preventing XCTest results from being incorrectly subtracted from Swift Testing summary counts. --- src/utils/xcodebuild-event-parser.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/utils/xcodebuild-event-parser.ts b/src/utils/xcodebuild-event-parser.ts index e4cc1625..ae165cdf 100644 --- a/src/utils/xcodebuild-event-parser.ts +++ b/src/utils/xcodebuild-event-parser.ts @@ -247,9 +247,11 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb skippedCount += increment; } - testCasesCompletedSinceSwiftTestingSummary += increment; - if (testCase.status === 'failed') { - testCasesFailedSinceSwiftTestingSummary += increment; + if (source === 'swift-testing') { + testCasesCompletedSinceSwiftTestingSummary += increment; + if (testCase.status === 'failed') { + testCasesFailedSinceSwiftTestingSummary += increment; + } } if (operation === 'TEST') { From 3038cacbd6533dff87f6915ba6884ce38e53cd67 Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 2 May 2026 19:03:00 +0100 Subject: [PATCH 5/6] fix(test): Count parameterized Swift Testing results once Swift Testing result lines can include parameterized case metadata such as "with 4 test cases", but Xcode's discovery and run summaries count that output as one reported test item. Count each result line once so CLI progress and final summaries stay aligned with discovered tests. Fixes GH-384 Co-Authored-By: OpenAI Codex --- .../__tests__/xcodebuild-event-parser.test.ts | 35 +++++++++++++++---- src/utils/xcodebuild-event-parser.ts | 2 +- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/utils/__tests__/xcodebuild-event-parser.test.ts b/src/utils/__tests__/xcodebuild-event-parser.test.ts index eeeb9cea..ab5cb10c 100644 --- a/src/utils/__tests__/xcodebuild-event-parser.test.ts +++ b/src/utils/__tests__/xcodebuild-event-parser.test.ts @@ -593,8 +593,8 @@ describe('xcodebuild-event-parser', () => { expect(summary).toMatchObject({ fragment: 'build-summary', operation: 'TEST', - totalTests: 5, - passedTests: 5, + totalTests: 2, + passedTests: 2, failedTests: 0, skippedTests: 0, }); @@ -624,7 +624,7 @@ describe('xcodebuild-event-parser', () => { ]); }); - it('does not double-count xcodebuild-formatted test lines when a Swift Testing summary follows', () => { + it('keeps xcodebuild-formatted test lines independent from Swift Testing summaries', () => { const events = collectEvents('TEST', [ { source: 'stdout', @@ -639,7 +639,7 @@ describe('xcodebuild-event-parser', () => { const progress = events.filter((event) => event.fragment === 'test-progress'); expect(progress).toEqual([ expect.objectContaining({ completed: 1, failed: 0, skipped: 0 }), - expect.objectContaining({ completed: 1, failed: 0, skipped: 0 }), + expect.objectContaining({ completed: 2, failed: 0, skipped: 0 }), ]); }); @@ -662,6 +662,29 @@ describe('xcodebuild-event-parser', () => { ]); }); + it('keeps parameterized Swift Testing result counts aligned with the run summary', () => { + const events = collectRunStateEvents([ + { + source: 'stdout', + text: '✔ Test "Parameterized test" with 4 test cases passed after 0.001 seconds.\n', + }, + { + source: 'stdout', + text: '✔ Test run with 1 test in 1 suite passed after 0.001 seconds.\n', + }, + ]); + + const summary = events.filter((event) => event.fragment === 'build-summary').at(-1); + expect(summary).toMatchObject({ + fragment: 'build-summary', + operation: 'TEST', + totalTests: 1, + passedTests: 1, + failedTests: 0, + skippedTests: 0, + }); + }); + it('processes full test lifecycle', () => { const events = collectEvents('TEST', [ { source: 'stdout', text: 'Resolve Package Graph\n' }, @@ -684,7 +707,7 @@ describe('xcodebuild-event-parser', () => { expect(fragments).toContain('test-failure'); }); - it('increments counts by caseCount for parameterized Swift Testing results', () => { + it('counts parameterized Swift Testing result lines once for progress', () => { const events = collectEvents('TEST', [ { source: 'stdout', @@ -695,7 +718,7 @@ describe('xcodebuild-event-parser', () => { const progress = events.filter((e) => e.fragment === 'test-progress'); expect(progress).toHaveLength(1); if (progress[0].fragment === 'test-progress') { - expect(progress[0].completed).toBe(3); + expect(progress[0].completed).toBe(1); } }); diff --git a/src/utils/xcodebuild-event-parser.ts b/src/utils/xcodebuild-event-parser.ts index ae165cdf..e1d6828c 100644 --- a/src/utils/xcodebuild-event-parser.ts +++ b/src/utils/xcodebuild-event-parser.ts @@ -236,7 +236,7 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb testCase: ParsedTestCase, source: 'xcodebuild' | 'swift-testing' = 'xcodebuild', ): void { - const increment = testCase.caseCount ?? 1; + const increment = 1; completedCount += increment; const durationMs = parseDurationMs(testCase.durationText); From 82d27ddd5e0a8725cfa4e94a900b65fadc1aad59 Mon Sep 17 00:00:00 2001 From: Cameron Cooke Date: Sat, 2 May 2026 19:21:07 +0100 Subject: [PATCH 6/6] fix(test): Reconcile xcodebuild Swift Testing lines Lowercase xcodebuild test case lines can represent Swift Testing results when they use slash-separated Swift test identifiers. Treat those lines as Swift Testing observations so a following Swift Testing run summary does not count the same tests again. Fixes GH-384 Co-Authored-By: OpenAI Codex --- .../__tests__/xcodebuild-event-parser.test.ts | 23 +++++++++++++++++-- src/utils/xcodebuild-event-parser.ts | 6 ++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/utils/__tests__/xcodebuild-event-parser.test.ts b/src/utils/__tests__/xcodebuild-event-parser.test.ts index ab5cb10c..f9cfa5e4 100644 --- a/src/utils/__tests__/xcodebuild-event-parser.test.ts +++ b/src/utils/__tests__/xcodebuild-event-parser.test.ts @@ -624,11 +624,11 @@ describe('xcodebuild-event-parser', () => { ]); }); - it('keeps xcodebuild-formatted test lines independent from Swift Testing summaries', () => { + it('keeps XCTest-style test case lines independent from Swift Testing summaries', () => { const events = collectEvents('TEST', [ { source: 'stdout', - text: "Test case 'WeatherTests/emptySearchReturnsNoResults()' passed on 'Clone 1' (0.001 seconds)\n", + text: "Test case 'WeatherUITests.testSearch()' passed on 'Clone 1' (0.001 seconds)\n", }, { source: 'stdout', @@ -643,6 +643,25 @@ describe('xcodebuild-event-parser', () => { ]); }); + it('does not double-count xcodebuild-formatted Swift Testing lines before a summary', () => { + const events = collectEvents('TEST', [ + { + source: 'stdout', + text: "Test case 'WeatherTests/emptySearchReturnsNoResults()' passed on 'Clone 1' (0.001 seconds)\n", + }, + { + source: 'stdout', + text: '✔ Test run with 1 test in 1 suite passed after 0.001 seconds.\n', + }, + ]); + + const progress = events.filter((event) => event.fragment === 'test-progress'); + expect(progress).toEqual([ + expect.objectContaining({ completed: 1, failed: 0, skipped: 0 }), + expect.objectContaining({ completed: 1, failed: 0, skipped: 0 }), + ]); + }); + it('counts additional failures reported only by a Swift Testing summary', () => { const events = collectEvents('TEST', [ { diff --git a/src/utils/xcodebuild-event-parser.ts b/src/utils/xcodebuild-event-parser.ts index e1d6828c..2698be05 100644 --- a/src/utils/xcodebuild-event-parser.ts +++ b/src/utils/xcodebuild-event-parser.ts @@ -310,7 +310,11 @@ export function createXcodebuildEventParser(options: EventParserOptions): Xcodeb const testCase = parseTestCaseLine(line); if (testCase) { - recordTestCaseResult(testCase); + const source = + /^Test case /u.test(line) && /\/.+\(\)$/u.test(testCase.rawName) + ? 'swift-testing' + : 'xcodebuild'; + recordTestCaseResult(testCase, source); return; }