From cf09fc725bccdb553d8d210f8a47223ebee67046 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 2 May 2026 14:34:45 -0400 Subject: [PATCH 1/4] Add farm view, stream controls, and native HID fallback --- README.md | 12 + cli/DFPrivateSimulatorDisplayBridge.m | 173 +++++++++- client/src/app/App.tsx | 4 + client/src/features/farm/FarmView.tsx | 310 +++++++++++++++++ .../src/features/stream/streamWorkerClient.ts | 31 +- client/src/features/stream/useLiveStream.ts | 27 ++ client/src/features/toolbar/Toolbar.tsx | 3 + client/src/styles/layout.css | 323 ++++++++++++++++++ docs/api/rest.md | 12 + docs/cli/commands.md | 8 + docs/extensions/browser-client.md | 7 + server/src/main.rs | 38 +++ server/src/transport/webrtc.rs | 90 ++++- skills/simdeck/SKILL.md | 6 +- 14 files changed, 1029 insertions(+), 15 deletions(-) create mode 100644 client/src/features/farm/FarmView.tsx diff --git a/README.md b/README.md index c5050c5b..8f99df10 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ view inside the editor. ## Features - Local simulator video stream over browser-native WebRTC H.264 +- Multi-simulator farm view at `/farm` with low-rate thumbnails and a focused full-rate stream - Full simulator control & inspection using private accessibility APIs - CoreSimulator chrome asset rendering for device bezels - NativeScript, React Native, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live @@ -62,6 +63,9 @@ simdeck "iPhone 17 Pro Max" Use `simdeck ui --open` or `simdeck daemon start` when you want a reusable background daemon instead. The no-subcommand lifecycle shortcuts are `simdeck -d` for detached start, `simdeck -k` to kill the background daemon, and `simdeck -r` to restart it. The served loopback browser UI receives the generated API access token automatically. LAN browsers pair with the printed code before receiving the API cookie. +Open `http://127.0.0.1:4310/farm` to monitor every simulator in one dashboard. +The farm uses thumbnail stream profiles for the grid and promotes the focused +simulator to a full-rate WebRTC stream. SimDeck Studio providers run the daemon on loopback and use `scripts/studio-provider-bridge.mjs` for outbound control-plane communication @@ -152,6 +156,7 @@ simdeck toggle-appearance simdeck pasteboard set "hello" simdeck pasteboard get simdeck screenshot --output screen.png +simdeck stream --frames 120 > stream.h264 simdeck describe simdeck describe --format agent --max-depth 4 simdeck describe --point 120,240 @@ -179,6 +184,13 @@ simdeck chrome-profile simdeck logs --seconds 30 --limit 200 ``` +`boot` prefers SimDeck's private CoreSimulator boot path so it can start devices +without launching Simulator.app, then falls back to `xcrun simctl` when private +booting is unavailable. + +`stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or +external tools such as `ffplay`. + `describe` uses the project daemon to prefer React Native, NativeScript, or UIKit in-app inspectors, then falls back to the built-in private CoreSimulator accessibility bridge. Use `--format agent` or `--format compact-json` for diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 77948435..9ae02e9b 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -32,10 +32,12 @@ typedef uint32_t IndigoHIDEdge; typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEventFn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, NSEventType type, NSSize displaySize, IndigoHIDEdge edge); +typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEvent9Fn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, uint32_t eventType, uint32_t direction, double unused1, double unused2, double widthPoints, double heightPoints); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardArbitraryFn)(int keyCode, int op); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardNSEventFn)(NSEvent *event); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForButtonFn)(uint32_t buttonCode, uint32_t operation, uint32_t target); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForHIDArbitraryFn)(uint32_t target, uint32_t page, uint32_t usage, uint32_t operation); +typedef IndigoHIDMessage *(*DFIndigoHIDServiceMessageFn)(void); #pragma pack(push, 4) typedef struct { @@ -91,6 +93,12 @@ static const uint32_t DFIndigoTouchTarget = 0x32; static const uint8_t DFIndigoEventTypeTouch = 0x02; static const uint32_t DFIndigoTouchEventKind = 0x0b; +static const uint32_t DFIndigoMouseEventDown = 1; +static const uint32_t DFIndigoMouseEventUp = 2; +static const uint32_t DFIndigoMouseEventDragged = 6; +static const uint32_t DFIndigoMouseDirectionDown = 1; +static const uint32_t DFIndigoMouseDirectionMove = 0; +static const uint32_t DFIndigoMouseDirectionUp = 2; static const int DFKeyboardDirectionDown = 1; static const int DFKeyboardDirectionUp = 2; static const uint32_t DFButtonDirectionDown = 1; @@ -187,6 +195,36 @@ static BOOL DFVerboseTouchLoggingEnabled(void) { return enabled; } +static BOOL DFShouldUseIndigoMouse9Path(void) { + static BOOL enabled = NO; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSString *override = NSProcessInfo.processInfo.environment[@"SIMDECK_INDIGO_MOUSE_9ARG"]; + if (override.length > 0) { + enabled = [override isEqualToString:@"1"] || + [override caseInsensitiveCompare:@"true"] == NSOrderedSame || + [override caseInsensitiveCompare:@"yes"] == NSOrderedSame; + return; + } + + FILE *pipe = popen("/usr/bin/xcodebuild -version 2>/dev/null", "r"); + if (pipe == NULL) { + return; + } + char buffer[256] = {0}; + if (fgets(buffer, sizeof(buffer), pipe) != NULL) { + NSString *line = [[NSString stringWithUTF8String:buffer] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + NSArray *parts = [line componentsSeparatedByString:@" "]; + if (parts.count >= 2 && [parts[0] isEqualToString:@"Xcode"]) { + NSInteger major = [[parts[1] componentsSeparatedByString:@"."].firstObject integerValue]; + enabled = major >= 26; + } + } + pclose(pipe); + }); + return enabled; +} + #pragma mark - SimulatorKit Swift symbol resolver // // We call into SimulatorKit's private Swift API by dlsym'ing mangled symbol @@ -1002,6 +1040,109 @@ static BOOL DFSendHIDMessage(id hidClient, IndigoHIDMessage *message, BOOL freeW return YES; } +static uint32_t DFIndigoMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phase) { + switch (phase) { + case DFPrivateSimulatorTouchPhaseBegan: + return DFIndigoMouseEventDown; + case DFPrivateSimulatorTouchPhaseMoved: + return DFIndigoMouseEventDragged; + case DFPrivateSimulatorTouchPhaseEnded: + case DFPrivateSimulatorTouchPhaseCancelled: + return DFIndigoMouseEventUp; + } +} + +static uint32_t DFIndigoMouseDirectionForPhase(DFPrivateSimulatorTouchPhase phase) { + switch (phase) { + case DFPrivateSimulatorTouchPhaseBegan: + return DFIndigoMouseDirectionDown; + case DFPrivateSimulatorTouchPhaseMoved: + return DFIndigoMouseDirectionMove; + case DFPrivateSimulatorTouchPhaseEnded: + case DFPrivateSimulatorTouchPhaseCancelled: + return DFIndigoMouseDirectionUp; + } +} + +static IndigoHIDMessage *DFCreateIndigoTouchMessage9(CGPoint normalizedPoint, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) { + if (!DFShouldUseIndigoMouse9Path()) { + return NULL; + } + DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent"); + if (mouseMessage == NULL) { + return NULL; + } + + CGPoint ratioPoint = CGPointMake( + fmax(0.0, fmin(1.0, normalizedPoint.x)), + fmax(0.0, fmin(1.0, normalizedPoint.y)) + ); + return mouseMessage(&ratioPoint, + NULL, + DFIndigoTouchTarget, + DFIndigoMouseEventTypeForPhase(phase), + DFIndigoMouseDirectionForPhase(phase), + 1.0, + 1.0, + displaySize.width, + displaySize.height); +} + +static IndigoHIDMessage *DFCreateIndigoMultiTouchMessage9(CGPoint normalizedPoint1, CGPoint normalizedPoint2, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) { + if (!DFShouldUseIndigoMouse9Path()) { + return NULL; + } + DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent"); + if (mouseMessage == NULL) { + return NULL; + } + + CGPoint ratioPoint = CGPointMake( + fmax(0.0, fmin(1.0, normalizedPoint1.x)), + fmax(0.0, fmin(1.0, normalizedPoint1.y)) + ); + CGPoint secondRatioPoint = CGPointMake( + fmax(0.0, fmin(1.0, normalizedPoint2.x)), + fmax(0.0, fmin(1.0, normalizedPoint2.y)) + ); + return mouseMessage(&ratioPoint, + &secondRatioPoint, + DFIndigoTouchTarget, + DFIndigoMouseEventTypeForPhase(phase), + DFIndigoMouseDirectionForPhase(phase), + 1.0, + 1.0, + displaySize.width, + displaySize.height); +} + +static void DFWarmIndigoHIDServices(id hidClient) { + if (hidClient == nil) { + return; + } + + DFIndigoHIDServiceMessageFn createPointer = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreatePointerService"); + DFIndigoHIDServiceMessageFn createMouse = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreateMouseService"); + NSError *error = nil; + if (createPointer != NULL) { + IndigoHIDMessage *message = createPointer(); + if (message != NULL) { + (void)DFSendHIDMessage(hidClient, message, YES, &error); + usleep(20 * 1000); + } + } + if (createMouse != NULL) { + IndigoHIDMessage *message = createMouse(); + if (message != NULL) { + (void)DFSendHIDMessage(hidClient, message, YES, &error); + usleep(20 * 1000); + } + } + if (error != nil) { + DFLog(@"Indigo HID service warm-up reported: %@", error.localizedDescription ?: @"unknown error"); + } +} + static DFIndigoMessage *DFCreateIndigoTouchMessage(CGPoint normalizedPoint, NSSize displaySize, BOOL touchDown, NSError **error) { DFIndigoHIDMessageForMouseNSEventFn mouseMessage = (DFIndigoHIDMessageForMouseNSEventFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent"); if (mouseMessage == NULL) { @@ -2378,6 +2519,7 @@ - (nullable instancetype)initWithUDID:(NSString *)udid if (_hidClient != nil) { DFLog(@"Created private SimulatorKit HID client for %@", udid); + DFWarmIndigoHIDServices(_hidClient); } else { DFLog(@"Failed to create private SimulatorKit HID client for %@: %@", udid, hidClientError.localizedDescription ?: @"unknown error"); } @@ -3311,14 +3453,18 @@ - (BOOL)sendTouchAtNormalizedX:(double)normalizedX phaseLabel = phase == DFPrivateSimulatorTouchPhaseEnded ? @"ended" : @"cancelled"; break; } - BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; - DFIndigoMessage *message = DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError); + IndigoHIDMessage *message = DFCreateIndigoTouchMessage9(CGPointMake(clampedX, clampedY), displaySize, phase); + BOOL freeWhenDone = YES; if (message == NULL) { - return; + BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; + message = (IndigoHIDMessage *)DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError); + if (message == NULL) { + return; + } } NSError *messageError = nil; - if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) { + if (!DFSendHIDMessage(self->_hidClient, message, freeWhenDone, &messageError)) { dispatchError = messageError ?: DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @"SimulatorKit rejected the Indigo HID touch packet." @@ -3394,14 +3540,25 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1 break; } - BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; - DFIndigoMessage *message = DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError); + IndigoHIDMessage *message = NULL; + const NSUInteger maxAttempts = phase == DFPrivateSimulatorTouchPhaseMoved ? 12 : 3; + for (NSUInteger attempt = 0; attempt < maxAttempts; attempt++) { + message = DFCreateIndigoMultiTouchMessage9(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, phase); + if (message != NULL) { + break; + } + usleep(5 * 1000); + } if (message == NULL) { - return; + BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; + message = (IndigoHIDMessage *)DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError); + if (message == NULL) { + return; + } } NSError *messageError = nil; - if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) { + if (!DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) { dispatchError = messageError ?: DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @"SimulatorKit rejected the Indigo HID multi-touch packet." diff --git a/client/src/app/App.tsx b/client/src/app/App.tsx index 75c76333..f0b133c0 100644 --- a/client/src/app/App.tsx +++ b/client/src/app/App.tsx @@ -1,5 +1,9 @@ import { AppShell } from "./AppShell"; +import { FarmView } from "../features/farm/FarmView"; export default function App() { + if (window.location.pathname === "/farm") { + return ; + } return ; } diff --git a/client/src/features/farm/FarmView.tsx b/client/src/features/farm/FarmView.tsx new file mode 100644 index 00000000..385c5d73 --- /dev/null +++ b/client/src/features/farm/FarmView.tsx @@ -0,0 +1,310 @@ +import { useCallback, useMemo, useState } from "react"; + +import { accessTokenFromLocation } from "../../api/client"; +import { apiUrl } from "../../api/config"; +import { bootSimulator, shutdownSimulator } from "../../api/controls"; +import type { SimulatorMetadata } from "../../api/types"; +import { simulatorRuntimeLabel } from "../simulators/simulatorDisplay"; +import { useSimulatorList } from "../simulators/useSimulatorList"; +import { useLiveStream } from "../stream/useLiveStream"; + +type FarmViewMode = "grid" | "list" | "wall"; + +function simulatorFamily(simulator: SimulatorMetadata): string { + const text = `${simulator.name} ${simulator.deviceTypeName ?? ""}`.toLowerCase(); + if (text.includes("ipad")) return "iPad"; + if (text.includes("watch")) return "Watch"; + if (text.includes("tv")) return "TV"; + return "iPhone"; +} + +function screenshotUrl(udid: string): string { + const url = new URL( + apiUrl(`/api/simulators/${encodeURIComponent(udid)}/screenshot.png`), + window.location.href, + ); + const token = accessTokenFromLocation(); + if (token) { + url.searchParams.set("simdeckToken", token); + } + url.searchParams.set("stamp", String(Date.now())); + return url.toString(); +} + +export function FarmView() { + const { isLoading, refresh, simulators } = useSimulatorList(); + const [search, setSearch] = useState(""); + const [family, setFamily] = useState("all"); + const [state, setState] = useState("all"); + const [view, setView] = useState("grid"); + const [selectedUDID, setSelectedUDID] = useState(""); + const [busyUDID, setBusyUDID] = useState(""); + + const filtered = useMemo(() => { + const needle = search.trim().toLowerCase(); + return simulators + .filter((simulator) => { + if (family !== "all" && simulatorFamily(simulator) !== family) { + return false; + } + if (state === "booted" && !simulator.isBooted) { + return false; + } + if (state === "shutdown" && simulator.isBooted) { + return false; + } + if (!needle) { + return true; + } + return [ + simulator.name, + simulator.udid, + simulatorRuntimeLabel(simulator), + simulator.deviceTypeName, + ] + .filter(Boolean) + .some((value) => value!.toLowerCase().includes(needle)); + }) + .sort((a, b) => Number(b.isBooted) - Number(a.isBooted) || a.name.localeCompare(b.name)); + }, [family, search, simulators, state]); + + const selectedSimulator = + filtered.find((simulator) => simulator.udid === selectedUDID) ?? + filtered.find((simulator) => simulator.isBooted) ?? + filtered[0] ?? + null; + + async function runLifecycle( + simulator: SimulatorMetadata, + action: "boot" | "shutdown", + ) { + setBusyUDID(simulator.udid); + try { + if (action === "boot") { + await bootSimulator(simulator.udid); + setSelectedUDID(simulator.udid); + } else { + await shutdownSimulator(simulator.udid); + } + await refresh(); + } finally { + setBusyUDID(""); + } + } + + return ( +
+
+
+ SimDeck Farm + + {filtered.filter((simulator) => simulator.isBooted).length} live /{" "} + {filtered.length} shown + +
+
+ setSearch(event.target.value)} + placeholder="Search simulators" + value={search} + /> + + +
+ {(["grid", "wall", "list"] as FarmViewMode[]).map((mode) => ( + + ))} +
+ + Single + +
+
+
+
+ {filtered.map((simulator) => ( + void runLifecycle(simulator, "boot")} + onSelect={() => setSelectedUDID(simulator.udid)} + onShutdown={() => void runLifecycle(simulator, "shutdown")} + simulator={simulator} + view={view} + /> + ))} + {!isLoading && filtered.length === 0 ? ( +
No simulators match the current filters.
+ ) : null} +
+ +
+
+ ); +} + +function FarmTile({ + busy, + isFocused, + onBoot, + onSelect, + onShutdown, + simulator, + view, +}: { + busy: boolean; + isFocused: boolean; + onBoot: () => void; + onSelect: () => void; + onShutdown: () => void; + simulator: SimulatorMetadata; + view: FarmViewMode; +}) { + const [canvas, setCanvas] = useState(null); + const handleCanvasRef = useCallback((node: HTMLCanvasElement | null) => { + setCanvas(node); + }, []); + const stream = useLiveStream({ + canvasElement: canvas, + paused: !simulator.isBooted || isFocused, + simulator, + streamProfile: "thumb", + }); + const runtime = simulatorRuntimeLabel(simulator); + const frameStyle = simulator.isBooted && !stream.hasFrame + ? { backgroundImage: `url("${screenshotUrl(simulator.udid)}")` } + : undefined; + + return ( +
+
+ {simulator.isBooted && !isFocused ? ( + + ) : simulator.isBooted ? ( + Focused + ) : ( + Shutdown + )} +
+
+
+ {simulator.name} + {runtime} +
+
+ {simulatorFamily(simulator)} + {simulator.isBooted ? `${stream.fps.toFixed(0)} fps` : "off"} +
+
+
+ {simulator.isBooted ? ( + + ) : ( + + )} +
+
+ ); +} + +function FarmFocus({ simulator }: { simulator: SimulatorMetadata }) { + const [canvas, setCanvas] = useState(null); + const handleCanvasRef = useCallback((node: HTMLCanvasElement | null) => { + setCanvas(node); + }, []); + const stream = useLiveStream({ + canvasElement: canvas, + paused: !simulator.isBooted, + simulator, + streamProfile: "focus", + }); + + return ( +
+
+
+ {simulator.name} + {simulatorRuntimeLabel(simulator)} +
+
+ {stream.status.state} + {stream.fps.toFixed(1)} fps +
+
+
+ {simulator.isBooted ? ( + + ) : ( + Boot this simulator to stream it. + )} +
+
+
+
UDID
+
{simulator.udid}
+
+
+
Stream
+
{stream.stats.width && stream.stats.height ? `${stream.stats.width}x${stream.stats.height}` : "Waiting"}
+
+
+
+ ); +} diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index c24571a3..d57b7356 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -35,6 +35,18 @@ export function sendWebRtcClientStats(stats: unknown): boolean { ); } +export function sendWebRtcStreamControl(options: { + forceKeyframe?: boolean; + fps?: number; + profile?: "focus" | "full" | "paused" | "thumb" | "thumbnail"; + snapshot?: boolean; +}): boolean { + return sendDataChannelMessage( + activeWebRtcControlChannel, + JSON.stringify({ ...options, type: "streamControl" }), + ); +} + function sendDataChannelMessage( channel: RTCDataChannel | null, encoded: string, @@ -63,6 +75,7 @@ interface StreamClientBackend { connect(target: StreamConnectTarget): void | Promise; destroy(): void; disconnect(): void; + sendControl?(payload: unknown): boolean; } class WebRtcStreamClient implements StreamClientBackend { @@ -318,6 +331,10 @@ class WebRtcStreamClient implements StreamClientBackend { this.onMessage({ type: "status", status: { state: "idle" } }); } + sendControl(payload: unknown): boolean { + return sendDataChannelMessage(this.controlChannel, JSON.stringify(payload)); + } + destroy() { this.disconnect(); } @@ -989,9 +1006,6 @@ export class StreamWorkerClient { constructor(onMessage: (message: WorkerToMainMessage) => void) { this.onMessage = onMessage; - if (activeStreamClient && activeStreamClient !== this) { - activeStreamClient.destroy(); - } activeStreamClient = this; } @@ -1038,6 +1052,17 @@ export class StreamWorkerClient { this.backend?.clear(); } + sendStreamControl(options: { + forceKeyframe?: boolean; + fps?: number; + profile?: "focus" | "full" | "paused" | "thumb" | "thumbnail"; + snapshot?: boolean; + }) { + return Boolean( + this.backend?.sendControl?.({ ...options, type: "streamControl" }), + ); + } + destroy() { if (this.disposed) { return; diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index 4b6fc62c..d9fadc2a 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -28,6 +28,7 @@ interface UseLiveStreamOptions { paused?: boolean; remote?: boolean; simulator: SimulatorMetadata | null; + streamProfile?: "focus" | "full" | "paused" | "thumb" | "thumbnail"; } interface UseLiveStreamResult { @@ -72,6 +73,7 @@ export function useLiveStream({ paused = false, remote = false, simulator, + streamProfile = "focus", }: UseLiveStreamOptions): UseLiveStreamResult { const clientTelemetryIdRef = useRef(""); const workerClientRef = useRef(null); @@ -253,6 +255,31 @@ export function useLiveStream({ }; }, [canvasElement, simulator?.isBooted, simulator?.udid, paused, remote]); + useEffect(() => { + if (paused || !simulator?.isBooted) { + return; + } + let attempts = 0; + const send = () => { + attempts += 1; + return Boolean( + workerClientRef.current?.sendStreamControl({ + forceKeyframe: streamProfile === "focus" || streamProfile === "full", + profile: streamProfile, + }), + ); + }; + if (send()) { + return; + } + const interval = window.setInterval(() => { + if (send() || attempts >= 8) { + window.clearInterval(interval); + } + }, 250); + return () => window.clearInterval(interval); + }, [paused, simulator?.isBooted, simulator?.udid, streamProfile]); + useEffect(() => { if (!simulator?.udid) { return; diff --git a/client/src/features/toolbar/Toolbar.tsx b/client/src/features/toolbar/Toolbar.tsx index 31ae7d30..50a5eab1 100644 --- a/client/src/features/toolbar/Toolbar.tsx +++ b/client/src/features/toolbar/Toolbar.tsx @@ -147,6 +147,9 @@ export function Toolbar({
+ + Farm + {selectedSimulator ? (
{showBootButton ? ( diff --git a/client/src/styles/layout.css b/client/src/styles/layout.css index ac1f7ae7..cb24e061 100644 --- a/client/src/styles/layout.css +++ b/client/src/styles/layout.css @@ -98,6 +98,309 @@ z-index: 8; } +.farm-app { + display: grid; + grid-template-rows: auto minmax(0, 1fr); + height: 100vh; + height: 100dvh; + overflow: hidden; + background: var(--bg); + color: var(--text); +} + +.farm-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + min-height: 48px; + padding: 8px 12px; + border-bottom: 1px solid var(--border); + background: var(--surface); +} + +.farm-title { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 150px; +} + +.farm-title strong { + font-size: 14px; +} + +.farm-title span, +.farm-focus-head span, +.farm-tile-meta span { + color: var(--text-secondary); + font-size: 12px; +} + +.farm-controls { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 8px; + min-width: 0; + flex: 1; +} + +.farm-search, +.farm-select { + height: 30px; + min-width: 0; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--surface-hover); + color: var(--text); + font: inherit; + font-size: 12px; +} + +.farm-search { + width: min(260px, 28vw); + padding: 0 10px; +} + +.farm-select { + padding: 0 8px; +} + +.farm-segments { + display: flex; + align-items: center; + height: 30px; + padding: 2px; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--surface-hover); +} + +.farm-segments button, +.farm-link, +.farm-tile-actions button { + height: 24px; + border: 0; + border-radius: 4px; + background: transparent; + color: var(--text); + font: inherit; + font-size: 12px; +} + +.farm-segments button { + padding: 0 8px; + text-transform: capitalize; +} + +.farm-segments button.active { + background: var(--surface); +} + +.farm-link { + display: inline-flex; + align-items: center; + padding: 0 8px; + text-decoration: none; +} + +.farm-main { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(320px, 28vw); + gap: 0; + min-height: 0; + overflow: hidden; +} + +.farm-tiles { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); + align-content: start; + gap: 10px; + min-height: 0; + overflow: auto; + padding: 10px; +} + +.farm-wall .farm-tiles { + grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); +} + +.farm-list .farm-tiles { + display: flex; + flex-direction: column; +} + +.farm-tile { + display: grid; + grid-template-rows: minmax(150px, 1fr) auto auto; + min-width: 0; + overflow: hidden; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--surface); +} + +.farm-tile.focused { + border-color: var(--accent); +} + +.farm-tile.list { + grid-template-columns: 144px minmax(0, 1fr) auto; + grid-template-rows: 96px; + align-items: center; +} + +.farm-tile-screen, +.farm-focus-screen { + display: grid; + place-items: center; + min-width: 0; + min-height: 0; + overflow: hidden; + background-color: #111; + background-position: center; + background-repeat: no-repeat; + background-size: contain; +} + +.farm-tile-screen { + aspect-ratio: 9 / 12; +} + +.farm-tile.list .farm-tile-screen { + width: 144px; + height: 96px; + aspect-ratio: auto; +} + +.farm-canvas, +.farm-focus-canvas { + width: 100%; + height: 100%; + object-fit: contain; +} + +.farm-tile-screen span, +.farm-focus-screen span, +.farm-empty { + color: var(--text-muted); + font-size: 12px; +} + +.farm-tile-meta { + display: flex; + justify-content: space-between; + gap: 8px; + min-width: 0; + padding: 8px; +} + +.farm-tile-meta > div:first-child { + min-width: 0; +} + +.farm-tile-meta strong { + display: block; + overflow: hidden; + font-size: 13px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.farm-tile-stats { + display: flex; + flex-direction: column; + align-items: flex-end; + flex-shrink: 0; +} + +.farm-tile-actions { + display: flex; + justify-content: flex-end; + padding: 0 8px 8px; +} + +.farm-tile.list .farm-tile-actions { + padding: 0 10px; +} + +.farm-tile-actions button { + padding: 0 8px; + background: var(--surface-hover); +} + +.farm-tile-actions button:disabled { + opacity: 0.45; +} + +.farm-focus { + min-width: 0; + min-height: 0; + overflow: auto; + border-left: 1px solid var(--border); + background: var(--surface); +} + +.farm-focus-inner { + display: grid; + grid-template-rows: auto minmax(240px, 1fr) auto; + gap: 12px; + min-height: 100%; + padding: 12px; +} + +.farm-focus-head { + display: flex; + justify-content: space-between; + gap: 12px; + min-width: 0; +} + +.farm-focus-head strong { + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.farm-focus-stats { + display: flex; + flex-direction: column; + align-items: flex-end; + flex-shrink: 0; +} + +.farm-focus-screen { + border: 1px solid var(--border); + border-radius: 8px; +} + +.farm-focus-detail { + display: grid; + gap: 8px; + margin: 0; +} + +.farm-focus-detail div { + min-width: 0; +} + +.farm-focus-detail dt { + color: var(--text-muted); + font-size: 10px; + font-weight: 700; + text-transform: uppercase; +} + +.farm-focus-detail dd { + margin: 2px 0 0; + overflow-wrap: anywhere; + color: var(--text-secondary); + font-size: 12px; +} + @media (max-width: 600px) { .app { grid-template-rows: auto minmax(0, 1fr); @@ -149,6 +452,26 @@ flex-direction: column; } + .farm-header, + .farm-controls { + align-items: stretch; + flex-direction: column; + } + + .farm-search { + width: 100%; + } + + .farm-main { + grid-template-columns: 1fr; + grid-template-rows: minmax(0, 1fr) minmax(240px, 38dvh); + } + + .farm-focus { + border-left: 0; + border-top: 1px solid var(--border); + } + .debug-grid { grid-template-columns: 1fr; } diff --git a/docs/api/rest.md b/docs/api/rest.md index 183146fa..4e9aea19 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -149,6 +149,18 @@ and the server responds with an SDP answer for a receive-only H.264 video track: The endpoint requires the active simulator stream to produce H.264-compatible samples. The bundled browser client always uses this endpoint. +The browser also opens a `simdeck-control` data channel. In addition to input +messages, clients can tune the stream attached to that peer: + +```json +{ "type": "streamControl", "profile": "thumb" } +``` + +Supported profiles are `thumb`/`thumbnail`, `focus`/`full`, and `paused`. +Clients may also send `fps`, `forceKeyframe`, or `snapshot` fields. The farm UI +uses this to keep background tiles low-rate while the focused simulator receives +a full-rate stream. + ### `POST /api/simulators/{udid}/open-url` Opens a URL inside the simulator: diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 0d34efac..0907770e 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -37,6 +37,9 @@ simdeck ui [--port 4310] [--bind 127.0.0.1] [--advertise-host ] ``` `--open` opens the authenticated local URL after the daemon is ready. +The same React bundle also serves `/farm`, a multi-simulator dashboard with +grid, wall, and list views. Booted simulators stream as low-rate thumbnails +until selected; the focused simulator requests a full-rate WebRTC profile. ### `studio expose` @@ -237,12 +240,17 @@ Batch input can come from `--step`, `--file`, or `--stdin`. It fails fast by def ```sh simdeck screenshot --output screen.png simdeck screenshot --stdout > screen.png +simdeck stream --frames 120 > stream.h264 simdeck pasteboard set "hello" simdeck pasteboard get simdeck logs --seconds 30 --limit 200 simdeck chrome-profile ``` +`stream` writes Annex B H.264 samples to stdout and runs until interrupted, or +until `--frames` samples have been written. It is intended for diagnostics and +external tools. + `logs` fetches recent simulator logs. `chrome-profile` returns the CoreSimulator chrome layout used by the browser viewport. ## HTTP Fast Path diff --git a/docs/extensions/browser-client.md b/docs/extensions/browser-client.md index b9124c30..e5f73ef4 100644 --- a/docs/extensions/browser-client.md +++ b/docs/extensions/browser-client.md @@ -29,6 +29,7 @@ client/ │ ├── simulators.ts │ └── types.ts ├── features/ + │ ├── farm/ │ ├── simulators/ │ ├── viewport/ │ ├── stream/ @@ -42,6 +43,7 @@ client/ | Folder | Responsibility | | ------------------------- | ----------------------------------------------------------------------- | | `api/` | Typed wrappers around the SimDeck REST API and shared TypeScript types. | +| `features/farm/` | Multi-simulator dashboard with thumbnail/focus stream profiles. | | `features/simulators/` | Sidebar list of simulators plus boot/shutdown affordances. | | `features/viewport/` | Frame canvas, chrome compositing, hit testing. | | `features/stream/` | WebRTC client, receiver stats, and video frame plumbing. | @@ -59,6 +61,11 @@ client/ 6. The browser renders the H.264 video track through native WebRTC playback. 7. Touch and key events round-trip through `POST /api/simulators//touch` and `/key`. +`/farm` mounts the same bundle in farm mode. Each booted tile owns a WebRTC +session and sends `streamControl` messages over the control data channel: +thumbnail tiles request a low-rate profile, while the focused simulator requests +full-rate video and a fresh keyframe. + ## Dev workflow The repo's `npm run dev` script runs the server and Vite together: diff --git a/server/src/main.rs b/server/src/main.rs index c367f5cb..a8ab6cf8 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -177,6 +177,11 @@ enum Command { #[arg(long)] stdout: bool, }, + Stream { + udid: String, + #[arg(long, default_value_t = 0)] + frames: u64, + }, #[command(name = "describe")] DescribeUi { udid: String, @@ -1939,6 +1944,7 @@ fn main() -> anyhow::Result<()> { } Ok(()) } + Command::Stream { udid, frames } => run_stream_stdout(&bridge, udid, frames), Command::DescribeUi { udid, point, @@ -2791,6 +2797,38 @@ fn default_screenshot_path(udid: &str) -> PathBuf { PathBuf::from(format!("Simulator Screenshot - {udid} - {timestamp}.png")) } +fn run_stream_stdout(bridge: &NativeBridge, udid: String, frames: u64) -> anyhow::Result<()> { + let metrics = Arc::new(Metrics::default()); + let session = simulators::session::SimulatorSession::new(bridge, udid, metrics) + .map_err(|error| anyhow::anyhow!("{error}"))?; + session + .ensure_started() + .map_err(|error| anyhow::anyhow!("{error}"))?; + session.request_keyframe(); + + let mut receiver = session.subscribe(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .context("create stream runtime")?; + let mut stdout = io::stdout().lock(); + let mut written = 0u64; + runtime.block_on(async { + loop { + if frames > 0 && written >= frames { + break; + } + let frame = receiver.recv().await?; + let sample = crate::transport::webrtc::h264_annex_b_sample(&frame) + .map_err(|error| anyhow::anyhow!("encode Annex B frame: {error}"))?; + stdout.write_all(&sample)?; + stdout.flush()?; + written += 1; + } + anyhow::Ok(()) + }) +} + #[allow(clippy::too_many_arguments)] fn describe_ui_snapshot( bridge: &NativeBridge, diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 77c62306..6a12d145 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; -use tokio::sync::broadcast; +use tokio::sync::{broadcast, mpsc}; use tokio::task; use tokio::time; use tracing::{info, warn}; @@ -166,11 +166,13 @@ pub async fn create_answer( .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, ); register_diagnostics(&peer_connection, &udid); + let (stream_control_tx, stream_control_rx) = mpsc::unbounded_channel(); register_control_data_channel( &peer_connection, session.clone(), state.clone(), udid.clone(), + stream_control_tx, ); let video_track = Arc::new(TrackLocalStaticRTP::new( @@ -232,6 +234,7 @@ pub async fn create_answer( video_track, cancellation_token, cancellation, + stream_control_rx, } .run(), ); @@ -344,17 +347,19 @@ fn register_control_data_channel( session: crate::simulators::session::SimulatorSession, state: AppState, udid: String, + stream_control_tx: mpsc::UnboundedSender, ) { peer_connection.on_data_channel(Box::new(move |channel: Arc| { let session = session.clone(); let state = state.clone(); let udid = udid.clone(); + let stream_control_tx = stream_control_tx.clone(); Box::pin(async move { let label = channel.label(); if label != WEBRTC_CONTROL_CHANNEL_LABEL && label != WEBRTC_TELEMETRY_CHANNEL_LABEL { return; } - attach_control_data_channel(channel, session, state, udid); + attach_control_data_channel(channel, session, state, udid, stream_control_tx); }) })); } @@ -364,11 +369,13 @@ fn attach_control_data_channel( session: crate::simulators::session::SimulatorSession, state: AppState, udid: String, + stream_control_tx: mpsc::UnboundedSender, ) { channel.on_message(Box::new(move |message: DataChannelMessage| { let session = session.clone(); let state = state.clone(); let udid = udid.clone(); + let stream_control_tx = stream_control_tx.clone(); Box::pin(async move { let Ok(text) = std::str::from_utf8(&message.data) else { warn!("Invalid WebRTC control message bytes for {udid}"); @@ -381,6 +388,23 @@ fn attach_control_data_channel( state.metrics.record_client_stream_stats(stats); } } + WebRtcDataChannelMessage::StreamControl { + fps, + force_keyframe, + profile, + snapshot, + } => { + let command = WebRtcStreamCommand::from_payload( + profile.as_deref(), + fps, + force_keyframe.unwrap_or(false), + snapshot.unwrap_or(false), + ); + if command.force_keyframe || command.snapshot { + session.request_keyframe(); + } + let _ = stream_control_tx.send(command); + } } return; } @@ -420,6 +444,42 @@ fn attach_control_data_channel( #[serde(tag = "type", rename_all = "camelCase")] enum WebRtcDataChannelMessage { ClientStats { stats: ClientStreamStats }, + StreamControl { + fps: Option, + #[serde(rename = "forceKeyframe")] + force_keyframe: Option, + profile: Option, + snapshot: Option, + }, +} + +#[derive(Clone, Debug)] +struct WebRtcStreamCommand { + force_keyframe: bool, + snapshot: bool, + target_frame_interval: Option, +} + +impl WebRtcStreamCommand { + fn from_payload( + profile: Option<&str>, + fps: Option, + force_keyframe: bool, + snapshot: bool, + ) -> Self { + let profile_fps = match profile.map(str::trim) { + Some("thumbnail") | Some("thumb") => Some(8), + Some("paused") => Some(1), + Some("focus") | Some("full") => Some(60), + _ => None, + }; + let fps = fps.or(profile_fps).map(|value| value.clamp(1, 60)); + Self { + force_keyframe, + snapshot, + target_frame_interval: fps.map(|fps| Duration::from_micros(1_000_000 / fps as u64)), + } + } } fn is_h264_codec(codec: &str) -> bool { @@ -660,6 +720,7 @@ struct WebRtcMediaStream { video_track: Arc, cancellation_token: broadcast::Sender<()>, cancellation: broadcast::Receiver<()>, + stream_control_rx: mpsc::UnboundedReceiver, } impl WebRtcMediaStream { @@ -673,6 +734,7 @@ impl WebRtcMediaStream { video_track, cancellation_token, mut cancellation, + mut stream_control_rx, } = self; let mut rx = session.subscribe(); let mut latest_keyframe = first_frame.clone(); @@ -693,6 +755,8 @@ impl WebRtcMediaStream { let mut refresh_sleep = Box::pin(time::sleep(refresh_floor)); let mut adaptive_refresh_interval = refresh_floor; let mut bootstrap_frames_remaining = WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS; + let mut target_frame_interval: Option = None; + let mut last_sent_at: Option = None; let mut waiting_for_keyframe = false; let mut peer_disconnected_since: Option = None; let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); @@ -707,6 +771,7 @@ impl WebRtcMediaStream { .await { Ok(true) => { + last_sent_at = Some(time::Instant::now()); state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); } Ok(false) => { @@ -792,6 +857,18 @@ impl WebRtcMediaStream { .as_mut() .reset(time::Instant::now() + adaptive_refresh_interval); } + command = stream_control_rx.recv() => { + let Some(command) = command else { + continue; + }; + target_frame_interval = command.target_frame_interval; + if command.force_keyframe || command.snapshot { + waiting_for_keyframe = true; + session.request_keyframe(); + } else { + session.request_refresh(); + } + } frame = rx.recv() => { let frame = match frame { Ok(frame) => frame, @@ -829,6 +906,12 @@ impl WebRtcMediaStream { latest_keyframe = frame.clone(); waiting_for_keyframe = false; } + if let (Some(interval), Some(previous)) = (target_frame_interval, last_sent_at) { + if previous.elapsed() < interval && !frame.is_keyframe { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + continue; + } + } let duration = send_timing.duration_for(&frame, realtime_stream); let started_at = time::Instant::now(); let write_result = write_frame_sample_with_timeout( @@ -843,6 +926,7 @@ impl WebRtcMediaStream { adaptive_interval_for_write(started_at.elapsed(), refresh_floor, refresh_ceiling); match write_result { Ok(true) => { + last_sent_at = Some(time::Instant::now()); state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); } Ok(false) => { @@ -1032,7 +1116,7 @@ pub fn realtime_stream_enabled() -> bool { }) } -fn h264_annex_b_sample(frame: &crate::transport::packet::FramePacket) -> anyhow::Result> { +pub fn h264_annex_b_sample(frame: &crate::transport::packet::FramePacket) -> anyhow::Result> { let data = frame.data.as_ref(); let description = frame.description.as_ref().map(bytes::Bytes::as_ref); let mut sample = Vec::with_capacity(data.len() + description.map_or(0, |bytes| bytes.len())); diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index 244ef2bf..95f33c5a 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -30,6 +30,8 @@ simdeck ui --bind 0.0.0.0 --advertise-host 192.168.1.50 --open `simdeck` without a subcommand starts a foreground workspace daemon, prints local and LAN HTTP URLs, prints a six-digit pairing code for LAN browsers, and stops on `q` or Ctrl-C. The optional single argument is a simulator name or UDID to select by default. Use `-d` for detached start, `-k` to kill the background daemon, and `-r` to restart it. Viewer: `http://127.0.0.1:4310` or `http://127.0.0.1:4310?device=`. +Farm viewer: `http://127.0.0.1:4310/farm` shows all simulators with +low-rate thumbnails and a focused full-rate stream. The browser uses WebRTC H.264 video for both hardware and software encoders. Add `--low-latency` on less capable runners to cap software H.264 at 15 fps, drop stale pending frames more aggressively, and cap the longest edge at 1170 px @@ -213,11 +215,13 @@ await simdeck.batch(udid, [ ```bash simdeck screenshot --output screen.png simdeck screenshot --stdout > screen.png +simdeck stream --frames 120 > stream.h264 simdeck logs --seconds 30 --limit 200 simdeck chrome-profile ``` -Use screenshots for still evidence. +Use screenshots for still evidence. Use `stream` when a diagnostic needs raw +H.264 samples for an external player or capture pipeline. ## Default Loop From 04fbcae71d401f3368f72cd44660509be058f381 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 2 May 2026 14:54:31 -0400 Subject: [PATCH 2/4] Remove farm view and related docs --- README.md | 4 - cli/DFPrivateSimulatorDisplayBridge.m | 48 +++- client/src/app/App.tsx | 4 - client/src/features/farm/FarmView.tsx | 310 ----------------------- client/src/features/toolbar/Toolbar.tsx | 3 - client/src/styles/layout.css | 323 ------------------------ docs/api/rest.md | 4 +- docs/cli/commands.md | 3 - docs/extensions/browser-client.md | 7 - skills/simdeck/SKILL.md | 2 - 10 files changed, 35 insertions(+), 673 deletions(-) delete mode 100644 client/src/features/farm/FarmView.tsx diff --git a/README.md b/README.md index 8f99df10..af358c44 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,6 @@ view inside the editor. ## Features - Local simulator video stream over browser-native WebRTC H.264 -- Multi-simulator farm view at `/farm` with low-rate thumbnails and a focused full-rate stream - Full simulator control & inspection using private accessibility APIs - CoreSimulator chrome asset rendering for device bezels - NativeScript, React Native, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live @@ -63,9 +62,6 @@ simdeck "iPhone 17 Pro Max" Use `simdeck ui --open` or `simdeck daemon start` when you want a reusable background daemon instead. The no-subcommand lifecycle shortcuts are `simdeck -d` for detached start, `simdeck -k` to kill the background daemon, and `simdeck -r` to restart it. The served loopback browser UI receives the generated API access token automatically. LAN browsers pair with the printed code before receiving the API cookie. -Open `http://127.0.0.1:4310/farm` to monitor every simulator in one dashboard. -The farm uses thumbnail stream profiles for the grid and promotes the focused -simulator to a full-rate WebRTC stream. SimDeck Studio providers run the daemon on loopback and use `scripts/studio-provider-bridge.mjs` for outbound control-plane communication diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 9ae02e9b..5e733593 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -136,6 +136,39 @@ return @"/Applications/Xcode.app/Contents/Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit"; } +static NSInteger DFXcodeMajorVersion(void) { + NSString *developerPath = nil; + const char *developerDir = getenv("DEVELOPER_DIR"); + if (developerDir != NULL && developerDir[0] != '\0') { + developerPath = [NSString stringWithUTF8String:developerDir]; + } else { + FILE *pipe = popen("/usr/bin/xcode-select -p 2>/dev/null", "r"); + if (pipe != NULL) { + char buffer[PATH_MAX] = {0}; + if (fgets(buffer, sizeof(buffer), pipe) != NULL) { + developerPath = [[NSString stringWithUTF8String:buffer] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + } + pclose(pipe); + } + } + if (developerPath.length == 0) { + return 0; + } + + NSString *contentsPath = developerPath; + if ([contentsPath.lastPathComponent isEqualToString:@"Developer"]) { + contentsPath = contentsPath.stringByDeletingLastPathComponent; + } + + NSDictionary *versionInfo = [NSDictionary dictionaryWithContentsOfFile:[contentsPath stringByAppendingPathComponent:@"version.plist"]]; + NSString *version = versionInfo[@"CFBundleShortVersionString"]; + if (version.length == 0) { + NSDictionary *info = [NSDictionary dictionaryWithContentsOfFile:[contentsPath stringByAppendingPathComponent:@"Info.plist"]]; + version = info[@"CFBundleShortVersionString"]; + } + return [[version componentsSeparatedByString:@"."].firstObject integerValue]; +} + typedef struct { __unsafe_unretained id unit; double value; @@ -207,20 +240,7 @@ static BOOL DFShouldUseIndigoMouse9Path(void) { return; } - FILE *pipe = popen("/usr/bin/xcodebuild -version 2>/dev/null", "r"); - if (pipe == NULL) { - return; - } - char buffer[256] = {0}; - if (fgets(buffer, sizeof(buffer), pipe) != NULL) { - NSString *line = [[NSString stringWithUTF8String:buffer] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; - NSArray *parts = [line componentsSeparatedByString:@" "]; - if (parts.count >= 2 && [parts[0] isEqualToString:@"Xcode"]) { - NSInteger major = [[parts[1] componentsSeparatedByString:@"."].firstObject integerValue]; - enabled = major >= 26; - } - } - pclose(pipe); + enabled = DFXcodeMajorVersion() >= 26; }); return enabled; } diff --git a/client/src/app/App.tsx b/client/src/app/App.tsx index f0b133c0..75c76333 100644 --- a/client/src/app/App.tsx +++ b/client/src/app/App.tsx @@ -1,9 +1,5 @@ import { AppShell } from "./AppShell"; -import { FarmView } from "../features/farm/FarmView"; export default function App() { - if (window.location.pathname === "/farm") { - return ; - } return ; } diff --git a/client/src/features/farm/FarmView.tsx b/client/src/features/farm/FarmView.tsx deleted file mode 100644 index 385c5d73..00000000 --- a/client/src/features/farm/FarmView.tsx +++ /dev/null @@ -1,310 +0,0 @@ -import { useCallback, useMemo, useState } from "react"; - -import { accessTokenFromLocation } from "../../api/client"; -import { apiUrl } from "../../api/config"; -import { bootSimulator, shutdownSimulator } from "../../api/controls"; -import type { SimulatorMetadata } from "../../api/types"; -import { simulatorRuntimeLabel } from "../simulators/simulatorDisplay"; -import { useSimulatorList } from "../simulators/useSimulatorList"; -import { useLiveStream } from "../stream/useLiveStream"; - -type FarmViewMode = "grid" | "list" | "wall"; - -function simulatorFamily(simulator: SimulatorMetadata): string { - const text = `${simulator.name} ${simulator.deviceTypeName ?? ""}`.toLowerCase(); - if (text.includes("ipad")) return "iPad"; - if (text.includes("watch")) return "Watch"; - if (text.includes("tv")) return "TV"; - return "iPhone"; -} - -function screenshotUrl(udid: string): string { - const url = new URL( - apiUrl(`/api/simulators/${encodeURIComponent(udid)}/screenshot.png`), - window.location.href, - ); - const token = accessTokenFromLocation(); - if (token) { - url.searchParams.set("simdeckToken", token); - } - url.searchParams.set("stamp", String(Date.now())); - return url.toString(); -} - -export function FarmView() { - const { isLoading, refresh, simulators } = useSimulatorList(); - const [search, setSearch] = useState(""); - const [family, setFamily] = useState("all"); - const [state, setState] = useState("all"); - const [view, setView] = useState("grid"); - const [selectedUDID, setSelectedUDID] = useState(""); - const [busyUDID, setBusyUDID] = useState(""); - - const filtered = useMemo(() => { - const needle = search.trim().toLowerCase(); - return simulators - .filter((simulator) => { - if (family !== "all" && simulatorFamily(simulator) !== family) { - return false; - } - if (state === "booted" && !simulator.isBooted) { - return false; - } - if (state === "shutdown" && simulator.isBooted) { - return false; - } - if (!needle) { - return true; - } - return [ - simulator.name, - simulator.udid, - simulatorRuntimeLabel(simulator), - simulator.deviceTypeName, - ] - .filter(Boolean) - .some((value) => value!.toLowerCase().includes(needle)); - }) - .sort((a, b) => Number(b.isBooted) - Number(a.isBooted) || a.name.localeCompare(b.name)); - }, [family, search, simulators, state]); - - const selectedSimulator = - filtered.find((simulator) => simulator.udid === selectedUDID) ?? - filtered.find((simulator) => simulator.isBooted) ?? - filtered[0] ?? - null; - - async function runLifecycle( - simulator: SimulatorMetadata, - action: "boot" | "shutdown", - ) { - setBusyUDID(simulator.udid); - try { - if (action === "boot") { - await bootSimulator(simulator.udid); - setSelectedUDID(simulator.udid); - } else { - await shutdownSimulator(simulator.udid); - } - await refresh(); - } finally { - setBusyUDID(""); - } - } - - return ( -
-
-
- SimDeck Farm - - {filtered.filter((simulator) => simulator.isBooted).length} live /{" "} - {filtered.length} shown - -
-
- setSearch(event.target.value)} - placeholder="Search simulators" - value={search} - /> - - -
- {(["grid", "wall", "list"] as FarmViewMode[]).map((mode) => ( - - ))} -
- - Single - -
-
-
-
- {filtered.map((simulator) => ( - void runLifecycle(simulator, "boot")} - onSelect={() => setSelectedUDID(simulator.udid)} - onShutdown={() => void runLifecycle(simulator, "shutdown")} - simulator={simulator} - view={view} - /> - ))} - {!isLoading && filtered.length === 0 ? ( -
No simulators match the current filters.
- ) : null} -
- -
-
- ); -} - -function FarmTile({ - busy, - isFocused, - onBoot, - onSelect, - onShutdown, - simulator, - view, -}: { - busy: boolean; - isFocused: boolean; - onBoot: () => void; - onSelect: () => void; - onShutdown: () => void; - simulator: SimulatorMetadata; - view: FarmViewMode; -}) { - const [canvas, setCanvas] = useState(null); - const handleCanvasRef = useCallback((node: HTMLCanvasElement | null) => { - setCanvas(node); - }, []); - const stream = useLiveStream({ - canvasElement: canvas, - paused: !simulator.isBooted || isFocused, - simulator, - streamProfile: "thumb", - }); - const runtime = simulatorRuntimeLabel(simulator); - const frameStyle = simulator.isBooted && !stream.hasFrame - ? { backgroundImage: `url("${screenshotUrl(simulator.udid)}")` } - : undefined; - - return ( -
-
- {simulator.isBooted && !isFocused ? ( - - ) : simulator.isBooted ? ( - Focused - ) : ( - Shutdown - )} -
-
-
- {simulator.name} - {runtime} -
-
- {simulatorFamily(simulator)} - {simulator.isBooted ? `${stream.fps.toFixed(0)} fps` : "off"} -
-
-
- {simulator.isBooted ? ( - - ) : ( - - )} -
-
- ); -} - -function FarmFocus({ simulator }: { simulator: SimulatorMetadata }) { - const [canvas, setCanvas] = useState(null); - const handleCanvasRef = useCallback((node: HTMLCanvasElement | null) => { - setCanvas(node); - }, []); - const stream = useLiveStream({ - canvasElement: canvas, - paused: !simulator.isBooted, - simulator, - streamProfile: "focus", - }); - - return ( -
-
-
- {simulator.name} - {simulatorRuntimeLabel(simulator)} -
-
- {stream.status.state} - {stream.fps.toFixed(1)} fps -
-
-
- {simulator.isBooted ? ( - - ) : ( - Boot this simulator to stream it. - )} -
-
-
-
UDID
-
{simulator.udid}
-
-
-
Stream
-
{stream.stats.width && stream.stats.height ? `${stream.stats.width}x${stream.stats.height}` : "Waiting"}
-
-
-
- ); -} diff --git a/client/src/features/toolbar/Toolbar.tsx b/client/src/features/toolbar/Toolbar.tsx index 50a5eab1..31ae7d30 100644 --- a/client/src/features/toolbar/Toolbar.tsx +++ b/client/src/features/toolbar/Toolbar.tsx @@ -147,9 +147,6 @@ export function Toolbar({
- - Farm - {selectedSimulator ? (
{showBootButton ? ( diff --git a/client/src/styles/layout.css b/client/src/styles/layout.css index cb24e061..ac1f7ae7 100644 --- a/client/src/styles/layout.css +++ b/client/src/styles/layout.css @@ -98,309 +98,6 @@ z-index: 8; } -.farm-app { - display: grid; - grid-template-rows: auto minmax(0, 1fr); - height: 100vh; - height: 100dvh; - overflow: hidden; - background: var(--bg); - color: var(--text); -} - -.farm-header { - display: flex; - align-items: center; - justify-content: space-between; - gap: 12px; - min-height: 48px; - padding: 8px 12px; - border-bottom: 1px solid var(--border); - background: var(--surface); -} - -.farm-title { - display: flex; - flex-direction: column; - gap: 2px; - min-width: 150px; -} - -.farm-title strong { - font-size: 14px; -} - -.farm-title span, -.farm-focus-head span, -.farm-tile-meta span { - color: var(--text-secondary); - font-size: 12px; -} - -.farm-controls { - display: flex; - align-items: center; - justify-content: flex-end; - gap: 8px; - min-width: 0; - flex: 1; -} - -.farm-search, -.farm-select { - height: 30px; - min-width: 0; - border: 1px solid var(--border); - border-radius: 6px; - background: var(--surface-hover); - color: var(--text); - font: inherit; - font-size: 12px; -} - -.farm-search { - width: min(260px, 28vw); - padding: 0 10px; -} - -.farm-select { - padding: 0 8px; -} - -.farm-segments { - display: flex; - align-items: center; - height: 30px; - padding: 2px; - border: 1px solid var(--border); - border-radius: 6px; - background: var(--surface-hover); -} - -.farm-segments button, -.farm-link, -.farm-tile-actions button { - height: 24px; - border: 0; - border-radius: 4px; - background: transparent; - color: var(--text); - font: inherit; - font-size: 12px; -} - -.farm-segments button { - padding: 0 8px; - text-transform: capitalize; -} - -.farm-segments button.active { - background: var(--surface); -} - -.farm-link { - display: inline-flex; - align-items: center; - padding: 0 8px; - text-decoration: none; -} - -.farm-main { - display: grid; - grid-template-columns: minmax(0, 1fr) minmax(320px, 28vw); - gap: 0; - min-height: 0; - overflow: hidden; -} - -.farm-tiles { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); - align-content: start; - gap: 10px; - min-height: 0; - overflow: auto; - padding: 10px; -} - -.farm-wall .farm-tiles { - grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); -} - -.farm-list .farm-tiles { - display: flex; - flex-direction: column; -} - -.farm-tile { - display: grid; - grid-template-rows: minmax(150px, 1fr) auto auto; - min-width: 0; - overflow: hidden; - border: 1px solid var(--border); - border-radius: 8px; - background: var(--surface); -} - -.farm-tile.focused { - border-color: var(--accent); -} - -.farm-tile.list { - grid-template-columns: 144px minmax(0, 1fr) auto; - grid-template-rows: 96px; - align-items: center; -} - -.farm-tile-screen, -.farm-focus-screen { - display: grid; - place-items: center; - min-width: 0; - min-height: 0; - overflow: hidden; - background-color: #111; - background-position: center; - background-repeat: no-repeat; - background-size: contain; -} - -.farm-tile-screen { - aspect-ratio: 9 / 12; -} - -.farm-tile.list .farm-tile-screen { - width: 144px; - height: 96px; - aspect-ratio: auto; -} - -.farm-canvas, -.farm-focus-canvas { - width: 100%; - height: 100%; - object-fit: contain; -} - -.farm-tile-screen span, -.farm-focus-screen span, -.farm-empty { - color: var(--text-muted); - font-size: 12px; -} - -.farm-tile-meta { - display: flex; - justify-content: space-between; - gap: 8px; - min-width: 0; - padding: 8px; -} - -.farm-tile-meta > div:first-child { - min-width: 0; -} - -.farm-tile-meta strong { - display: block; - overflow: hidden; - font-size: 13px; - text-overflow: ellipsis; - white-space: nowrap; -} - -.farm-tile-stats { - display: flex; - flex-direction: column; - align-items: flex-end; - flex-shrink: 0; -} - -.farm-tile-actions { - display: flex; - justify-content: flex-end; - padding: 0 8px 8px; -} - -.farm-tile.list .farm-tile-actions { - padding: 0 10px; -} - -.farm-tile-actions button { - padding: 0 8px; - background: var(--surface-hover); -} - -.farm-tile-actions button:disabled { - opacity: 0.45; -} - -.farm-focus { - min-width: 0; - min-height: 0; - overflow: auto; - border-left: 1px solid var(--border); - background: var(--surface); -} - -.farm-focus-inner { - display: grid; - grid-template-rows: auto minmax(240px, 1fr) auto; - gap: 12px; - min-height: 100%; - padding: 12px; -} - -.farm-focus-head { - display: flex; - justify-content: space-between; - gap: 12px; - min-width: 0; -} - -.farm-focus-head strong { - display: block; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} - -.farm-focus-stats { - display: flex; - flex-direction: column; - align-items: flex-end; - flex-shrink: 0; -} - -.farm-focus-screen { - border: 1px solid var(--border); - border-radius: 8px; -} - -.farm-focus-detail { - display: grid; - gap: 8px; - margin: 0; -} - -.farm-focus-detail div { - min-width: 0; -} - -.farm-focus-detail dt { - color: var(--text-muted); - font-size: 10px; - font-weight: 700; - text-transform: uppercase; -} - -.farm-focus-detail dd { - margin: 2px 0 0; - overflow-wrap: anywhere; - color: var(--text-secondary); - font-size: 12px; -} - @media (max-width: 600px) { .app { grid-template-rows: auto minmax(0, 1fr); @@ -452,26 +149,6 @@ flex-direction: column; } - .farm-header, - .farm-controls { - align-items: stretch; - flex-direction: column; - } - - .farm-search { - width: 100%; - } - - .farm-main { - grid-template-columns: 1fr; - grid-template-rows: minmax(0, 1fr) minmax(240px, 38dvh); - } - - .farm-focus { - border-left: 0; - border-top: 1px solid var(--border); - } - .debug-grid { grid-template-columns: 1fr; } diff --git a/docs/api/rest.md b/docs/api/rest.md index 4e9aea19..ffc1c7e1 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -157,9 +157,7 @@ messages, clients can tune the stream attached to that peer: ``` Supported profiles are `thumb`/`thumbnail`, `focus`/`full`, and `paused`. -Clients may also send `fps`, `forceKeyframe`, or `snapshot` fields. The farm UI -uses this to keep background tiles low-rate while the focused simulator receives -a full-rate stream. +Clients may also send `fps`, `forceKeyframe`, or `snapshot` fields. ### `POST /api/simulators/{udid}/open-url` diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 0907770e..52ad183c 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -37,9 +37,6 @@ simdeck ui [--port 4310] [--bind 127.0.0.1] [--advertise-host ] ``` `--open` opens the authenticated local URL after the daemon is ready. -The same React bundle also serves `/farm`, a multi-simulator dashboard with -grid, wall, and list views. Booted simulators stream as low-rate thumbnails -until selected; the focused simulator requests a full-rate WebRTC profile. ### `studio expose` diff --git a/docs/extensions/browser-client.md b/docs/extensions/browser-client.md index e5f73ef4..b9124c30 100644 --- a/docs/extensions/browser-client.md +++ b/docs/extensions/browser-client.md @@ -29,7 +29,6 @@ client/ │ ├── simulators.ts │ └── types.ts ├── features/ - │ ├── farm/ │ ├── simulators/ │ ├── viewport/ │ ├── stream/ @@ -43,7 +42,6 @@ client/ | Folder | Responsibility | | ------------------------- | ----------------------------------------------------------------------- | | `api/` | Typed wrappers around the SimDeck REST API and shared TypeScript types. | -| `features/farm/` | Multi-simulator dashboard with thumbnail/focus stream profiles. | | `features/simulators/` | Sidebar list of simulators plus boot/shutdown affordances. | | `features/viewport/` | Frame canvas, chrome compositing, hit testing. | | `features/stream/` | WebRTC client, receiver stats, and video frame plumbing. | @@ -61,11 +59,6 @@ client/ 6. The browser renders the H.264 video track through native WebRTC playback. 7. Touch and key events round-trip through `POST /api/simulators//touch` and `/key`. -`/farm` mounts the same bundle in farm mode. Each booted tile owns a WebRTC -session and sends `streamControl` messages over the control data channel: -thumbnail tiles request a low-rate profile, while the focused simulator requests -full-rate video and a fresh keyframe. - ## Dev workflow The repo's `npm run dev` script runs the server and Vite together: diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index 95f33c5a..481debc7 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -30,8 +30,6 @@ simdeck ui --bind 0.0.0.0 --advertise-host 192.168.1.50 --open `simdeck` without a subcommand starts a foreground workspace daemon, prints local and LAN HTTP URLs, prints a six-digit pairing code for LAN browsers, and stops on `q` or Ctrl-C. The optional single argument is a simulator name or UDID to select by default. Use `-d` for detached start, `-k` to kill the background daemon, and `-r` to restart it. Viewer: `http://127.0.0.1:4310` or `http://127.0.0.1:4310?device=`. -Farm viewer: `http://127.0.0.1:4310/farm` shows all simulators with -low-rate thumbnails and a focused full-rate stream. The browser uses WebRTC H.264 video for both hardware and software encoders. Add `--low-latency` on less capable runners to cap software H.264 at 15 fps, drop stale pending frames more aggressively, and cap the longest edge at 1170 px From cd3e8bab9bad0f94a122602aef7f041529b254f8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 2 May 2026 14:57:45 -0400 Subject: [PATCH 3/4] Format WebRTC transport --- server/src/transport/webrtc.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 6a12d145..6c0910c3 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -443,7 +443,9 @@ fn attach_control_data_channel( #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "camelCase")] enum WebRtcDataChannelMessage { - ClientStats { stats: ClientStreamStats }, + ClientStats { + stats: ClientStreamStats, + }, StreamControl { fps: Option, #[serde(rename = "forceKeyframe")] @@ -1116,7 +1118,9 @@ pub fn realtime_stream_enabled() -> bool { }) } -pub fn h264_annex_b_sample(frame: &crate::transport::packet::FramePacket) -> anyhow::Result> { +pub fn h264_annex_b_sample( + frame: &crate::transport::packet::FramePacket, +) -> anyhow::Result> { let data = frame.data.as_ref(); let description = frame.description.as_ref().map(bytes::Bytes::as_ref); let mut sample = Vec::with_capacity(data.len() + description.map_or(0, |bytes| bytes.len())); From a6b78cde0744b7b22bbc7978e61f073bf285324b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 2 May 2026 15:36:42 -0400 Subject: [PATCH 4/4] Fix WebRTC data channel clippy lint --- server/src/transport/webrtc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 6c0910c3..e5d2fffc 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -385,7 +385,7 @@ fn attach_control_data_channel( match message { WebRtcDataChannelMessage::ClientStats { stats } => { if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { - state.metrics.record_client_stream_stats(stats); + state.metrics.record_client_stream_stats(*stats); } } WebRtcDataChannelMessage::StreamControl { @@ -444,7 +444,7 @@ fn attach_control_data_channel( #[serde(tag = "type", rename_all = "camelCase")] enum WebRtcDataChannelMessage { ClientStats { - stats: ClientStreamStats, + stats: Box, }, StreamControl { fps: Option,