Add stream control APIs and harden simulator streaming

README.md

@@ -152,6 +152,7 @@ simdeck toggle-appearance <udid>
 simdeck pasteboard set <udid> "hello"
 simdeck pasteboard get <udid>
 simdeck screenshot <udid> --output screen.png
+simdeck stream <udid> --frames 120 > stream.h264
 simdeck describe <udid>
 simdeck describe <udid> --format agent --max-depth 4
 simdeck describe <udid> --point 120,240
@@ -179,6 +180,13 @@ simdeck chrome-profile <udid>
 simdeck logs <udid> --seconds 30 --limit 200
 ```
 
+`boot` prefers SimDeck's private CoreSimulator boot path so it can start devices
+without launching Simulator.app, then falls back to `xcrun simctl` when private
+booting is unavailable.
+
+`stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or
+external tools such as `ffplay`.
+
 `describe` uses the project daemon to prefer React Native, NativeScript, or
 UIKit in-app inspectors, then falls back to the built-in private CoreSimulator
 accessibility bridge. Use `--format agent` or `--format compact-json` for

cli/DFPrivateSimulatorDisplayBridge.m

@@ -32,10 +32,12 @@
 typedef uint32_t IndigoHIDEdge;
 
 typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEventFn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, NSEventType type, NSSize displaySize, IndigoHIDEdge edge);
+typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEvent9Fn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, uint32_t eventType, uint32_t direction, double unused1, double unused2, double widthPoints, double heightPoints);
 typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardArbitraryFn)(int keyCode, int op);
 typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardNSEventFn)(NSEvent *event);
 typedef IndigoHIDMessage *(*DFIndigoHIDMessageForButtonFn)(uint32_t buttonCode, uint32_t operation, uint32_t target);
 typedef IndigoHIDMessage *(*DFIndigoHIDMessageForHIDArbitraryFn)(uint32_t target, uint32_t page, uint32_t usage, uint32_t operation);
+typedef IndigoHIDMessage *(*DFIndigoHIDServiceMessageFn)(void);
 
 #pragma pack(push, 4)
 typedef struct {
@@ -91,6 +93,12 @@
 static const uint32_t DFIndigoTouchTarget = 0x32;
 static const uint8_t DFIndigoEventTypeTouch = 0x02;
 static const uint32_t DFIndigoTouchEventKind = 0x0b;
+static const uint32_t DFIndigoMouseEventDown = 1;
+static const uint32_t DFIndigoMouseEventUp = 2;
+static const uint32_t DFIndigoMouseEventDragged = 6;
+static const uint32_t DFIndigoMouseDirectionDown = 1;
+static const uint32_t DFIndigoMouseDirectionMove = 0;
+static const uint32_t DFIndigoMouseDirectionUp = 2;
 static const int DFKeyboardDirectionDown = 1;
 static const int DFKeyboardDirectionUp = 2;
 static const uint32_t DFButtonDirectionDown = 1;
@@ -128,6 +136,39 @@
     return @"/Applications/Xcode.app/Contents/Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit";
 }
 
+static NSInteger DFXcodeMajorVersion(void) {
+    NSString *developerPath = nil;
+    const char *developerDir = getenv("DEVELOPER_DIR");
+    if (developerDir != NULL && developerDir[0] != '\0') {
+        developerPath = [NSString stringWithUTF8String:developerDir];
+    } else {
+        FILE *pipe = popen("/usr/bin/xcode-select -p 2>/dev/null", "r");
+        if (pipe != NULL) {
+            char buffer[PATH_MAX] = {0};
+            if (fgets(buffer, sizeof(buffer), pipe) != NULL) {
+                developerPath = [[NSString stringWithUTF8String:buffer] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]];
+            }
+            pclose(pipe);
+        }
+    }
+    if (developerPath.length == 0) {
+        return 0;
+    }
+
+    NSString *contentsPath = developerPath;
+    if ([contentsPath.lastPathComponent isEqualToString:@"Developer"]) {
+        contentsPath = contentsPath.stringByDeletingLastPathComponent;
+    }
+
+    NSDictionary *versionInfo = [NSDictionary dictionaryWithContentsOfFile:[contentsPath stringByAppendingPathComponent:@"version.plist"]];
+    NSString *version = versionInfo[@"CFBundleShortVersionString"];
+    if (version.length == 0) {
+        NSDictionary *info = [NSDictionary dictionaryWithContentsOfFile:[contentsPath stringByAppendingPathComponent:@"Info.plist"]];
+        version = info[@"CFBundleShortVersionString"];
+    }
+    return [[version componentsSeparatedByString:@"."].firstObject integerValue];
+}
+
 typedef struct {
     __unsafe_unretained id unit;
     double value;
@@ -187,6 +228,23 @@ static BOOL DFVerboseTouchLoggingEnabled(void) {
     return enabled;
 }
 
+static BOOL DFShouldUseIndigoMouse9Path(void) {
+    static BOOL enabled = NO;
+    static dispatch_once_t onceToken;
+    dispatch_once(&onceToken, ^{
+        NSString *override = NSProcessInfo.processInfo.environment[@"SIMDECK_INDIGO_MOUSE_9ARG"];
+        if (override.length > 0) {
+            enabled = [override isEqualToString:@"1"] ||
+                      [override caseInsensitiveCompare:@"true"] == NSOrderedSame ||
+                      [override caseInsensitiveCompare:@"yes"] == NSOrderedSame;
+            return;
+        }
+
+        enabled = DFXcodeMajorVersion() >= 26;
+    });
+    return enabled;
+}
+
 #pragma mark - SimulatorKit Swift symbol resolver
 //
 // We call into SimulatorKit's private Swift API by dlsym'ing mangled symbol
@@ -1002,6 +1060,109 @@ static BOOL DFSendHIDMessage(id hidClient, IndigoHIDMessage *message, BOOL freeW
     return YES;
 }
 
+static uint32_t DFIndigoMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phase) {
+    switch (phase) {
+    case DFPrivateSimulatorTouchPhaseBegan:
+        return DFIndigoMouseEventDown;
+    case DFPrivateSimulatorTouchPhaseMoved:
+        return DFIndigoMouseEventDragged;
+    case DFPrivateSimulatorTouchPhaseEnded:
+    case DFPrivateSimulatorTouchPhaseCancelled:
+        return DFIndigoMouseEventUp;
+    }
+}
+
+static uint32_t DFIndigoMouseDirectionForPhase(DFPrivateSimulatorTouchPhase phase) {
+    switch (phase) {
+    case DFPrivateSimulatorTouchPhaseBegan:
+        return DFIndigoMouseDirectionDown;
+    case DFPrivateSimulatorTouchPhaseMoved:
+        return DFIndigoMouseDirectionMove;
+    case DFPrivateSimulatorTouchPhaseEnded:
+    case DFPrivateSimulatorTouchPhaseCancelled:
+        return DFIndigoMouseDirectionUp;
+    }
+}
+
+static IndigoHIDMessage *DFCreateIndigoTouchMessage9(CGPoint normalizedPoint, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) {
+    if (!DFShouldUseIndigoMouse9Path()) {
+        return NULL;
+    }
+    DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
+    if (mouseMessage == NULL) {
+        return NULL;
+    }
+
+    CGPoint ratioPoint = CGPointMake(
+        fmax(0.0, fmin(1.0, normalizedPoint.x)),
+        fmax(0.0, fmin(1.0, normalizedPoint.y))
+    );
+    return mouseMessage(&ratioPoint,
+                        NULL,
+                        DFIndigoTouchTarget,
+                        DFIndigoMouseEventTypeForPhase(phase),
+                        DFIndigoMouseDirectionForPhase(phase),
+                        1.0,
+                        1.0,
+                        displaySize.width,
+                        displaySize.height);
+}
+
+static IndigoHIDMessage *DFCreateIndigoMultiTouchMessage9(CGPoint normalizedPoint1, CGPoint normalizedPoint2, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) {
+    if (!DFShouldUseIndigoMouse9Path()) {
+        return NULL;
+    }
+    DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
+    if (mouseMessage == NULL) {
+        return NULL;
+    }
+
+    CGPoint ratioPoint = CGPointMake(
+        fmax(0.0, fmin(1.0, normalizedPoint1.x)),
+        fmax(0.0, fmin(1.0, normalizedPoint1.y))
+    );
+    CGPoint secondRatioPoint = CGPointMake(
+        fmax(0.0, fmin(1.0, normalizedPoint2.x)),
+        fmax(0.0, fmin(1.0, normalizedPoint2.y))
+    );
+    return mouseMessage(&ratioPoint,
+                        &secondRatioPoint,
+                        DFIndigoTouchTarget,
+                        DFIndigoMouseEventTypeForPhase(phase),
+                        DFIndigoMouseDirectionForPhase(phase),
+                        1.0,
+                        1.0,
+                        displaySize.width,
+                        displaySize.height);
+}
+
+static void DFWarmIndigoHIDServices(id hidClient) {
+    if (hidClient == nil) {
+        return;
+    }
+
+    DFIndigoHIDServiceMessageFn createPointer = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreatePointerService");
+    DFIndigoHIDServiceMessageFn createMouse = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreateMouseService");
+    NSError *error = nil;
+    if (createPointer != NULL) {
+        IndigoHIDMessage *message = createPointer();
+        if (message != NULL) {
+            (void)DFSendHIDMessage(hidClient, message, YES, &error);
+            usleep(20 * 1000);
+        }
+    }
+    if (createMouse != NULL) {
+        IndigoHIDMessage *message = createMouse();
+        if (message != NULL) {
+            (void)DFSendHIDMessage(hidClient, message, YES, &error);
+            usleep(20 * 1000);
+        }
+    }
+    if (error != nil) {
+        DFLog(@"Indigo HID service warm-up reported: %@", error.localizedDescription ?: @"unknown error");
+    }
+}
+
 static DFIndigoMessage *DFCreateIndigoTouchMessage(CGPoint normalizedPoint, NSSize displaySize, BOOL touchDown, NSError **error) {
     DFIndigoHIDMessageForMouseNSEventFn mouseMessage = (DFIndigoHIDMessageForMouseNSEventFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
     if (mouseMessage == NULL) {
@@ -2378,6 +2539,7 @@ - (nullable instancetype)initWithUDID:(NSString *)udid
 
         if (_hidClient != nil) {
             DFLog(@"Created private SimulatorKit HID client for %@", udid);
+            DFWarmIndigoHIDServices(_hidClient);
         } else {
             DFLog(@"Failed to create private SimulatorKit HID client for %@: %@", udid, hidClientError.localizedDescription ?: @"unknown error");
         }
@@ -3311,14 +3473,18 @@ - (BOOL)sendTouchAtNormalizedX:(double)normalizedX
             phaseLabel = phase == DFPrivateSimulatorTouchPhaseEnded ? @"ended" : @"cancelled";
             break;
         }
-        BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
-        DFIndigoMessage *message = DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError);
+        IndigoHIDMessage *message = DFCreateIndigoTouchMessage9(CGPointMake(clampedX, clampedY), displaySize, phase);
+        BOOL freeWhenDone = YES;
         if (message == NULL) {
-            return;
+            BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
+            message = (IndigoHIDMessage *)DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError);
+            if (message == NULL) {
+                return;
+            }
         }
 
         NSError *messageError = nil;
-        if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) {
+        if (!DFSendHIDMessage(self->_hidClient, message, freeWhenDone, &messageError)) {
             dispatchError = messageError ?: DFMakeError(
                 DFPrivateSimulatorErrorCodeTouchDispatchFailed,
                 @"SimulatorKit rejected the Indigo HID touch packet."
@@ -3394,14 +3560,25 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1
             break;
         }
 
-        BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
-        DFIndigoMessage *message = DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError);
+        IndigoHIDMessage *message = NULL;
+        const NSUInteger maxAttempts = phase == DFPrivateSimulatorTouchPhaseMoved ? 12 : 3;
+        for (NSUInteger attempt = 0; attempt < maxAttempts; attempt++) {
+            message = DFCreateIndigoMultiTouchMessage9(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, phase);
+            if (message != NULL) {
+                break;
+            }
+            usleep(5 * 1000);
+        }
         if (message == NULL) {
-            return;
+            BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved;
+            message = (IndigoHIDMessage *)DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError);
+            if (message == NULL) {
+                return;
+            }
         }
 
         NSError *messageError = nil;
-        if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) {
+        if (!DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) {
             dispatchError = messageError ?: DFMakeError(
                 DFPrivateSimulatorErrorCodeTouchDispatchFailed,
                 @"SimulatorKit rejected the Indigo HID multi-touch packet."

client/src/features/stream/streamWorkerClient.ts

@@ -35,6 +35,18 @@ export function sendWebRtcClientStats(stats: unknown): boolean {
   );
 }
 
+export function sendWebRtcStreamControl(options: {
+  forceKeyframe?: boolean;
+  fps?: number;
+  profile?: "focus" | "full" | "paused" | "thumb" | "thumbnail";
+  snapshot?: boolean;
+}): boolean {
+  return sendDataChannelMessage(
+    activeWebRtcControlChannel,
+    JSON.stringify({ ...options, type: "streamControl" }),
+  );
+}
+
 function sendDataChannelMessage(
   channel: RTCDataChannel | null,
   encoded: string,
@@ -63,6 +75,7 @@ interface StreamClientBackend {
   connect(target: StreamConnectTarget): void | Promise<void>;
   destroy(): void;
   disconnect(): void;
+  sendControl?(payload: unknown): boolean;
 }
 
 class WebRtcStreamClient implements StreamClientBackend {
@@ -318,6 +331,10 @@ class WebRtcStreamClient implements StreamClientBackend {
     this.onMessage({ type: "status", status: { state: "idle" } });
   }
 
+  sendControl(payload: unknown): boolean {
+    return sendDataChannelMessage(this.controlChannel, JSON.stringify(payload));
+  }
+
   destroy() {
     this.disconnect();
   }
@@ -989,9 +1006,6 @@ export class StreamWorkerClient {
 
   constructor(onMessage: (message: WorkerToMainMessage) => void) {
     this.onMessage = onMessage;
-    if (activeStreamClient && activeStreamClient !== this) {
-      activeStreamClient.destroy();
-    }
     activeStreamClient = this;
   }
 
@@ -1038,6 +1052,17 @@ export class StreamWorkerClient {
     this.backend?.clear();
   }
 
+  sendStreamControl(options: {
+    forceKeyframe?: boolean;
+    fps?: number;
+    profile?: "focus" | "full" | "paused" | "thumb" | "thumbnail";
+    snapshot?: boolean;
+  }) {
+    return Boolean(
+      this.backend?.sendControl?.({ ...options, type: "streamControl" }),
+    );
+  }
+
   destroy() {
     if (this.disposed) {
       return;

client/src/features/stream/useLiveStream.ts

@@ -28,6 +28,7 @@ interface UseLiveStreamOptions {
   paused?: boolean;
   remote?: boolean;
   simulator: SimulatorMetadata | null;
+  streamProfile?: "focus" | "full" | "paused" | "thumb" | "thumbnail";
 }
 
 interface UseLiveStreamResult {
@@ -72,6 +73,7 @@ export function useLiveStream({
   paused = false,
   remote = false,
   simulator,
+  streamProfile = "focus",
 }: UseLiveStreamOptions): UseLiveStreamResult {
   const clientTelemetryIdRef = useRef("");
   const workerClientRef = useRef<StreamWorkerClient | null>(null);
@@ -253,6 +255,31 @@ export function useLiveStream({
     };
   }, [canvasElement, simulator?.isBooted, simulator?.udid, paused, remote]);
 
+  useEffect(() => {
+    if (paused || !simulator?.isBooted) {
+      return;
+    }
+    let attempts = 0;
+    const send = () => {
+      attempts += 1;
+      return Boolean(
+        workerClientRef.current?.sendStreamControl({
+          forceKeyframe: streamProfile === "focus" || streamProfile === "full",
+          profile: streamProfile,
+        }),
+      );
+    };
+    if (send()) {
+      return;
+    }
+    const interval = window.setInterval(() => {
+      if (send() || attempts >= 8) {
+        window.clearInterval(interval);
+      }
+    }, 250);
+    return () => window.clearInterval(interval);
+  }, [paused, simulator?.isBooted, simulator?.udid, streamProfile]);
+
   useEffect(() => {
     if (!simulator?.udid) {
       return;