diff --git a/README.md b/README.md index c5050c5..af358c4 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ simdeck toggle-appearance simdeck pasteboard set "hello" simdeck pasteboard get simdeck screenshot --output screen.png +simdeck stream --frames 120 > stream.h264 simdeck describe simdeck describe --format agent --max-depth 4 simdeck describe --point 120,240 @@ -179,6 +180,13 @@ simdeck chrome-profile simdeck logs --seconds 30 --limit 200 ``` +`boot` prefers SimDeck's private CoreSimulator boot path so it can start devices +without launching Simulator.app, then falls back to `xcrun simctl` when private +booting is unavailable. + +`stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or +external tools such as `ffplay`. + `describe` uses the project daemon to prefer React Native, NativeScript, or UIKit in-app inspectors, then falls back to the built-in private CoreSimulator accessibility bridge. Use `--format agent` or `--format compact-json` for diff --git a/cli/DFPrivateSimulatorDisplayBridge.m b/cli/DFPrivateSimulatorDisplayBridge.m index 7794843..5e73359 100644 --- a/cli/DFPrivateSimulatorDisplayBridge.m +++ b/cli/DFPrivateSimulatorDisplayBridge.m @@ -32,10 +32,12 @@ typedef uint32_t IndigoHIDEdge; typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEventFn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, NSEventType type, NSSize displaySize, IndigoHIDEdge edge); +typedef IndigoHIDMessage *(*DFIndigoHIDMessageForMouseNSEvent9Fn)(CGPoint *location, CGPoint *windowLocation, uint32_t target, uint32_t eventType, uint32_t direction, double unused1, double unused2, double widthPoints, double heightPoints); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardArbitraryFn)(int keyCode, int op); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForKeyboardNSEventFn)(NSEvent *event); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForButtonFn)(uint32_t buttonCode, uint32_t operation, uint32_t target); typedef IndigoHIDMessage *(*DFIndigoHIDMessageForHIDArbitraryFn)(uint32_t target, uint32_t page, uint32_t usage, uint32_t operation); +typedef IndigoHIDMessage *(*DFIndigoHIDServiceMessageFn)(void); #pragma pack(push, 4) typedef struct { @@ -91,6 +93,12 @@ static const uint32_t DFIndigoTouchTarget = 0x32; static const uint8_t DFIndigoEventTypeTouch = 0x02; static const uint32_t DFIndigoTouchEventKind = 0x0b; +static const uint32_t DFIndigoMouseEventDown = 1; +static const uint32_t DFIndigoMouseEventUp = 2; +static const uint32_t DFIndigoMouseEventDragged = 6; +static const uint32_t DFIndigoMouseDirectionDown = 1; +static const uint32_t DFIndigoMouseDirectionMove = 0; +static const uint32_t DFIndigoMouseDirectionUp = 2; static const int DFKeyboardDirectionDown = 1; static const int DFKeyboardDirectionUp = 2; static const uint32_t DFButtonDirectionDown = 1; @@ -128,6 +136,39 @@ return @"/Applications/Xcode.app/Contents/Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit"; } +static NSInteger DFXcodeMajorVersion(void) { + NSString *developerPath = nil; + const char *developerDir = getenv("DEVELOPER_DIR"); + if (developerDir != NULL && developerDir[0] != '\0') { + developerPath = [NSString stringWithUTF8String:developerDir]; + } else { + FILE *pipe = popen("/usr/bin/xcode-select -p 2>/dev/null", "r"); + if (pipe != NULL) { + char buffer[PATH_MAX] = {0}; + if (fgets(buffer, sizeof(buffer), pipe) != NULL) { + developerPath = [[NSString stringWithUTF8String:buffer] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]]; + } + pclose(pipe); + } + } + if (developerPath.length == 0) { + return 0; + } + + NSString *contentsPath = developerPath; + if ([contentsPath.lastPathComponent isEqualToString:@"Developer"]) { + contentsPath = contentsPath.stringByDeletingLastPathComponent; + } + + NSDictionary *versionInfo = [NSDictionary dictionaryWithContentsOfFile:[contentsPath stringByAppendingPathComponent:@"version.plist"]]; + NSString *version = versionInfo[@"CFBundleShortVersionString"]; + if (version.length == 0) { + NSDictionary *info = [NSDictionary dictionaryWithContentsOfFile:[contentsPath stringByAppendingPathComponent:@"Info.plist"]]; + version = info[@"CFBundleShortVersionString"]; + } + return [[version componentsSeparatedByString:@"."].firstObject integerValue]; +} + typedef struct { __unsafe_unretained id unit; double value; @@ -187,6 +228,23 @@ static BOOL DFVerboseTouchLoggingEnabled(void) { return enabled; } +static BOOL DFShouldUseIndigoMouse9Path(void) { + static BOOL enabled = NO; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSString *override = NSProcessInfo.processInfo.environment[@"SIMDECK_INDIGO_MOUSE_9ARG"]; + if (override.length > 0) { + enabled = [override isEqualToString:@"1"] || + [override caseInsensitiveCompare:@"true"] == NSOrderedSame || + [override caseInsensitiveCompare:@"yes"] == NSOrderedSame; + return; + } + + enabled = DFXcodeMajorVersion() >= 26; + }); + return enabled; +} + #pragma mark - SimulatorKit Swift symbol resolver // // We call into SimulatorKit's private Swift API by dlsym'ing mangled symbol @@ -1002,6 +1060,109 @@ static BOOL DFSendHIDMessage(id hidClient, IndigoHIDMessage *message, BOOL freeW return YES; } +static uint32_t DFIndigoMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phase) { + switch (phase) { + case DFPrivateSimulatorTouchPhaseBegan: + return DFIndigoMouseEventDown; + case DFPrivateSimulatorTouchPhaseMoved: + return DFIndigoMouseEventDragged; + case DFPrivateSimulatorTouchPhaseEnded: + case DFPrivateSimulatorTouchPhaseCancelled: + return DFIndigoMouseEventUp; + } +} + +static uint32_t DFIndigoMouseDirectionForPhase(DFPrivateSimulatorTouchPhase phase) { + switch (phase) { + case DFPrivateSimulatorTouchPhaseBegan: + return DFIndigoMouseDirectionDown; + case DFPrivateSimulatorTouchPhaseMoved: + return DFIndigoMouseDirectionMove; + case DFPrivateSimulatorTouchPhaseEnded: + case DFPrivateSimulatorTouchPhaseCancelled: + return DFIndigoMouseDirectionUp; + } +} + +static IndigoHIDMessage *DFCreateIndigoTouchMessage9(CGPoint normalizedPoint, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) { + if (!DFShouldUseIndigoMouse9Path()) { + return NULL; + } + DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent"); + if (mouseMessage == NULL) { + return NULL; + } + + CGPoint ratioPoint = CGPointMake( + fmax(0.0, fmin(1.0, normalizedPoint.x)), + fmax(0.0, fmin(1.0, normalizedPoint.y)) + ); + return mouseMessage(&ratioPoint, + NULL, + DFIndigoTouchTarget, + DFIndigoMouseEventTypeForPhase(phase), + DFIndigoMouseDirectionForPhase(phase), + 1.0, + 1.0, + displaySize.width, + displaySize.height); +} + +static IndigoHIDMessage *DFCreateIndigoMultiTouchMessage9(CGPoint normalizedPoint1, CGPoint normalizedPoint2, NSSize displaySize, DFPrivateSimulatorTouchPhase phase) { + if (!DFShouldUseIndigoMouse9Path()) { + return NULL; + } + DFIndigoHIDMessageForMouseNSEvent9Fn mouseMessage = (DFIndigoHIDMessageForMouseNSEvent9Fn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent"); + if (mouseMessage == NULL) { + return NULL; + } + + CGPoint ratioPoint = CGPointMake( + fmax(0.0, fmin(1.0, normalizedPoint1.x)), + fmax(0.0, fmin(1.0, normalizedPoint1.y)) + ); + CGPoint secondRatioPoint = CGPointMake( + fmax(0.0, fmin(1.0, normalizedPoint2.x)), + fmax(0.0, fmin(1.0, normalizedPoint2.y)) + ); + return mouseMessage(&ratioPoint, + &secondRatioPoint, + DFIndigoTouchTarget, + DFIndigoMouseEventTypeForPhase(phase), + DFIndigoMouseDirectionForPhase(phase), + 1.0, + 1.0, + displaySize.width, + displaySize.height); +} + +static void DFWarmIndigoHIDServices(id hidClient) { + if (hidClient == nil) { + return; + } + + DFIndigoHIDServiceMessageFn createPointer = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreatePointerService"); + DFIndigoHIDServiceMessageFn createMouse = (DFIndigoHIDServiceMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageToCreateMouseService"); + NSError *error = nil; + if (createPointer != NULL) { + IndigoHIDMessage *message = createPointer(); + if (message != NULL) { + (void)DFSendHIDMessage(hidClient, message, YES, &error); + usleep(20 * 1000); + } + } + if (createMouse != NULL) { + IndigoHIDMessage *message = createMouse(); + if (message != NULL) { + (void)DFSendHIDMessage(hidClient, message, YES, &error); + usleep(20 * 1000); + } + } + if (error != nil) { + DFLog(@"Indigo HID service warm-up reported: %@", error.localizedDescription ?: @"unknown error"); + } +} + static DFIndigoMessage *DFCreateIndigoTouchMessage(CGPoint normalizedPoint, NSSize displaySize, BOOL touchDown, NSError **error) { DFIndigoHIDMessageForMouseNSEventFn mouseMessage = (DFIndigoHIDMessageForMouseNSEventFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent"); if (mouseMessage == NULL) { @@ -2378,6 +2539,7 @@ - (nullable instancetype)initWithUDID:(NSString *)udid if (_hidClient != nil) { DFLog(@"Created private SimulatorKit HID client for %@", udid); + DFWarmIndigoHIDServices(_hidClient); } else { DFLog(@"Failed to create private SimulatorKit HID client for %@: %@", udid, hidClientError.localizedDescription ?: @"unknown error"); } @@ -3311,14 +3473,18 @@ - (BOOL)sendTouchAtNormalizedX:(double)normalizedX phaseLabel = phase == DFPrivateSimulatorTouchPhaseEnded ? @"ended" : @"cancelled"; break; } - BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; - DFIndigoMessage *message = DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError); + IndigoHIDMessage *message = DFCreateIndigoTouchMessage9(CGPointMake(clampedX, clampedY), displaySize, phase); + BOOL freeWhenDone = YES; if (message == NULL) { - return; + BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; + message = (IndigoHIDMessage *)DFCreateIndigoTouchMessage(CGPointMake(clampedX, clampedY), displaySize, touchDown, &dispatchError); + if (message == NULL) { + return; + } } NSError *messageError = nil; - if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) { + if (!DFSendHIDMessage(self->_hidClient, message, freeWhenDone, &messageError)) { dispatchError = messageError ?: DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @"SimulatorKit rejected the Indigo HID touch packet." @@ -3394,14 +3560,25 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1 break; } - BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; - DFIndigoMessage *message = DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError); + IndigoHIDMessage *message = NULL; + const NSUInteger maxAttempts = phase == DFPrivateSimulatorTouchPhaseMoved ? 12 : 3; + for (NSUInteger attempt = 0; attempt < maxAttempts; attempt++) { + message = DFCreateIndigoMultiTouchMessage9(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, phase); + if (message != NULL) { + break; + } + usleep(5 * 1000); + } if (message == NULL) { - return; + BOOL touchDown = phase == DFPrivateSimulatorTouchPhaseBegan || phase == DFPrivateSimulatorTouchPhaseMoved; + message = (IndigoHIDMessage *)DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1), CGPointMake(x2, y2), displaySize, touchDown, &dispatchError); + if (message == NULL) { + return; + } } NSError *messageError = nil; - if (!DFSendHIDMessage(self->_hidClient, (IndigoHIDMessage *)message, YES, &messageError)) { + if (!DFSendHIDMessage(self->_hidClient, message, YES, &messageError)) { dispatchError = messageError ?: DFMakeError( DFPrivateSimulatorErrorCodeTouchDispatchFailed, @"SimulatorKit rejected the Indigo HID multi-touch packet." diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index c24571a..d57b735 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -35,6 +35,18 @@ export function sendWebRtcClientStats(stats: unknown): boolean { ); } +export function sendWebRtcStreamControl(options: { + forceKeyframe?: boolean; + fps?: number; + profile?: "focus" | "full" | "paused" | "thumb" | "thumbnail"; + snapshot?: boolean; +}): boolean { + return sendDataChannelMessage( + activeWebRtcControlChannel, + JSON.stringify({ ...options, type: "streamControl" }), + ); +} + function sendDataChannelMessage( channel: RTCDataChannel | null, encoded: string, @@ -63,6 +75,7 @@ interface StreamClientBackend { connect(target: StreamConnectTarget): void | Promise; destroy(): void; disconnect(): void; + sendControl?(payload: unknown): boolean; } class WebRtcStreamClient implements StreamClientBackend { @@ -318,6 +331,10 @@ class WebRtcStreamClient implements StreamClientBackend { this.onMessage({ type: "status", status: { state: "idle" } }); } + sendControl(payload: unknown): boolean { + return sendDataChannelMessage(this.controlChannel, JSON.stringify(payload)); + } + destroy() { this.disconnect(); } @@ -989,9 +1006,6 @@ export class StreamWorkerClient { constructor(onMessage: (message: WorkerToMainMessage) => void) { this.onMessage = onMessage; - if (activeStreamClient && activeStreamClient !== this) { - activeStreamClient.destroy(); - } activeStreamClient = this; } @@ -1038,6 +1052,17 @@ export class StreamWorkerClient { this.backend?.clear(); } + sendStreamControl(options: { + forceKeyframe?: boolean; + fps?: number; + profile?: "focus" | "full" | "paused" | "thumb" | "thumbnail"; + snapshot?: boolean; + }) { + return Boolean( + this.backend?.sendControl?.({ ...options, type: "streamControl" }), + ); + } + destroy() { if (this.disposed) { return; diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index 4b6fc62..d9fadc2 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -28,6 +28,7 @@ interface UseLiveStreamOptions { paused?: boolean; remote?: boolean; simulator: SimulatorMetadata | null; + streamProfile?: "focus" | "full" | "paused" | "thumb" | "thumbnail"; } interface UseLiveStreamResult { @@ -72,6 +73,7 @@ export function useLiveStream({ paused = false, remote = false, simulator, + streamProfile = "focus", }: UseLiveStreamOptions): UseLiveStreamResult { const clientTelemetryIdRef = useRef(""); const workerClientRef = useRef(null); @@ -253,6 +255,31 @@ export function useLiveStream({ }; }, [canvasElement, simulator?.isBooted, simulator?.udid, paused, remote]); + useEffect(() => { + if (paused || !simulator?.isBooted) { + return; + } + let attempts = 0; + const send = () => { + attempts += 1; + return Boolean( + workerClientRef.current?.sendStreamControl({ + forceKeyframe: streamProfile === "focus" || streamProfile === "full", + profile: streamProfile, + }), + ); + }; + if (send()) { + return; + } + const interval = window.setInterval(() => { + if (send() || attempts >= 8) { + window.clearInterval(interval); + } + }, 250); + return () => window.clearInterval(interval); + }, [paused, simulator?.isBooted, simulator?.udid, streamProfile]); + useEffect(() => { if (!simulator?.udid) { return; diff --git a/docs/api/rest.md b/docs/api/rest.md index 183146f..ffc1c7e 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -149,6 +149,16 @@ and the server responds with an SDP answer for a receive-only H.264 video track: The endpoint requires the active simulator stream to produce H.264-compatible samples. The bundled browser client always uses this endpoint. +The browser also opens a `simdeck-control` data channel. In addition to input +messages, clients can tune the stream attached to that peer: + +```json +{ "type": "streamControl", "profile": "thumb" } +``` + +Supported profiles are `thumb`/`thumbnail`, `focus`/`full`, and `paused`. +Clients may also send `fps`, `forceKeyframe`, or `snapshot` fields. + ### `POST /api/simulators/{udid}/open-url` Opens a URL inside the simulator: diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 0d34efa..52ad183 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -237,12 +237,17 @@ Batch input can come from `--step`, `--file`, or `--stdin`. It fails fast by def ```sh simdeck screenshot --output screen.png simdeck screenshot --stdout > screen.png +simdeck stream --frames 120 > stream.h264 simdeck pasteboard set "hello" simdeck pasteboard get simdeck logs --seconds 30 --limit 200 simdeck chrome-profile ``` +`stream` writes Annex B H.264 samples to stdout and runs until interrupted, or +until `--frames` samples have been written. It is intended for diagnostics and +external tools. + `logs` fetches recent simulator logs. `chrome-profile` returns the CoreSimulator chrome layout used by the browser viewport. ## HTTP Fast Path diff --git a/server/src/main.rs b/server/src/main.rs index c367f5c..a8ab6cf 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -177,6 +177,11 @@ enum Command { #[arg(long)] stdout: bool, }, + Stream { + udid: String, + #[arg(long, default_value_t = 0)] + frames: u64, + }, #[command(name = "describe")] DescribeUi { udid: String, @@ -1939,6 +1944,7 @@ fn main() -> anyhow::Result<()> { } Ok(()) } + Command::Stream { udid, frames } => run_stream_stdout(&bridge, udid, frames), Command::DescribeUi { udid, point, @@ -2791,6 +2797,38 @@ fn default_screenshot_path(udid: &str) -> PathBuf { PathBuf::from(format!("Simulator Screenshot - {udid} - {timestamp}.png")) } +fn run_stream_stdout(bridge: &NativeBridge, udid: String, frames: u64) -> anyhow::Result<()> { + let metrics = Arc::new(Metrics::default()); + let session = simulators::session::SimulatorSession::new(bridge, udid, metrics) + .map_err(|error| anyhow::anyhow!("{error}"))?; + session + .ensure_started() + .map_err(|error| anyhow::anyhow!("{error}"))?; + session.request_keyframe(); + + let mut receiver = session.subscribe(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .context("create stream runtime")?; + let mut stdout = io::stdout().lock(); + let mut written = 0u64; + runtime.block_on(async { + loop { + if frames > 0 && written >= frames { + break; + } + let frame = receiver.recv().await?; + let sample = crate::transport::webrtc::h264_annex_b_sample(&frame) + .map_err(|error| anyhow::anyhow!("encode Annex B frame: {error}"))?; + stdout.write_all(&sample)?; + stdout.flush()?; + written += 1; + } + anyhow::Ok(()) + }) +} + #[allow(clippy::too_many_arguments)] fn describe_ui_snapshot( bridge: &NativeBridge, diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index 77c6230..e5d2fff 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; -use tokio::sync::broadcast; +use tokio::sync::{broadcast, mpsc}; use tokio::task; use tokio::time; use tracing::{info, warn}; @@ -166,11 +166,13 @@ pub async fn create_answer( .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, ); register_diagnostics(&peer_connection, &udid); + let (stream_control_tx, stream_control_rx) = mpsc::unbounded_channel(); register_control_data_channel( &peer_connection, session.clone(), state.clone(), udid.clone(), + stream_control_tx, ); let video_track = Arc::new(TrackLocalStaticRTP::new( @@ -232,6 +234,7 @@ pub async fn create_answer( video_track, cancellation_token, cancellation, + stream_control_rx, } .run(), ); @@ -344,17 +347,19 @@ fn register_control_data_channel( session: crate::simulators::session::SimulatorSession, state: AppState, udid: String, + stream_control_tx: mpsc::UnboundedSender, ) { peer_connection.on_data_channel(Box::new(move |channel: Arc| { let session = session.clone(); let state = state.clone(); let udid = udid.clone(); + let stream_control_tx = stream_control_tx.clone(); Box::pin(async move { let label = channel.label(); if label != WEBRTC_CONTROL_CHANNEL_LABEL && label != WEBRTC_TELEMETRY_CHANNEL_LABEL { return; } - attach_control_data_channel(channel, session, state, udid); + attach_control_data_channel(channel, session, state, udid, stream_control_tx); }) })); } @@ -364,11 +369,13 @@ fn attach_control_data_channel( session: crate::simulators::session::SimulatorSession, state: AppState, udid: String, + stream_control_tx: mpsc::UnboundedSender, ) { channel.on_message(Box::new(move |message: DataChannelMessage| { let session = session.clone(); let state = state.clone(); let udid = udid.clone(); + let stream_control_tx = stream_control_tx.clone(); Box::pin(async move { let Ok(text) = std::str::from_utf8(&message.data) else { warn!("Invalid WebRTC control message bytes for {udid}"); @@ -378,9 +385,26 @@ fn attach_control_data_channel( match message { WebRtcDataChannelMessage::ClientStats { stats } => { if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { - state.metrics.record_client_stream_stats(stats); + state.metrics.record_client_stream_stats(*stats); } } + WebRtcDataChannelMessage::StreamControl { + fps, + force_keyframe, + profile, + snapshot, + } => { + let command = WebRtcStreamCommand::from_payload( + profile.as_deref(), + fps, + force_keyframe.unwrap_or(false), + snapshot.unwrap_or(false), + ); + if command.force_keyframe || command.snapshot { + session.request_keyframe(); + } + let _ = stream_control_tx.send(command); + } } return; } @@ -419,7 +443,45 @@ fn attach_control_data_channel( #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "camelCase")] enum WebRtcDataChannelMessage { - ClientStats { stats: ClientStreamStats }, + ClientStats { + stats: Box, + }, + StreamControl { + fps: Option, + #[serde(rename = "forceKeyframe")] + force_keyframe: Option, + profile: Option, + snapshot: Option, + }, +} + +#[derive(Clone, Debug)] +struct WebRtcStreamCommand { + force_keyframe: bool, + snapshot: bool, + target_frame_interval: Option, +} + +impl WebRtcStreamCommand { + fn from_payload( + profile: Option<&str>, + fps: Option, + force_keyframe: bool, + snapshot: bool, + ) -> Self { + let profile_fps = match profile.map(str::trim) { + Some("thumbnail") | Some("thumb") => Some(8), + Some("paused") => Some(1), + Some("focus") | Some("full") => Some(60), + _ => None, + }; + let fps = fps.or(profile_fps).map(|value| value.clamp(1, 60)); + Self { + force_keyframe, + snapshot, + target_frame_interval: fps.map(|fps| Duration::from_micros(1_000_000 / fps as u64)), + } + } } fn is_h264_codec(codec: &str) -> bool { @@ -660,6 +722,7 @@ struct WebRtcMediaStream { video_track: Arc, cancellation_token: broadcast::Sender<()>, cancellation: broadcast::Receiver<()>, + stream_control_rx: mpsc::UnboundedReceiver, } impl WebRtcMediaStream { @@ -673,6 +736,7 @@ impl WebRtcMediaStream { video_track, cancellation_token, mut cancellation, + mut stream_control_rx, } = self; let mut rx = session.subscribe(); let mut latest_keyframe = first_frame.clone(); @@ -693,6 +757,8 @@ impl WebRtcMediaStream { let mut refresh_sleep = Box::pin(time::sleep(refresh_floor)); let mut adaptive_refresh_interval = refresh_floor; let mut bootstrap_frames_remaining = WEBRTC_BOOTSTRAP_KEYFRAME_REPEATS; + let mut target_frame_interval: Option = None; + let mut last_sent_at: Option = None; let mut waiting_for_keyframe = false; let mut peer_disconnected_since: Option = None; let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); @@ -707,6 +773,7 @@ impl WebRtcMediaStream { .await { Ok(true) => { + last_sent_at = Some(time::Instant::now()); state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); } Ok(false) => { @@ -792,6 +859,18 @@ impl WebRtcMediaStream { .as_mut() .reset(time::Instant::now() + adaptive_refresh_interval); } + command = stream_control_rx.recv() => { + let Some(command) = command else { + continue; + }; + target_frame_interval = command.target_frame_interval; + if command.force_keyframe || command.snapshot { + waiting_for_keyframe = true; + session.request_keyframe(); + } else { + session.request_refresh(); + } + } frame = rx.recv() => { let frame = match frame { Ok(frame) => frame, @@ -829,6 +908,12 @@ impl WebRtcMediaStream { latest_keyframe = frame.clone(); waiting_for_keyframe = false; } + if let (Some(interval), Some(previous)) = (target_frame_interval, last_sent_at) { + if previous.elapsed() < interval && !frame.is_keyframe { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + continue; + } + } let duration = send_timing.duration_for(&frame, realtime_stream); let started_at = time::Instant::now(); let write_result = write_frame_sample_with_timeout( @@ -843,6 +928,7 @@ impl WebRtcMediaStream { adaptive_interval_for_write(started_at.elapsed(), refresh_floor, refresh_ceiling); match write_result { Ok(true) => { + last_sent_at = Some(time::Instant::now()); state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); } Ok(false) => { @@ -1032,7 +1118,9 @@ pub fn realtime_stream_enabled() -> bool { }) } -fn h264_annex_b_sample(frame: &crate::transport::packet::FramePacket) -> anyhow::Result> { +pub fn h264_annex_b_sample( + frame: &crate::transport::packet::FramePacket, +) -> anyhow::Result> { let data = frame.data.as_ref(); let description = frame.description.as_ref().map(bytes::Bytes::as_ref); let mut sample = Vec::with_capacity(data.len() + description.map_or(0, |bytes| bytes.len())); diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index 244ef2b..481debc 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -213,11 +213,13 @@ await simdeck.batch(udid, [ ```bash simdeck screenshot --output screen.png simdeck screenshot --stdout > screen.png +simdeck stream --frames 120 > stream.h264 simdeck logs --seconds 30 --limit 200 simdeck chrome-profile ``` -Use screenshots for still evidence. +Use screenshots for still evidence. Use `stream` when a diagnostic needs raw +H.264 samples for an external player or capture pipeline. ## Default Loop