Skip to content

Commit 7337c06

Browse files
louis030195louis030195
committed
desync keypress in thread, remove ui element capture on noisy events
1 parent 5baea85 commit 7337c06

File tree

2 files changed

+168
-157
lines changed

2 files changed

+168
-157
lines changed

‎terminator-workflow-recorder/src/recorder/windows/mod.rs‎

Lines changed: 87 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,7 @@ impl WindowsRecorder {
354354
&uia_processor_events_counter,
355355
);
356356
}
357+
// TODO we never are going to capture ui element for mouse movement and scroll, too noisy?
357358
UIAInputRequest::MouseMove { position } => {
358359
Self::handle_mouse_move_request(
359360
&position,
@@ -373,6 +374,13 @@ impl WindowsRecorder {
373374
&uia_processor_events_counter,
374375
);
375376
}
377+
UIAInputRequest::KeyPressForCompletion { key_code } => {
378+
Self::handle_key_press_for_completion_request(
379+
key_code,
380+
&uia_processor_text_input,
381+
&uia_processor_event_tx,
382+
);
383+
}
376384
}
377385
}
378386
});
@@ -405,37 +413,11 @@ impl WindowsRecorder {
405413

406414
// Check for completion trigger keys (Enter, Tab)
407415
if key_code == 0x0D || key_code == 0x09 {
408-
let is_suggestion_enter = if key_code == 0x0D {
409-
text_input.handle_enter_key()
410-
} else {
411-
false
412-
};
413-
let completion_reason = if is_suggestion_enter {
414-
"suggestion_enter"
415-
} else {
416-
"trigger_key"
417-
};
418-
419-
if text_input.should_emit_completion(completion_reason) {
420-
let input_method = if is_suggestion_enter {
421-
Some(crate::TextInputMethod::Suggestion)
422-
} else {
423-
None
424-
};
425-
if let Some(text_event) =
426-
text_input.get_completion_event(input_method)
427-
{
428-
let _ = event_tx.send(
429-
WorkflowEvent::TextInputCompleted(text_event),
430-
);
431-
if let Some(element) =
432-
&tracker.as_ref().map(|t| t.element.clone())
433-
{
434-
*tracker = Some(TextInputTracker::new(
435-
element.clone(),
436-
));
437-
}
438-
}
416+
// Offload the blocking work to the UIA thread
417+
let request =
418+
UIAInputRequest::KeyPressForCompletion { key_code };
419+
if uia_event_tx.send(request).is_err() {
420+
debug!("Failed to send key press completion request to UIA thread");
439421
}
440422
}
441423
}
@@ -632,62 +614,49 @@ impl WindowsRecorder {
632614

633615
if should_record {
634616
let position = Position { x, y };
635-
if capture_ui_elements_rdev {
636-
let request = UIAInputRequest::MouseMove { position };
637-
let _ = uia_event_tx.send(request);
638-
} else {
639-
let mouse_event = MouseEvent {
640-
event_type: MouseEventType::Move,
641-
button: MouseButton::Left,
642-
position,
643-
scroll_delta: None,
644-
drag_start: None,
645-
metadata: EventMetadata {
646-
ui_element: None,
647-
timestamp: Some(Self::capture_timestamp()),
648-
},
649-
};
650-
Self::send_filtered_event_static(
651-
&event_tx,
652-
&config,
653-
&performance_last_event_time,
654-
&performance_events_counter,
655-
WorkflowEvent::Mouse(mouse_event),
656-
);
657-
}
617+
618+
let mouse_event = MouseEvent {
619+
event_type: MouseEventType::Move,
620+
button: MouseButton::Left,
621+
position,
622+
scroll_delta: None,
623+
drag_start: None,
624+
metadata: EventMetadata {
625+
ui_element: None,
626+
timestamp: Some(Self::capture_timestamp()),
627+
},
628+
};
629+
Self::send_filtered_event_static(
630+
&event_tx,
631+
&config,
632+
&performance_last_event_time,
633+
&performance_events_counter,
634+
WorkflowEvent::Mouse(mouse_event),
635+
);
658636
}
659637
}
660638
EventType::Wheel { delta_x, delta_y } => {
661639
if let Some((x, y)) = *last_mouse_pos.lock().unwrap() {
662640
let position = Position { x, y };
663-
if capture_ui_elements_rdev {
664-
let request = UIAInputRequest::Wheel {
665-
delta: (delta_x as i32, delta_y as i32),
666-
position,
667-
};
668-
if uia_event_tx.send(request).is_err() {
669-
debug!("Failed to send wheel event to UIA processor thread");
670-
}
671-
} else {
672-
let mouse_event = MouseEvent {
673-
event_type: MouseEventType::Wheel,
674-
button: MouseButton::Middle, // Common for wheel
675-
position,
676-
scroll_delta: Some((delta_x as i32, delta_y as i32)),
677-
drag_start: None,
678-
metadata: EventMetadata {
679-
ui_element: None,
680-
timestamp: Some(Self::capture_timestamp()),
681-
},
682-
};
683-
Self::send_filtered_event_static(
684-
&event_tx,
685-
&config,
686-
&performance_last_event_time,
687-
&performance_events_counter,
688-
WorkflowEvent::Mouse(mouse_event),
689-
);
690-
}
641+
642+
let mouse_event = MouseEvent {
643+
event_type: MouseEventType::Wheel,
644+
button: MouseButton::Middle, // Common for wheel
645+
position,
646+
scroll_delta: Some((delta_x as i32, delta_y as i32)),
647+
drag_start: None,
648+
metadata: EventMetadata {
649+
ui_element: None,
650+
timestamp: Some(Self::capture_timestamp()),
651+
},
652+
};
653+
Self::send_filtered_event_static(
654+
&event_tx,
655+
&config,
656+
&performance_last_event_time,
657+
&performance_events_counter,
658+
WorkflowEvent::Mouse(mouse_event),
659+
);
691660
}
692661
}
693662
}
@@ -2166,81 +2135,42 @@ impl WindowsRecorder {
21662135
}
21672136
}
21682137
}
2169-
}
21702138

2171-
/// Convert a Key to a u32
2172-
fn key_to_u32(key: &Key) -> u32 {
2173-
match key {
2174-
Key::KeyA => 0x41,
2175-
Key::KeyB => 0x42,
2176-
Key::KeyC => 0x43,
2177-
Key::KeyD => 0x44,
2178-
Key::KeyE => 0x45,
2179-
Key::KeyF => 0x46,
2180-
Key::KeyG => 0x47,
2181-
Key::KeyH => 0x48,
2182-
Key::KeyI => 0x49,
2183-
Key::KeyJ => 0x4A,
2184-
Key::KeyK => 0x4B,
2185-
Key::KeyL => 0x4C,
2186-
Key::KeyM => 0x4D,
2187-
Key::KeyN => 0x4E,
2188-
Key::KeyO => 0x4F,
2189-
Key::KeyP => 0x50,
2190-
Key::KeyQ => 0x51,
2191-
Key::KeyR => 0x52,
2192-
Key::KeyS => 0x53,
2193-
Key::KeyT => 0x54,
2194-
Key::KeyU => 0x55,
2195-
Key::KeyV => 0x56,
2196-
Key::KeyW => 0x57,
2197-
Key::KeyX => 0x58,
2198-
Key::KeyY => 0x59,
2199-
Key::KeyZ => 0x5A,
2200-
Key::Num0 => 0x30,
2201-
Key::Num1 => 0x31,
2202-
Key::Num2 => 0x32,
2203-
Key::Num3 => 0x33,
2204-
Key::Num4 => 0x34,
2205-
Key::Num5 => 0x35,
2206-
Key::Num6 => 0x36,
2207-
Key::Num7 => 0x37,
2208-
Key::Num8 => 0x38,
2209-
Key::Num9 => 0x39,
2210-
Key::Escape => 0x1B,
2211-
Key::Backspace => 0x08,
2212-
Key::Tab => 0x09,
2213-
Key::Return => 0x0D,
2214-
Key::Space => 0x20,
2215-
Key::LeftArrow => 0x25,
2216-
Key::UpArrow => 0x26,
2217-
Key::RightArrow => 0x27,
2218-
Key::DownArrow => 0x28,
2219-
Key::Delete => 0x2E,
2220-
Key::Home => 0x24,
2221-
Key::End => 0x23,
2222-
Key::PageUp => 0x21,
2223-
Key::PageDown => 0x22,
2224-
Key::F1 => 0x70,
2225-
Key::F2 => 0x71,
2226-
Key::F3 => 0x72,
2227-
Key::F4 => 0x73,
2228-
Key::F5 => 0x74,
2229-
Key::F6 => 0x75,
2230-
Key::F7 => 0x76,
2231-
Key::F8 => 0x77,
2232-
Key::F9 => 0x78,
2233-
Key::F10 => 0x79,
2234-
Key::F11 => 0x7A,
2235-
Key::F12 => 0x7B,
2236-
Key::ShiftLeft => 0xA0,
2237-
Key::ShiftRight => 0xA1,
2238-
Key::ControlLeft => 0xA2,
2239-
Key::ControlRight => 0xA3,
2240-
Key::Alt => 0xA4,
2241-
Key::AltGr => 0xA5,
2242-
Key::MetaLeft => 0x5B,
2243-
Key::MetaRight => 0x5C,
2244-
_ => 0,
2139+
/// Handles a key press completion request from the input listener thread.
2140+
/// This function performs the UI Automation calls and is expected to run on a dedicated UIA thread.
2141+
fn handle_key_press_for_completion_request(
2142+
key_code: u32,
2143+
current_text_input: &Arc<Mutex<Option<TextInputTracker>>>,
2144+
event_tx: &broadcast::Sender<WorkflowEvent>,
2145+
) {
2146+
if let Ok(mut tracker) = current_text_input.lock() {
2147+
if let Some(ref mut text_input) = tracker.as_mut() {
2148+
let is_suggestion_enter = if key_code == 0x0D {
2149+
text_input.handle_enter_key()
2150+
} else {
2151+
false
2152+
};
2153+
let completion_reason = if is_suggestion_enter {
2154+
"suggestion_enter"
2155+
} else {
2156+
"trigger_key"
2157+
};
2158+
2159+
if text_input.should_emit_completion(completion_reason) {
2160+
let input_method = if is_suggestion_enter {
2161+
Some(crate::TextInputMethod::Suggestion)
2162+
} else {
2163+
None
2164+
};
2165+
if let Some(text_event) = text_input.get_completion_event(input_method) {
2166+
let _ = event_tx.send(WorkflowEvent::TextInputCompleted(text_event));
2167+
// Reset the tracker to continue tracking on the same element
2168+
if let Some(element) = &tracker.as_ref().map(|t| t.element.clone()) {
2169+
*tracker = Some(TextInputTracker::new(element.clone()));
2170+
}
2171+
}
2172+
}
2173+
}
2174+
}
22452175
}
22462176
}

‎terminator-workflow-recorder/src/recorder/windows/structs.rs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use crate::events::EventMetadata;
2+
use rdev::Key;
23
use std::time::Instant;
34
use terminator::UIElement;
45
use tracing::{debug, error};
@@ -21,6 +22,9 @@ pub enum UIAInputRequest {
2122
delta: (i32, i32),
2223
position: crate::events::Position,
2324
},
25+
KeyPressForCompletion {
26+
key_code: u32,
27+
},
2428
}
2529

2630
/// Text input tracking state
@@ -290,3 +294,80 @@ impl Default for BrowserTabTracker {
290294
}
291295
}
292296
}
297+
298+
/// Convert a Key to a u32
299+
pub fn key_to_u32(key: &Key) -> u32 {
300+
match key {
301+
Key::KeyA => 0x41,
302+
Key::KeyB => 0x42,
303+
Key::KeyC => 0x43,
304+
Key::KeyD => 0x44,
305+
Key::KeyE => 0x45,
306+
Key::KeyF => 0x46,
307+
Key::KeyG => 0x47,
308+
Key::KeyH => 0x48,
309+
Key::KeyI => 0x49,
310+
Key::KeyJ => 0x4A,
311+
Key::KeyK => 0x4B,
312+
Key::KeyL => 0x4C,
313+
Key::KeyM => 0x4D,
314+
Key::KeyN => 0x4E,
315+
Key::KeyO => 0x4F,
316+
Key::KeyP => 0x50,
317+
Key::KeyQ => 0x51,
318+
Key::KeyR => 0x52,
319+
Key::KeyS => 0x53,
320+
Key::KeyT => 0x54,
321+
Key::KeyU => 0x55,
322+
Key::KeyV => 0x56,
323+
Key::KeyW => 0x57,
324+
Key::KeyX => 0x58,
325+
Key::KeyY => 0x59,
326+
Key::KeyZ => 0x5A,
327+
Key::Num0 => 0x30,
328+
Key::Num1 => 0x31,
329+
Key::Num2 => 0x32,
330+
Key::Num3 => 0x33,
331+
Key::Num4 => 0x34,
332+
Key::Num5 => 0x35,
333+
Key::Num6 => 0x36,
334+
Key::Num7 => 0x37,
335+
Key::Num8 => 0x38,
336+
Key::Num9 => 0x39,
337+
Key::Escape => 0x1B,
338+
Key::Backspace => 0x08,
339+
Key::Tab => 0x09,
340+
Key::Return => 0x0D,
341+
Key::Space => 0x20,
342+
Key::LeftArrow => 0x25,
343+
Key::UpArrow => 0x26,
344+
Key::RightArrow => 0x27,
345+
Key::DownArrow => 0x28,
346+
Key::Delete => 0x2E,
347+
Key::Home => 0x24,
348+
Key::End => 0x23,
349+
Key::PageUp => 0x21,
350+
Key::PageDown => 0x22,
351+
Key::F1 => 0x70,
352+
Key::F2 => 0x71,
353+
Key::F3 => 0x72,
354+
Key::F4 => 0x73,
355+
Key::F5 => 0x74,
356+
Key::F6 => 0x75,
357+
Key::F7 => 0x76,
358+
Key::F8 => 0x77,
359+
Key::F9 => 0x78,
360+
Key::F10 => 0x79,
361+
Key::F11 => 0x7A,
362+
Key::F12 => 0x7B,
363+
Key::ShiftLeft => 0xA0,
364+
Key::ShiftRight => 0xA1,
365+
Key::ControlLeft => 0xA2,
366+
Key::ControlRight => 0xA3,
367+
Key::Alt => 0xA4,
368+
Key::AltGr => 0xA5,
369+
Key::MetaLeft => 0x5B,
370+
Key::MetaRight => 0x5C,
371+
_ => 0,
372+
}
373+
}

0 commit comments

Comments
 (0)