diff --git a/Cargo.toml b/Cargo.toml index 7c169b4..6fe136a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ zeroize = "1.8" winit = "0.30" [target."cfg(windows)".dependencies] -windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging", "Win32_Graphics_Gdi", "Win32_UI_Input_KeyboardAndMouse", "Win32_System_Console", "Win32_Security_Credentials"] } +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging", "Win32_Graphics_Gdi", "Win32_UI_Input_KeyboardAndMouse", "Win32_System_Console", "Win32_Security_Credentials", "Win32_Storage_Xps"] } [build-dependencies] embed-resource = "3.0.12" diff --git a/README.md b/README.md index be452c0..3883966 100644 --- a/README.md +++ b/README.md @@ -170,13 +170,15 @@ MCP client configuration (Claude Desktop, Claude Code, Cursor …): Tools: `list_connections`, `open_connection`, `scan_network`, `port_scan`, `dns_lookup`, `subnet`, `check_certificate`, and for the open window `window_status`, `window_list_tabs`, `window_read_tab`, -`window_type`, `window_switch_tab`, `window_close_tab`, `window_open_tool`, `window_notify`. +`window_screenshot`, `window_type`, `window_switch_tab`, `window_close_tab`, `window_open_tool`, `window_notify`. A fully encrypted connection list is read with the master password in `AC_MASTER`, a list kept in KeePass with `AC_KEEPASS`. -**Control the open window** (Settings » AI agents): `admincraft cli window tabs | read [n] | type | +**Control the open window** (Settings » AI agents): `admincraft cli window tabs | read [n] | screenshot [n] [file.png] | type | switch | close | tool [target]`. The window listens on 127.0.0.1 only, with a token in -the data folder; typing into terminals has to be allowed separately. +the data folder; typing has to be allowed separately. `read` gives a tab's text; where there is +none — a PuTTY, Remote Desktop or VNC window embedded in the tab — `screenshot` gives a picture +(MCP: an image). Typing works in terminal tabs and in embedded PuTTY windows. ## Keyboard diff --git a/ROADMAP.md b/ROADMAP.md index 4364fa3..26ecb01 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -63,7 +63,14 @@ first, run `cargo fmt --check`, `cargo clippy --all-targets --locked -- -D warni requests, tests green after each. This is the precondition for the next item. 9. **Shared CI.** Replace the jobs in `ci.yml` and `release.yml` by calls to the shared gate and release workflows in `storytold/craft-common-actions` (they expect `cargo xtask ci`). -10. **Releases:** add an Intel macOS build; sign the Windows and macOS programs once +10. **Agents in Remote Desktop and VNC windows.** Agents can look at every tab + (`window screenshot`) and type into terminal tabs and embedded PuTTY windows. Remote Desktop + and VNC viewers don't take posted characters: they need real key and mouse input + (`SendInput`, which needs the keyboard focus and so interrupts the person at the computer). + If this is built: its own switch in the settings (it hands the whole remote desktop to the + agent), mouse clicks at a position of the screenshot, and special keys (Ctrl+C, arrows, + Tab) for PuTTY too. `src/embed.rs` and `src/app_control.rs`. +11. **Releases:** add an Intel macOS build; sign the Windows and macOS programs once certificates exist. Done and to be tried against real servers: the Remote Desktop login through the Credential diff --git a/docs/README.de.md b/docs/README.de.md index 573c373..78ff8ed 100644 --- a/docs/README.de.md +++ b/docs/README.de.md @@ -64,7 +64,7 @@ Portabler Verbindungs-Manager für Windows, Linux und macOS – ähnlich mRemote ## Für Skripte und KI-Agenten admincraft cli list | scan | ports | dns | subnet | cert - admincraft cli window tabs | read [n] | type | switch | tool [ziel] + admincraft cli window tabs | read [n] | screenshot [n] [datei.png] | type | switch | tool [ziel] admincraft mcp MCP-Server für KI-Assistenten (stdin/stdout) `cli window …` steuert das **offene Fenster** (Einstellungen » KI-Agenten): Tabs lesen, Verbindungen und diff --git a/docs/help/de.md b/docs/help/de.md index 360b8d0..fbec539 100644 --- a/docs/help/de.md +++ b/docs/help/de.md @@ -324,7 +324,8 @@ admincraft cli window tabs die offenen Tabs admincraft cli window read 2 was Tab 2 zeigt (Terminal, Ergebnisse) admincraft cli window tool ping web01 einen Werkzeug-Tab öffnen admincraft cli window switch 2 einen Tab nach vorn holen -admincraft cli window type 2 uptime in einen Terminal-Tab tippen + Enter +admincraft cli window screenshot 2 t.png Bild von Tab 2 (auch eingebettetes PuTTY / Remotedesktop) +admincraft cli window type 2 uptime in Terminal-Tab oder eingebettetes PuTTY tippen + Enter ``` Der MCP-Server bietet dasselbe als `window_…`-Werkzeuge. In Terminals tippen geht erst, wenn Sie es in den diff --git a/docs/help/en.md b/docs/help/en.md index 4a934b7..a9e557e 100644 --- a/docs/help/en.md +++ b/docs/help/en.md @@ -320,7 +320,8 @@ admincraft cli window tabs the open tabs admincraft cli window read 2 what tab 2 shows (terminal screen, results) admincraft cli window tool ping web01 open a tool tab admincraft cli window switch 2 bring a tab to the front -admincraft cli window type 2 uptime type into a terminal tab and press Enter +admincraft cli window screenshot 2 t.png picture of tab 2 (embedded PuTTY / Remote Desktop too) +admincraft cli window type 2 uptime type into a terminal tab or embedded PuTTY and press Enter ``` The MCP server offers the same as `window_…` tools. Typing into terminals is only possible after you diff --git a/docs/help/fr.md b/docs/help/fr.md index 64ca3c5..a593018 100644 --- a/docs/help/fr.md +++ b/docs/help/fr.md @@ -326,7 +326,8 @@ admincraft cli window tabs les onglets ouverts admincraft cli window read 2 ce que montre l'onglet 2 admincraft cli window tool ping web01 ouvrir un onglet d'outil admincraft cli window switch 2 mettre un onglet au premier plan -admincraft cli window type 2 uptime taper dans un terminal + Entrée +admincraft cli window screenshot 2 t.png image de l'onglet 2 (PuTTY / Bureau à distance intégrés aussi) +admincraft cli window type 2 uptime taper dans un terminal ou un PuTTY intégré + Entrée ``` Le serveur MCP offre la même chose sous forme d'outils `window_…`. Taper dans les terminaux n'est possible diff --git a/docs/help/it.md b/docs/help/it.md index 0c9f0f3..306fec5 100644 --- a/docs/help/it.md +++ b/docs/help/it.md @@ -325,7 +325,8 @@ admincraft cli window tabs le schede aperte admincraft cli window read 2 cosa mostra la scheda 2 admincraft cli window tool ping web01 apre una scheda strumento admincraft cli window switch 2 porta una scheda in primo piano -admincraft cli window type 2 uptime scrive in un terminale + Invio +admincraft cli window screenshot 2 t.png immagine della scheda 2 (anche PuTTY / Desktop remoto incorporati) +admincraft cli window type 2 uptime scrive in un terminale o in un PuTTY incorporato + Invio ``` Il server MCP offre lo stesso come strumenti `window_…`. Scrivere nei terminali è possibile solo dopo averlo diff --git a/docs/help/uk.md b/docs/help/uk.md index cc2d0f4..b43200f 100644 --- a/docs/help/uk.md +++ b/docs/help/uk.md @@ -324,7 +324,8 @@ admincraft cli window tabs відкриті вкладки admincraft cli window read 2 що показує вкладка 2 admincraft cli window tool ping web01 відкрити вкладку інструмента admincraft cli window switch 2 вивести вкладку наперед -admincraft cli window type 2 uptime ввести в термінал + Enter +admincraft cli window screenshot 2 t.png зображення вкладки 2 (також вбудовані PuTTY / віддалений робочий стіл) +admincraft cli window type 2 uptime ввести в термінал або вбудований PuTTY + Enter ``` MCP-сервер пропонує те саме як інструменти `window_…`. Вводити текст у термінали можна лише після дозволу в diff --git a/lang/de.lang b/lang/de.lang index 1bddac4..818efe1 100644 --- a/lang/de.lang +++ b/lang/de.lang @@ -722,7 +722,7 @@ Open download page = Download-Seite öffnen AI agents = KI-Agenten AI agents and scripts on this computer may control this window = KI-Agenten und Skripte auf diesem Computer dürfen dieses Fenster steuern Only programs on this computer that can read the data folder. Passwords are never given out. = Nur Programme auf diesem Computer, die den Datenordner lesen können. Passwörter werden nie herausgegeben. -… and may type into terminal tabs = … und dürfen in Terminal-Tabs tippen +… and may type into terminal tabs and embedded PuTTY windows = … und dürfen in Terminal-Tabs und eingebettete PuTTY-Fenster tippen Careful: an agent can then run commands on your servers. Every typing is shown in the window. = Vorsicht: Ein Agent kann dann Befehle auf Ihren Servern ausführen. Jedes Tippen wird im Fenster angezeigt. Connect an agent: = Agent anbinden: or = oder diff --git a/lang/fr.lang b/lang/fr.lang index 74a79a6..754d50a 100644 --- a/lang/fr.lang +++ b/lang/fr.lang @@ -740,7 +740,7 @@ Open download page = Ouvrir la page de téléchargement AI agents = Agents IA AI agents and scripts on this computer may control this window = Les agents IA et scripts de cet ordinateur peuvent piloter cette fenêtre Only programs on this computer that can read the data folder. Passwords are never given out. = Seuls les programmes de cet ordinateur pouvant lire le dossier de données. Les mots de passe ne sont jamais transmis. -… and may type into terminal tabs = … et peuvent taper dans les onglets terminal +… and may type into terminal tabs and embedded PuTTY windows = … et peuvent taper dans les onglets terminal et les fenêtres PuTTY intégrées Careful: an agent can then run commands on your servers. Every typing is shown in the window. = Attention : un agent peut alors exécuter des commandes sur vos serveurs. Chaque saisie est affichée dans la fenêtre. Connect an agent: = Connecter un agent : or = ou diff --git a/lang/it.lang b/lang/it.lang index b625dfe..f773e7d 100644 --- a/lang/it.lang +++ b/lang/it.lang @@ -740,7 +740,7 @@ Open download page = Apri la pagina di download AI agents = Agenti IA AI agents and scripts on this computer may control this window = Agenti IA e script su questo computer possono controllare questa finestra Only programs on this computer that can read the data folder. Passwords are never given out. = Solo programmi su questo computer che possono leggere la cartella dati. Le password non vengono mai fornite. -… and may type into terminal tabs = … e possono scrivere nelle schede terminale +… and may type into terminal tabs and embedded PuTTY windows = … e possono scrivere nelle schede terminale e nelle finestre PuTTY incorporate Careful: an agent can then run commands on your servers. Every typing is shown in the window. = Attenzione: un agente può allora eseguire comandi sui tuoi server. Ogni digitazione viene mostrata nella finestra. Connect an agent: = Collega un agente: or = oppure diff --git a/lang/uk.lang b/lang/uk.lang index 9873b85..5346a66 100644 --- a/lang/uk.lang +++ b/lang/uk.lang @@ -740,7 +740,7 @@ Open download page = Відкрити сторінку завантаження AI agents = ШІ-агенти AI agents and scripts on this computer may control this window = ШІ-агенти та скрипти на цьому комп'ютері можуть керувати цим вікном Only programs on this computer that can read the data folder. Passwords are never given out. = Лише програми на цьому комп'ютері, що можуть читати папку даних. Паролі ніколи не видаються. -… and may type into terminal tabs = … і можуть вводити текст у вкладки термінала +… and may type into terminal tabs and embedded PuTTY windows = … і можуть вводити текст у вкладки термінала та вбудовані вікна PuTTY Careful: an agent can then run commands on your servers. Every typing is shown in the window. = Обережно: агент тоді може виконувати команди на ваших серверах. Кожне введення показується у вікні. Connect an agent: = Підключити агента: or = або diff --git a/src/agent.rs b/src/agent.rs index 94d1dca..39eb120 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -9,7 +9,7 @@ use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; use std::time::Duration; /// Tool name, description, JSON schema of its arguments. -fn tool_list() -> Vec<(&'static str, &'static str, Value)> { +pub(crate) fn tool_list() -> Vec<(&'static str, &'static str, Value)> { vec![ ( "list_connections", @@ -69,9 +69,14 @@ fn tool_list() -> Vec<(&'static str, &'static str, Value)> { "What a tab of the open window shows as text: a terminal's screen, ping/DNS output, port or network scan results, a certificate check. Default: the active tab.", json!({"type": "object", "properties": {"index": {"type": "integer"}}}), ), + ( + "window_screenshot", + "A picture (PNG) of what a tab of the open window shows, including a PuTTY, Remote Desktop or VNC window embedded in it, where window_read_tab has no text. The tab is brought to the front. Default: the active tab.", + json!({"type": "object", "properties": {"index": {"type": "integer"}}}), + ), ( "window_type", - "Type text into a terminal tab of the open window (SSH, Telnet, PowerShell, shell). Only works when the person allowed it in Settings » AI agents. Set enter=true to press Enter after the text.", + "Type text into a terminal tab of the open window (SSH, Telnet, PowerShell, shell) or into an embedded PuTTY window (look at the result with window_screenshot). Only works when the person allowed it in Settings » AI agents. Set enter=true to press Enter after the text.", json!({"type": "object", "properties": {"index": {"type": "integer"}, "text": {"type": "string"}, "enter": {"type": "boolean"}}, "required": ["text"]}), ), ( @@ -254,7 +259,8 @@ const HELP: &str = "AdminCraft command line (JSON output) admincraft cli cert [port] TLS certificate check admincraft cli window tabs tabs of the open window admincraft cli window read [index] text a tab shows (terminal screen, results) - admincraft cli window type type into a terminal tab (+ Enter), if allowed + admincraft cli window screenshot [index] [file.png] picture of a tab (embedded PuTTY/RDP too) + admincraft cli window type type into a terminal tab or embedded PuTTY (+ Enter), if allowed admincraft cli window switch|close bring a tab to the front / close it admincraft cli window tool [target] open a tool: ping, portscan, dns, netscan, cert, whois, http, snmp … admincraft cli window [json] any command, e.g. status, notify {\"text\":\"hi\"} @@ -278,6 +284,7 @@ pub fn cli(args: &[String]) -> i32 { let (cmd, params) = match a(1).as_str() { "tabs" | "" => ("list_tabs".to_string(), json!({})), "read" => ("read_tab".to_string(), json!({"index": idx(a(2))})), + "screenshot" => ("screenshot".to_string(), json!({"index": idx(a(2)), "path": a(3)})), "type" => ( "type_text".to_string(), json!({"index": idx(a(2)), "text": args.get(3..).map(|x| x.join(" ")).unwrap_or_default(), "enter": true}), @@ -359,11 +366,22 @@ pub fn handle(line: &str, out: &mut impl Write) { "tools/call" => { let name = req.pointer("/params/name").and_then(|v| v.as_str()).unwrap_or(""); let args = req.pointer("/params/arguments").cloned().unwrap_or(json!({})); - let (text, is_error) = match call(name, &args) { + let result = call(name, &args); + // a screenshot goes back as a picture, not as the name of a file + let picture = match (&result, name) { + (Ok(v), "window_screenshot") => v["file"].as_str().and_then(|f| std::fs::read(f).ok()), + _ => None, + }; + let (text, is_error) = match result { Ok(v) => (serde_json::to_string_pretty(&v).unwrap_or_default(), false), Err(e) => (e, true), }; - reply(out, &id, json!({"content": [{"type": "text", "text": text}], "isError": is_error})); + let mut content = vec![json!({"type": "text", "text": text})]; + if let Some(png) = picture { + use base64::Engine as _; + content.insert(0, json!({"type": "image", "data": base64::engine::general_purpose::STANDARD.encode(png), "mimeType": "image/png"})); + } + reply(out, &id, json!({"content": content, "isError": is_error})); } _ => error(out, &id, -32601, "method not found"), } diff --git a/src/app.rs b/src/app.rs index a6e8718..4b99000 100644 --- a/src/app.rs +++ b/src/app.rs @@ -257,6 +257,8 @@ pub struct AdminApp { help_request: Option, /// The channel agents use to control this window (None when switched off). control: Option, + /// A picture of a tab an agent asked for; answered a few frames later, once the tab is drawn. + agent_shot: Option, /// Result of "Check for updates" (None inside while it runs). update_result: Option, releases_url: String, @@ -362,6 +364,7 @@ impl AdminApp { show_keys: false, help_request: None, control: None, + agent_shot: None, update_result: None, releases_url: crate::update::releases_page(), focus_search: false, @@ -499,8 +502,10 @@ impl AdminApp { "embedtest" => { // check of the window embedding with a harmless program (Character Map) let e = Entry { name: tr("Embedding (test)").into(), protocol: Protocol::Vnc, ..Default::default() }; - let prog = if host.is_empty() { "charmap.exe".to_string() } else { host.clone() }; - match std::process::Command::new(&prog).spawn() { + // a program with arguments works too: embedtest:"C:\Tools\putty.exe" -raw -P 5599 127.0.0.1 + let mut words = launch::split_words(&host).into_iter(); + let prog = words.next().unwrap_or_else(|| "charmap.exe".to_string()); + match std::process::Command::new(&prog).args(words).spawn() { Ok(child) => { let pid = child.id(); app.add_tab( @@ -3662,7 +3667,8 @@ impl AdminApp { .on_hover_text(tr("Only programs on this computer that can read the data folder. Passwords are never given out.")) .changed(); ui.add_enabled_ui(self.store.data.settings.agent_control, |ui| { - changed |= ui.checkbox(&mut self.store.data.settings.agent_typing, tr("… and may type into terminal tabs")).changed(); + changed |= + ui.checkbox(&mut self.store.data.settings.agent_typing, tr("… and may type into terminal tabs and embedded PuTTY windows")).changed(); if self.store.data.settings.agent_typing { ui.colored_label( Color32::from_rgb(255, 183, 77), @@ -4294,6 +4300,7 @@ impl eframe::App for AdminApp { } embed::set_active(active_embed); self.tab_screenshot(&ctx, now); + self.agent_shot_step(&ctx); // background jobs: tunnels, monitoring, changes by colleagues, dialogs extra::poll_tunnels(&mut self.tunnels); diff --git a/src/app_control.rs b/src/app_control.rs index a2f73a2..a217763 100644 --- a/src/app_control.rs +++ b/src/app_control.rs @@ -4,7 +4,106 @@ use super::*; use serde_json::{Value, json}; +/// A picture of a tab an agent asked for. The tab is brought to the front first; the picture +/// is taken a few frames later, when the tab (and a window embedded in it) is drawn. +pub(super) struct AgentShot { + reply: std::sync::mpsc::Sender, + tab: usize, + file: PathBuf, + frames: u32, +} + +/// Frames to wait after switching to the tab (an embedded window is placed one frame late). +const SHOT_FRAMES: u32 = 6; +/// Longest text an agent may type into an embedded window at once. +const TYPE_LIMIT: usize = 4000; + +fn save_png(file: &std::path::Path, w: usize, h: usize, rgba: &[u8]) -> Result { + if let Some(dir) = file.parent() { + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + } + let (w32, h32) = (u32::try_from(w).map_err(|e| e.to_string())?, u32::try_from(h).map_err(|e| e.to_string())?); + image::save_buffer(file, rgba, w32, h32, image::ExtendedColorType::Rgba8).map_err(|e| e.to_string())?; + Ok(json!({ "file": file.display().to_string(), "width": w, "height": h })) +} + impl AdminApp { + /// "screenshot {index?, path?}": remembers the request; `agent_shot_step` answers it. + fn start_agent_shot(&mut self, r: crate::control::Request, ctx: &egui::Context) { + if self.agent_shot.is_some() { + let _ = r.reply.send(json!({ "error": "another screenshot is being taken" })); + return; + } + let i = match self.tab_index(&r.args) { + Ok(i) => i, + Err(e) => { + let _ = r.reply.send(json!({ "error": e })); + return; + } + }; + let file = match r.args["path"].as_str().map(str::trim).filter(|p| !p.is_empty()) { + // only ever a picture file: a control command must not be a way to overwrite other files + Some(p) if !p.to_ascii_lowercase().ends_with(".png") => { + let _ = r.reply.send(json!({ "error": "the file name must end in .png" })); + return; + } + Some(p) => PathBuf::from(p), + None => std::env::temp_dir().join("admincraft").join(format!("tab-{}-{i}.png", std::process::id())), + }; + self.active = i; + self.agent_shot = Some(AgentShot { reply: r.reply, tab: i, file, frames: 0 }); + ctx.request_repaint(); + } + + /// Takes the picture an agent asked for once the tab has been drawn. + pub(super) fn agent_shot_step(&mut self, ctx: &egui::Context) { + let Some(shot) = &mut self.agent_shot else { return }; + shot.frames += 1; + if shot.frames < SHOT_FRAMES { + ctx.request_repaint(); + return; + } + let Some(shot) = self.agent_shot.take() else { return }; + if shot.tab >= self.tabs.len() || shot.tab != self.active { + let _ = shot.reply.send(json!({ "error": "the tab was closed or changed meanwhile" })); + return; + } + let tab = self.tab_json(shot.tab); + let r = self.last_content; + let ppp = ctx.pixels_per_point(); + let px = PxRect { + x: (r.min.x * ppp).round() as i32, + y: (r.min.y * ppp).round() as i32, + w: (r.width() * ppp).round() as i32, + h: (r.height() * ppp).round() as i32, + }; + // from the window itself where the system can do that (embedded windows included, and + // nothing of a window lying on top); elsewhere our own drawing + match self.parent_hwnd.and_then(|h| embed::capture_window(h, px)) { + Some((w, h, rgba)) => { + let mut answer = save_png(&shot.file, w, h, &rgba).unwrap_or_else(|e| json!({ "error": e })); + if answer.get("error").is_none() { + answer["tab"] = tab; + } + let _ = shot.reply.send(answer); + } + None => { + let c2 = ctx.clone(); + ctx.request_screenshot(move |img| { + let part = img.region(&r, Some(c2.pixels_per_point())); + let [w, h] = part.size; + let rgba: Vec = part.pixels.iter().flat_map(|p| p.to_array()).collect(); + let mut answer = save_png(&shot.file, w, h, &rgba).unwrap_or_else(|e| json!({ "error": e })); + if answer.get("error").is_none() { + answer["tab"] = tab.clone(); + } + let _ = shot.reply.send(answer); + }); + ctx.request_repaint(); + } + } + } + /// Starts or stops the control channel to match the setting. pub(super) fn sync_control(&mut self, ctx: &egui::Context) { let want = self.store.data.settings.agent_control && self.shot.is_none(); @@ -31,6 +130,11 @@ impl AdminApp { } } for r in reqs { + if r.cmd == "screenshot" { + // answered some frames later, when the tab is on the screen + self.start_agent_shot(r, ctx); + continue; + } let v = self.control_answer(&r.cmd, &r.args, ctx, now).unwrap_or_else(|e| json!({ "error": e })); let _ = r.reply.send(v); } @@ -170,7 +274,25 @@ impl AdminApp { } None => Err("this terminal is not running".to_string()), }, - _ => Err("only terminal tabs take typing".to_string()), + // an embedded PuTTY window takes the characters as messages; Remote Desktop + // and VNC viewers don't, they need a real keyboard + TabKind::Proc(p) => match &p.embed { + Some(em) if em.attached() && em.class() == "PuTTY" => { + if text.chars().count() > TYPE_LIMIT { + Err(format!("at most {TYPE_LIMIT} characters at once")) + } else if em.type_text(&text) && (!enter || em.type_text("\r")) { + Ok(json!({ "typed": text.chars().count(), "enter": enter, "window": "PuTTY" })) + } else { + Err("the PuTTY window did not take the text".to_string()) + } + } + Some(em) if em.attached() => Err(format!( + "typing works in terminal tabs and embedded PuTTY windows, not in this window ({}); look at it with the screenshot command", + em.class() + )), + _ => Err("this window is not embedded in the tab (its own window, or not started yet)".to_string()), + }, + _ => Err("only terminal tabs and embedded PuTTY windows take typing".to_string()), }?; self.note(tf("An agent typed into \"{0}\".", &[&title]), false, now); Ok(typed) diff --git a/src/control.rs b/src/control.rs index 6991dbe..2fd6912 100644 --- a/src/control.rs +++ b/src/control.rs @@ -159,7 +159,8 @@ pub const COMMANDS: &[(&str, &str)] = &[ ("switch_tab", "{index}: bring a tab to the front"), ("close_tab", "{index}: close a tab"), ("read_tab", "{index?}: the text a tab shows (terminal screen, ping/DNS output, scan results)"), - ("type_text", "{index?, text, enter?}: type into a terminal tab (only when allowed in the settings)"), + ("screenshot", "{index?, path?}: a PNG picture of a tab (embedded PuTTY / Remote Desktop / VNC windows too); answers with the file"), + ("type_text", "{index?, text, enter?}: type into a terminal tab or an embedded PuTTY window (only when allowed in the settings)"), ("open_tool", "{tool, target?}: ping, traceroute, portscan, dns, netscan, subnet, cert, whois, http, snmp, help, settings"), ("notify", "{text}: show a short message in the window"), ]; diff --git a/src/embed.rs b/src/embed.rs index 3bc185f..f28240c 100644 --- a/src/embed.rs +++ b/src/embed.rs @@ -20,7 +20,7 @@ mod imp { use std::sync::atomic::{AtomicIsize, Ordering}; use windows_sys::Win32::Foundation::{HWND, LPARAM, LRESULT, POINT, RECT, WPARAM}; use windows_sys::Win32::Graphics::Gdi::*; - use windows_sys::Win32::UI::Input::KeyboardAndMouse::{GetFocus, SetFocus}; + use windows_sys::Win32::UI::Input::KeyboardAndMouse::{GetFocus, SetFocus, VK_RETURN}; use windows_sys::Win32::UI::WindowsAndMessaging::*; /// The embedded window shown in the active tab (0 = none). @@ -116,6 +116,61 @@ mod imp { } } + /// Picture of part of our window taken from the window itself, not from the screen: it + /// shows our window (embedded windows included) even when another window lies on top, and + /// never what that other window shows. + pub fn capture_window(parent: isize, r: PxRect) -> Option<(usize, usize, Vec)> { + use windows_sys::Win32::Storage::Xps::PrintWindow; + /// PW_CLIENTONLY | PW_RENDERFULLCONTENT: the client area as the compositor draws it. + const CLIENT_AS_DRAWN: u32 = 1 | 2; + // SAFETY: every handle is created here and released before returning; the pixel buffer + // is as large as the bitmap GetDIBits fills (width * height * 4 bytes). + unsafe { + let mut client: RECT = std::mem::zeroed(); + if GetClientRect(parent as HWND, &mut client) == 0 { + return None; + } + let (cw, ch) = (client.right - client.left, client.bottom - client.top); + // the wanted part, kept inside the window + let x = r.x.clamp(0, cw); + let y = r.y.clamp(0, ch); + let w = r.w.min(cw - x); + let h = r.h.min(ch - y); + if cw <= 0 || ch <= 0 || w <= 0 || h <= 0 || cw > 16_384 || ch > 16_384 { + return None; + } + let screen = GetDC(std::ptr::null_mut()); + let mem = CreateCompatibleDC(screen); + let bmp = CreateCompatibleBitmap(screen, cw, ch); + let old = SelectObject(mem, bmp as HGDIOBJ); + let drawn = PrintWindow(parent as HWND, mem, CLIENT_AS_DRAWN); + SelectObject(mem, old); + let mut info: BITMAPINFO = std::mem::zeroed(); + info.bmiHeader.biSize = std::mem::size_of::() as u32; + info.bmiHeader.biWidth = cw; + info.bmiHeader.biHeight = -ch; // top-down + info.bmiHeader.biPlanes = 1; + info.bmiHeader.biBitCount = 32; + info.bmiHeader.biCompression = BI_RGB; + let mut all = vec![0u8; (cw as usize) * (ch as usize) * 4]; + let lines = if drawn != 0 { GetDIBits(mem, bmp, 0, ch as u32, all.as_mut_ptr() as *mut _, &mut info, DIB_RGB_COLORS) } else { 0 }; + DeleteObject(bmp as HGDIOBJ); + DeleteDC(mem); + ReleaseDC(std::ptr::null_mut(), screen); + if lines == 0 { + return None; + } + let (x, y, w, h, cw) = (x as usize, y as usize, w as usize, h as usize, cw as usize); + let mut out = Vec::with_capacity(w * h * 4); + for row in all.chunks_exact(cw * 4).skip(y).take(h) { + for px in row[x * 4..(x + w) * 4].as_chunks::<4>().0 { + out.extend_from_slice(&[px[2], px[1], px[0], 255]); // BGRA -> RGBA + } + } + Some((w, h, out)) + } + } + pub struct Embedded { pub pid: u32, class_hint: &'static [&'static str], @@ -295,6 +350,36 @@ mod imp { } } + /// The window class of the embedded window ("PuTTY", "TscShellContainerClass" …). + pub fn class(&self) -> String { + if self.attached() { class_name(self.hwnd) } else { String::new() } + } + + /// Types into the embedded window by posting the characters to it. The keyboard focus + /// stays where it is, so the person at the computer is not interrupted. Works for + /// windows that take characters themselves (PuTTY); `false` when a message was refused. + pub fn type_text(&self, text: &str) -> bool { + if !self.attached() { + return false; + } + /// Scan code of the Return key, as a real keyboard reports it. + const RETURN_SCAN: isize = 0x1C << 16; + let post = |msg: u32, wp: WPARAM, lp: LPARAM| { + // SAFETY: a plain message with two integers to a window handle; a window that is + // gone makes the call fail, which is reported. + unsafe { PostMessageW(self.hwnd, msg, wp, lp) != 0 } + }; + text.replace("\r\n", "\r").replace('\n', "\r").encode_utf16().all(|unit| { + if unit == 13 { + // Enter as a key press, not as a character: with PuTTY's own line editing + // (raw, serial, some Telnet servers) a posted carriage return is only shown as ^M + post(WM_KEYDOWN, VK_RETURN as WPARAM, 1 | RETURN_SCAN) && post(WM_KEYUP, VK_RETURN as WPARAM, 1 | RETURN_SCAN | (0b11 << 30)) + } else { + post(WM_CHAR, unit as WPARAM, 0) + } + }) + } + /// Gives the window back as a normal window of its own. pub fn detach(&mut self) { if !self.attached() { @@ -332,6 +417,9 @@ mod imp { pub fn capture(_parent: isize, _r: PxRect) -> Option<(usize, usize, Vec)> { None } + pub fn capture_window(_parent: isize, _r: PxRect) -> Option<(usize, usize, Vec)> { + None + } impl Embedded { pub fn new(_pid: u32, _class_hint: &'static [&'static str]) -> Self { @@ -351,12 +439,18 @@ mod imp { } pub fn place(&mut self, _rect: Option) {} pub fn focus(&self) {} + pub fn class(&self) -> String { + String::new() + } + pub fn type_text(&self, _text: &str) -> bool { + false + } pub fn detach(&mut self) {} pub fn close(&mut self) {} } } -pub use imp::{Embedded, capture, install_focus_hook, set_active}; +pub use imp::{Embedded, capture, capture_window, install_focus_hook, set_active}; /// Can this system put other programs' windows into tabs? pub const SUPPORTED: bool = cfg!(windows); diff --git a/src/tests.rs b/src/tests.rs index c4bfab7..97b1230 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -784,3 +784,19 @@ fn putty_password_goes_through_a_file() { assert!(!kept_path.exists()); let _ = std::fs::remove_dir_all(dir); } + +#[test] +fn agents_can_look_at_and_type_into_embedded_windows() { + // the control channel and the MCP server both offer the picture of a tab + assert!(crate::control::COMMANDS.iter().any(|(cmd, _)| *cmd == "screenshot")); + let tools = crate::agent::tool_list(); + assert!(tools.iter().any(|(name, _, _)| *name == "window_screenshot")); + let typing = tools.iter().find(|(name, _, _)| *name == "window_type").expect("window_type"); + assert!(typing.1.contains("PuTTY"), "the description says where typing works"); + // a window that is not embedded takes no typing and has no class (nothing is sent anywhere) + let not_embedded = crate::embed::Embedded::new(0, &[]); + assert!(!not_embedded.type_text("x")); + assert_eq!(not_embedded.class(), ""); + // outside our own window there is nothing to take a picture of + assert!(crate::embed::capture_window(0, crate::embed::PxRect { x: 0, y: 0, w: 10, h: 10 }).is_none()); +}