diff --git a/README.md b/README.md index 83acd67..45263a0 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Trove 官方插件库。发现喜欢的 Curio,装进你的工作台,让常 ### Harmony Connector -让设备,更近一步。连接 Mac 与鸿蒙手机,整夹收发照片与文档,随时掌握传输速度;管理应用、保存截图,让日常操作在桌面从容完成。 +让设备,更近一步。连接 Mac 与鸿蒙手机,整夹收发照片与文档,随时掌握传输速度;管理应用、查看手机画面,用鼠标和键盘完成日常操作。 ## 按你的习惯组合 diff --git a/curios/harmony-connector/build.mjs b/curios/harmony-connector/build.mjs index c8594d4..1ba4dcb 100644 --- a/curios/harmony-connector/build.mjs +++ b/curios/harmony-connector/build.mjs @@ -28,6 +28,7 @@ const sources = [ archive: "libmtp-1.1.23.tar.gz", sha256: "74a2b6e8cb4a0304e95b995496ea3ac644c29371649b892b856e22f12a0bdeed", options: ["--disable-mtpz"], + patch: "libmtp-close-timeout.patch", }, ]; function run(command, args, cwd = plugin, env = {}) { @@ -56,11 +57,18 @@ for (const source of sources) { throw new Error(`Source checksum mismatch: ${source.archive}`); const directory = path.join(build, source.name); const stamp = path.join(directory, ".harmony-built"); - const identity = `${process.arch}:macos13:${source.sha256}`; + const patchFile = + source.patch && path.join(root, "vendor/harmony-connector", source.patch); + const patchBytes = patchFile ? await readFile(patchFile) : null; + const patchHash = patchBytes + ? createHash("sha256").update(patchBytes).digest("hex") + : ""; + const identity = `${process.arch}:macos13:${source.sha256}${patchHash ? ":" + patchHash : ""}`; const previous = await readFile(stamp, "utf8").catch(() => ""); if (previous !== identity) { await rm(directory, { recursive: true, force: true }); run("tar", ["-xf", archive, "-C", build]); + if (patchFile) run("patch", ["--batch", "-p1", "-i", patchFile], directory); run( "./configure", [ @@ -82,6 +90,8 @@ for (const source of sources) { path.join(plugin, "legal", `${source.name}-COPYING`), ); await copyFile(archive, path.join(plugin, "legal/sources", source.archive)); + if (patchFile) + await copyFile(patchFile, path.join(plugin, "legal/sources", source.patch)); } if (process.argv.includes("--dependencies-only")) process.exit(0); run( diff --git a/curios/harmony-connector/curio.json b/curios/harmony-connector/curio.json index 578907f..2c67dbb 100644 --- a/curios/harmony-connector/curio.json +++ b/curios/harmony-connector/curio.json @@ -2,8 +2,8 @@ "schemaVersion": 1, "id": "dev.trove.harmony-connector", "name": "Harmony Connector", - "description": "连接鸿蒙设备,收发完整文件夹、管理应用,轻松保存截图和日志。", - "version": "0.2.0", + "description": "连接鸿蒙设备,收发完整文件夹、管理应用,查看手机画面并通过键鼠操作。", + "version": "0.3.0", "engine": { "trove": ">=0.1.0 <1.0.0" }, "icon": "assets/icon.svg", "ui": { diff --git a/curios/harmony-connector/legal/THIRD-PARTY.md b/curios/harmony-connector/legal/THIRD-PARTY.md index c8c2921..093ec6f 100644 --- a/curios/harmony-connector/legal/THIRD-PARTY.md +++ b/curios/harmony-connector/legal/THIRD-PARTY.md @@ -1,7 +1,8 @@ # Harmony Connector — Third-party software Harmony Connector dynamically links the following libraries. This distribution -includes their original license texts and complete, unmodified source archives. +includes their original license texts, complete upstream source archives and +the patches applied to those sources. | Library | Version | Source | License text in this package | | --- | --- | --- | --- | @@ -24,7 +25,9 @@ The source repository contains the same script at `curios/harmony-connector/build.mjs` and archives under `vendor/harmony-connector/`. Requires macOS, Xcode Command Line Tools, make and pkg-config. Libraries are compiled with deployment target macOS 13.0, shared linkage, no static library, -and libmtp's MTPZ support disabled. No changes are made to the upstream sources. +and libmtp's MTPZ support disabled. `sources/libmtp-close-timeout.patch` bounds +CloseSession to two seconds without reducing file-transfer timeouts. The build +cache includes the patch hash; the existing release/reset/close sequence is retained. Mach-O install names are adjusted after compilation; libraries are ad-hoc signed. From the source repository, run: diff --git a/curios/harmony-connector/native/Cargo.lock b/curios/harmony-connector/native/Cargo.lock index aa488ac..ac8c5c8 100644 --- a/curios/harmony-connector/native/Cargo.lock +++ b/curios/harmony-connector/native/Cargo.lock @@ -23,6 +23,18 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "2.13.2" @@ -44,6 +56,18 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "cc" version = "1.4.5" @@ -180,10 +204,12 @@ dependencies = [ [[package]] name = "harmony-connector" -version = "0.2.0" +version = "0.3.0" dependencies = [ "anyhow", + "base64", "cc", + "image", "libc", "png", "serde", @@ -199,6 +225,21 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -259,6 +300,25 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -287,6 +347,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "quote" version = "1.0.47" @@ -486,3 +552,18 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/curios/harmony-connector/native/Cargo.toml b/curios/harmony-connector/native/Cargo.toml index b257187..a121b8e 100644 --- a/curios/harmony-connector/native/Cargo.toml +++ b/curios/harmony-connector/native/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "harmony-connector" -version = "0.2.0" +version = "0.3.0" edition = "2021" publish = false @@ -13,6 +13,8 @@ libc = "0.2" sha2 = "0.10" zip = { version = "4.6", default-features = false, features = ["deflate"] } png = "0.18" +image = { version = "0.25", default-features = false, features = ["jpeg", "png"] } +base64 = "0.22" [build-dependencies] cc = "1" diff --git a/curios/harmony-connector/native/src/diagnostics.rs b/curios/harmony-connector/native/src/diagnostics.rs index 4582152..9b188ca 100644 --- a/curios/harmony-connector/native/src/diagnostics.rs +++ b/curios/harmony-connector/native/src/diagnostics.rs @@ -1,11 +1,19 @@ use crate::mtp::{self, DeviceIo, NativeDevice, ROOT}; use serde_json::json; -use std::time::Instant; +use std::time::{Duration, Instant}; // Explicitly invoked, read-only session exercise. Never run during a user's plugin session. pub fn reconnect() -> anyhow::Result<()> { for cycle in 1..=3 { - let devices = mtp::scan()?; + // A USB reset may temporarily remove the device while macOS re-enumerates it. + let discovery = Instant::now(); + let devices = loop { + let devices = mtp::scan()?; + if !devices.is_empty() || discovery.elapsed() >= Duration::from_secs(10) { + break devices; + } + std::thread::sleep(Duration::from_millis(250)); + }; anyhow::ensure!( devices.len() == 1, "重连验收需要恰好一台可用的 USB 文件传输设备,当前发现 {} 台。", @@ -13,7 +21,7 @@ pub fn reconnect() -> anyhow::Result<()> { ); println!( "{}", - json!({"cycle":cycle,"stage":"opening","device":devices[0].id}) + json!({"cycle":cycle,"stage":"opening","device":devices[0].id,"discoverySeconds":discovery.elapsed().as_secs_f64()}) ); let opened = Instant::now(); let mut device = NativeDevice::open(&devices[0])?; @@ -39,3 +47,67 @@ pub fn reconnect() -> anyhow::Result<()> { println!("{}", json!({"passed":true,"cycles":3,"writes":false})); Ok(()) } + +// Uses the same management entry points as the UI. It never enables control or sends input. +pub fn screen(args: &[String]) -> anyhow::Result<()> { + anyhow::ensure!( + !args.is_empty() && args.len() <= 3, + "用法:--check-screen /absolute/path/to/hdc [设备 ID] [保存目录]" + ); + let manager = crate::hdc::Manager::new(); + let tool = manager.call("configure", json!({"path":args[0]}))?; + println!("{}", json!({"stage":"tool","version":tool["version"]})); + let devices = manager.call("devices", json!({}))?; + let connected: Vec<_> = devices["devices"] + .as_array() + .unwrap() + .iter() + .filter(|d| d["state"] == "Connected") + .collect(); + let target = if let Some(target) = args.get(1) { + target.clone() + } else { + anyhow::ensure!( + connected.len() == 1, + "需要恰好一台已授权 HDC 设备,或明确指定设备 ID。当前已连接 {} 台。", + connected.len() + ); + connected[0]["id"].as_str().unwrap().to_owned() + }; + let opened = manager.call("screenStart", json!({"target":target}))?; + let id = json!({"session":opened["session"]}); + println!( + "{}", + json!({"stage":"opened","inputCapabilities":opened["inputCapabilities"],"control":false}) + ); + let started = Instant::now(); + let result = (|| -> anyhow::Result<()> { + for _ in 0..3 { + let frame = manager.call("screenFrame", id.clone())?; + println!( + "{}", + json!({"stage":"frame","sequence":frame["sequence"],"width":frame["width"],"height":frame["height"],"captureMs":frame["captureMs"]}) + ); + } + if let Some(directory) = args.get(2) { + let saved = manager.call( + "screenSave", + json!({"session":opened["session"],"directory":directory}), + )?; + println!( + "{}", + json!({"stage":"saved","path":saved["path"],"bytes":saved["bytes"]}) + ); + } + println!( + "{}", + json!({"passed":true,"frames":3,"seconds":started.elapsed().as_secs_f64(),"inputSent":false}) + ); + Ok(()) + })(); + let stopped = manager.call("screenStop", id); + manager.stop(); + result?; + stopped?; + Ok(()) +} diff --git a/curios/harmony-connector/native/src/hdc.rs b/curios/harmony-connector/native/src/hdc.rs index d05f171..afe97f6 100644 --- a/curios/harmony-connector/native/src/hdc.rs +++ b/curios/harmony-connector/native/src/hdc.rs @@ -1,4 +1,6 @@ mod files; +mod screen; +mod screen_input; use crate::{ command, hdc_data, model::{Fault, Result}, @@ -24,6 +26,7 @@ impl Drop for Guard { } } pub struct Manager { + screen: screen::Screen, path: Mutex>, settings: Option, busy: Arc, @@ -204,6 +207,7 @@ impl Manager { .filter(|p| executable(p)) .or_else(discover); Arc::new(Self { + screen: screen::Screen::default(), path: Mutex::new(path), settings, busy: Arc::new(AtomicBool::new(false)), @@ -246,6 +250,7 @@ impl Manager { pub fn stop(&self) { self.stopped.store(true, Ordering::Release); self.cancel.store(true, Ordering::Release); + let _ = self.screen.close(None); } pub fn call(self: &Arc, method: &str, value: Value) -> Result { if method == "status" { @@ -266,8 +271,24 @@ impl Manager { } return Ok(json!({"requested":job["state"] == "running"})); } + if method == "screenStop" { + let r: screen::Id = parse(value)?; + return self.screen.close(Some(r.session)); + } + if method == "screenControl" { + return self.screen.control(parse(value)?); + } + if method == "screenSave" { + return self.screen.save(parse(value)?); + } let guard = self.enter()?; match method { + "screenStart" => { + let r: Target = parse(value)?; + self.screen.start(self.tool()?, r.target) + } + "screenFrame" => self.screen.frame(parse(value)?), + "screenInput" => self.screen.input(parse(value)?), "tool" => { let path = self.path.lock().unwrap().clone(); let Some(path) = path else { diff --git a/curios/harmony-connector/native/src/hdc/screen.rs b/curios/harmony-connector/native/src/hdc/screen.rs new file mode 100644 index 0000000..353b9d9 --- /dev/null +++ b/curios/harmony-connector/native/src/hdc/screen.rs @@ -0,0 +1,334 @@ +use super::screen_input::{self, Input}; +use super::*; +use base64::{engine::general_purpose::STANDARD, Engine}; +use image::{codecs::jpeg::JpegEncoder, ImageFormat, ImageReader}; +use std::{ + collections::HashSet, + io::{BufReader, Write}, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +const LEASE: Duration = Duration::from_secs(15); +struct Frame { + jpeg: Vec, + width: u32, + height: u32, + sequence: u64, + geometry: u64, + captured: Instant, + capture_ms: u128, +} +struct Session { + id: u64, + target: String, + path: PathBuf, + cancel: Arc, + control: bool, + supported: HashSet, + frame: Option, + touched: Instant, +} +#[derive(Default)] +pub(super) struct Screen { + current: Mutex>, + next: AtomicU64, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Id { + pub session: u64, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Control { + pub session: u64, + pub enabled: bool, + pub geometry: Option, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Event { + pub session: u64, + pub geometry: u64, + pub sequence: u64, + pub input: Input, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Save { + pub session: u64, + pub directory: PathBuf, +} +fn missing() -> Fault { + Fault::new("screen.closed", "投屏已停止,请重新开始。") +} +impl Screen { + pub fn close(&self, id: Option) -> Result { + let mut current = self.current.lock().unwrap(); + if let Some(id) = id { + if current.as_ref().is_some_and(|s| s.id != id) { + return Err(missing()); + } + } + if let Some(session) = current.take() { + session.cancel.store(true, Ordering::Release); + } + Ok(json!({"stopped":true})) + } + pub fn start(&self, tool: Tool, target: String) -> Result { + hdc_data::target(&target)?; + { + let current = self.current.lock().unwrap(); + if current + .as_ref() + .is_some_and(|s| s.touched.elapsed() < LEASE) + { + return Err(Fault::new("screen.active", "请先停止当前投屏。")); + } + } + tool.connected(&target)?; + let help = tool.shell(&target, "uitest help")?; + if !help.contains("screenCap") { + return Err(Fault::new( + "screen.unavailable", + "设备没有开放屏幕采集,请检查调试工具与设备授权。", + )); + } + let input_help = tool + .shell(&target, "uitest uiInput help") + .unwrap_or_default(); + let supported = screen_input::capabilities(&input_help); + let id = self.next.fetch_add(1, Ordering::Relaxed) + 1; + let mut capabilities: Vec<_> = supported.iter().cloned().collect(); + capabilities.sort(); + self.close(None)?; + let mut current = self.current.lock().unwrap(); + if tool.cancel.load(Ordering::Acquire) { + return Err(missing()); + } + *current = Some(Session { + id, + target: target.clone(), + path: tool.path, + cancel: Arc::new(AtomicBool::new(false)), + control: false, + supported, + frame: None, + touched: Instant::now(), + }); + Ok( + json!({"session":id,"target":target,"inputCapabilities":capabilities,"control":false,"mode":"uitest-frames"}), + ) + } + fn access(&self, id: u64, f: impl FnOnce(&mut Session) -> Result) -> Result { + let mut current = self.current.lock().unwrap(); + let s = current + .as_mut() + .filter(|s| s.id == id && !s.cancel.load(Ordering::Acquire)) + .ok_or_else(missing)?; + if s.touched.elapsed() > LEASE { + s.control = false; + s.cancel.store(true, Ordering::Release); + return Err(Fault::new( + "screen.expired", + "投屏连接已暂停过久,请重新开始。", + )); + } + s.touched = Instant::now(); + f(s) + } + fn context(&self, id: u64) -> Result<(Tool, String)> { + self.access(id, |s| { + Ok(( + Tool { + path: s.path.clone(), + cancel: s.cancel.clone(), + }, + s.target.clone(), + )) + }) + } + pub fn control(&self, request: Control) -> Result { + self.access(request.session, |s| { + if request.enabled && request.geometry != s.frame.as_ref().map(|f| f.geometry) { + return Err(Fault::new( + "screen.stale_frame", + "画面方向已变化,请等待新画面后启用控制。", + )); + } + if request.enabled && (s.supported.is_empty() || s.frame.is_none()) { + return Err(Fault::new( + "screen.not_ready", + "先等待手机画面,并确认设备开放输入能力。", + )); + } + s.control = request.enabled; + Ok(json!({"control":s.control})) + }) + } + pub fn frame(&self, request: Id) -> Result { + let (tool, target) = self.context(request.session)?; + let started = Instant::now(); + let temporary = tempfile::NamedTempFile::new()?; + let result = (|| { + capture(&tool, &target, temporary.path())?; + let (jpeg, width, height) = encode_frame(temporary.path())?; + self.access(request.session, |s| { + let changed = s.frame.as_ref().is_some_and(|f| (f.width,f.height) != (width,height)); + let sequence = s.frame.as_ref().map_or(1, |f| f.sequence + 1); + let geometry = s.frame.as_ref().map_or(1, |f| f.geometry + u64::from(changed)); + if changed { s.control = false; } + let capture_ms = started.elapsed().as_millis(); + let value = json!({"session":s.id,"sequence":sequence,"geometry":geometry,"width":width,"height":height,"dataUrl":format!("data:image/jpeg;base64,{}",STANDARD.encode(&jpeg)),"captureMs":capture_ms,"control":s.control,"rotated":changed}); + s.frame = Some(Frame { jpeg,width,height,sequence,geometry,captured:Instant::now(),capture_ms }); + Ok(value) + }) + })(); + if result.is_err() { + let _ = self.close(Some(request.session)); + } + result + } + pub fn input(&self, request: Event) -> Result { + let (tool, target, cmd) = self.access(request.session, |s| { + if !s.control { + return Err(Fault::new("screen.read_only", "请先启用控制。")); + } + let frame = s.frame.as_ref().ok_or_else(missing)?; + if frame.geometry != request.geometry + || request.sequence == 0 + || request.sequence > frame.sequence + || frame.sequence - request.sequence > 1 + || frame.captured.elapsed() > Duration::from_secs(3) + || frame.capture_ms > 5000 + { + return Err(Fault::new( + "screen.stale_frame", + "画面已变化或响应过慢,请等待新画面后再操作。", + )); + } + let cmd = request + .input + .command(frame.width, frame.height, &s.supported)?; + Ok(( + Tool { + path: s.path.clone(), + cancel: s.cancel.clone(), + }, + s.target.clone(), + cmd, + )) + })?; + let result = tool.shell(&target, &cmd); + if result.is_err() { + self.access(request.session, |s| { + s.control = false; + Ok(()) + }) + .ok(); + } + result.map(|_| json!({"executed":true})) + } + pub fn save(&self, request: Save) -> Result { + let jpeg = self.access(request.session, |s| { + let frame = s + .frame + .as_ref() + .ok_or_else(|| Fault::new("screen.not_ready", "请等待手机画面后再保存。"))?; + Ok(frame.jpeg.clone()) + })?; + if !request.directory.is_absolute() || !request.directory.is_dir() { + return Err(Fault::new( + "file.invalid_directory", + "请选择保存画面的目录。", + )); + } + let directory = request.directory.canonicalize()?; + let mut temporary = tempfile::NamedTempFile::new_in(&directory)?; + temporary.write_all(&jpeg)?; + temporary.as_file().sync_all()?; + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let destination = directory.join(format!("harmony-screen-{stamp}.jpg")); + temporary + .persist_noclobber(&destination) + .map_err(|e| Fault::new("file.save_failed", e.error.to_string()))?; + Ok(json!({"path":destination,"bytes":jpeg.len()})) + } +} +fn capture(tool: &Tool, target: &str, destination: &Path) -> Result<()> { + let nonce = tempfile::Builder::new() + .prefix("harmony-screen-") + .tempdir()?; + let remote = format!( + "/data/local/tmp/{}.png", + nonce.path().file_name().unwrap().to_string_lossy() + ); + let result = (|| { + tool.run( + &[ + "-t", + target, + "shell", + &format!("uitest screenCap -p {remote}"), + ], + 6, + )?; + tool.run( + &[ + "-t", + target, + "file", + "recv", + &remote, + destination.to_str().unwrap(), + ], + 6, + )?; + if destination.metadata()?.len() > 32 * 1024 * 1024 { + return Err(Fault::new( + "screen.frame_too_large", + "设备画面超过支持范围。", + )); + } + Ok(()) + })(); + let cleanup = Tool { + path: tool.path.clone(), + cancel: Arc::new(AtomicBool::new(false)), + }; + let _ = cleanup.run(&["-t", target, "shell", &format!("rm -f {remote}")], 2); + result +} +fn encode_frame(path: &Path) -> Result<(Vec, u32, u32)> { + let mut reader = + ImageReader::with_format(BufReader::new(fs::File::open(path)?), ImageFormat::Png); + let mut limits = image::Limits::default(); + limits.max_image_width = Some(8192); + limits.max_image_height = Some(8192); + limits.max_alloc = Some(64 * 1024 * 1024); + reader.limits(limits); + let image = reader + .decode() + .map_err(|_| Fault::new("screen.invalid_frame", "手机未返回有效画面,投屏已停止。"))?; + let (width, height) = (image.width(), image.height()); + if width == 0 || height == 0 { + return Err(Fault::new("screen.invalid_frame", "画面尺寸无效。")); + } + let scaled = if width > 1600 || height > 1600 { + image.thumbnail(1600, 1600) + } else { + image + } + .to_rgb8(); + let mut jpeg = Vec::new(); + JpegEncoder::new_with_quality(&mut jpeg, 82) + .encode_image(&scaled) + .map_err(|_| Fault::new("screen.encode_failed", "无法处理手机画面。"))?; + if jpeg.len() > 2 * 1024 * 1024 { + return Err(Fault::new("screen.frame_too_large", "画面数据过大。")); + } + Ok((jpeg, width, height)) +} diff --git a/curios/harmony-connector/native/src/hdc/screen_input.rs b/curios/harmony-connector/native/src/hdc/screen_input.rs new file mode 100644 index 0000000..36aa7f9 --- /dev/null +++ b/curios/harmony-connector/native/src/hdc/screen_input.rs @@ -0,0 +1,161 @@ +use crate::model::{Fault, Result}; +use serde::Deserialize; +use std::collections::HashSet; + +#[derive(Deserialize)] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub(super) enum Input { + Click { + x: u32, + y: u32, + }, + DoubleClick { + x: u32, + y: u32, + }, + LongClick { + x: u32, + y: u32, + }, + Swipe { + x: u32, + y: u32, + to_x: u32, + to_y: u32, + speed: u32, + }, + Drag { + x: u32, + y: u32, + to_x: u32, + to_y: u32, + speed: u32, + }, + Text { + text: String, + }, + Key { + key: Key, + }, +} +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) enum Key { + Back, + Home, + Enter, + Backspace, + Delete, + Tab, + Up, + Down, + Left, + Right, + Escape, + VolumeUp, + VolumeDown, +} +impl Key { + fn code(&self) -> &'static str { + match self { + Self::Back => "Back", + Self::Home => "Home", + Self::Enter => "2054", + Self::Backspace => "2055", + Self::Delete => "2071", + Self::Tab => "2049", + Self::Up => "2012", + Self::Down => "2013", + Self::Left => "2014", + Self::Right => "2015", + Self::Escape => "2070", + Self::VolumeUp => "16", + Self::VolumeDown => "17", + } + } +} +pub(super) fn capabilities(help: &str) -> HashSet { + let words: HashSet<&str> = help.split(|c: char| !c.is_ascii_alphanumeric()).collect(); + [ + "click", + "doubleClick", + "longClick", + "swipe", + "drag", + "text", + "keyEvent", + ] + .into_iter() + .filter(|v| words.contains(v)) + .map(String::from) + .collect() +} +impl Input { + pub fn command(&self, width: u32, height: u32, supported: &HashSet) -> Result { + let point = |x, y| { + if x < width && y < height { + Ok(format!("{x} {y}")) + } else { + Err(Fault::new("screen.outside", "操作位置超出当前屏幕。")) + } + }; + let (name, args) = match self { + Self::Click { x, y } => ("click", point(*x, *y)?), + Self::DoubleClick { x, y } => ("doubleClick", point(*x, *y)?), + Self::LongClick { x, y } => ("longClick", point(*x, *y)?), + Self::Swipe { + x, + y, + to_x, + to_y, + speed, + } + | Self::Drag { + x, + y, + to_x, + to_y, + speed, + } => { + if !(200..=40_000).contains(speed) { + return Err(Fault::new("screen.invalid_speed", "滑动速度超出支持范围。")); + } + ( + if matches!(self, Self::Drag { .. }) { + "drag" + } else { + "swipe" + }, + format!("{} {} {speed}", point(*x, *y)?, point(*to_x, *to_y)?), + ) + } + Self::Text { text } => { + if text.is_empty() + || text.len() > 4096 + || text + .chars() + .any(|c| c.is_control() && !matches!(c, '\n' | '\t')) + { + return Err(Fault::new( + "screen.invalid_text", + "请输入不超过 4096 字节且不含控制字符的文本。", + )); + } + ("text", format!("'{}'", text.replace('\'', "'\"'\"'"))) + } + Self::Key { key } => ("keyEvent", key.code().into()), + }; + if !supported.contains(name) { + return Err(Fault::new( + "screen.input_unavailable", + "设备未开放此输入操作。", + )); + } + Ok(format!("uitest uiInput {name} {args}")) + } +} diff --git a/curios/harmony-connector/native/src/hdc/screen_tests.rs b/curios/harmony-connector/native/src/hdc/screen_tests.rs new file mode 100644 index 0000000..fa0006a --- /dev/null +++ b/curios/harmony-connector/native/src/hdc/screen_tests.rs @@ -0,0 +1,254 @@ +use super::*; +use base64::{engine::general_purpose::STANDARD, Engine}; + +fn phone() -> (tempfile::TempDir, Arc) { + let (directory, manager) = fixture(false); + fs::write(directory.path().join("hdc"), r#"#!/bin/sh +printf '%s\n' "$*" >> "$0.calls" +case "$*" in + 'list targets -v') printf 'phone USB Connected device\n' ;; + '-t phone shell uitest help') printf 'screenCap uiInput\n' ;; + '-t phone shell uitest uiInput help') printf 'click doubleClick longClick swipe drag text keyEvent\n' ;; + '-t phone shell uitest screenCap -p '*) + if [ -f "$0.slow" ]; then touch "$0.capturing"; sleep 20; fi + printf 'Screen capture saved\n' ;; + '-t phone file recv '*) cp "$0.png" "$6" ;; + '-t phone shell rm -f /data/local/tmp/harmony-screen-'*) printf 'cleaned\n' ;; + '-t phone shell uitest uiInput '*) + if [ -f "$0.fail-input" ]; then printf '[Fail] input not permitted\n'; else printf 'completed\n'; fi ;; + *) printf '[Fail] unexpected invocation\n' ;; +esac +"#).unwrap(); + frame_file(&directory, 600, 1200, [30, 80, 120]); + (directory, manager) +} +fn frame_file(directory: &tempfile::TempDir, width: u32, height: u32, rgb: [u8; 3]) { + image::RgbImage::from_pixel(width, height, image::Rgb(rgb)) + .save(directory.path().join("hdc.png")) + .unwrap(); +} +fn start(manager: &Arc) -> Value { + let opened = manager + .call("screenStart", json!({"target":"phone"})) + .unwrap(); + assert_eq!(opened["control"], false); + json!({"session":opened["session"]}) +} +fn input(id: &Value, frame: &Value, event: Value) -> Value { + json!({"session":id["session"],"sequence":frame["sequence"],"geometry":frame["geometry"],"input":event}) +} +fn enable(manager: &Arc, id: &Value, frame: &Value) { + manager + .call( + "screenControl", + json!({"session":id["session"],"enabled":true,"geometry":frame["geometry"]}), + ) + .unwrap(); +} + +#[test] +fn shows_real_decodable_frames_and_saves_latest_without_overwriting() { + let (directory, manager) = phone(); + let id = start(&manager); + let first = manager.call("screenFrame", id.clone()).unwrap(); + assert_eq!( + (first["width"].as_u64(), first["height"].as_u64()), + (Some(600), Some(1200)) + ); + let decode = |v: &Value| { + STANDARD + .decode( + v["dataUrl"] + .as_str() + .unwrap() + .strip_prefix("data:image/jpeg;base64,") + .unwrap(), + ) + .unwrap() + }; + let jpeg = decode(&first); + assert_eq!(image::load_from_memory(&jpeg).unwrap().height(), 1200); + frame_file(&directory, 600, 1200, [220, 60, 30]); + let next = manager.call("screenFrame", id.clone()).unwrap(); + assert!(next["sequence"].as_u64() > first["sequence"].as_u64()); + assert_ne!(decode(&next), jpeg); + let destination = tempfile::tempdir().unwrap(); + fs::write(destination.path().join("keep.jpg"), b"existing").unwrap(); + let saved = manager + .call( + "screenSave", + json!({"session":id["session"],"directory":destination.path()}), + ) + .unwrap(); + assert_eq!( + fs::read(saved["path"].as_str().unwrap()).unwrap(), + decode(&next) + ); + assert_eq!( + fs::read(destination.path().join("keep.jpg")).unwrap(), + b"existing" + ); + manager.call("screenStop", id).unwrap(); +} + +#[test] +fn only_explicit_control_of_current_geometry_can_send_input() { + let (directory, manager) = phone(); + let id = start(&manager); + let frame = manager.call("screenFrame", id.clone()).unwrap(); + let request = input(&id, &frame, json!({"kind":"click","x":40,"y":80})); + assert_eq!( + manager + .call("screenInput", request.clone()) + .unwrap_err() + .code, + "screen.read_only" + ); + enable(&manager, &id, &frame); + assert_eq!( + manager.call("screenInput", request.clone()).unwrap()["executed"], + true + ); + frame_file(&directory, 1200, 600, [30, 80, 120]); + let rotated = manager.call("screenFrame", id.clone()).unwrap(); + assert_eq!(rotated["control"], false); + enable(&manager, &id, &rotated); + assert_eq!( + manager.call("screenInput", request).unwrap_err().code, + "screen.stale_frame" + ); + let calls = fs::read_to_string(directory.path().join("hdc.calls")).unwrap(); + assert_eq!( + calls + .lines() + .filter(|line| line.contains("uiInput click")) + .count(), + 1 + ); +} + +#[test] +fn rejects_out_of_bounds_unsupported_and_old_events_without_sending_them() { + let (directory, manager) = phone(); + let id = start(&manager); + let first = manager.call("screenFrame", id.clone()).unwrap(); + enable(&manager, &id, &first); + for event in [ + json!({"kind":"click","x":600,"y":10}), + json!({"kind":"click","x":-1,"y":10}), + json!({"kind":"shell","command":"reboot"}), + json!({"kind":"swipe","x":0,"y":0,"toX":1,"toY":1,"speed":1}), + ] { + assert!(manager + .call("screenInput", input(&id, &first, event)) + .is_err()); + } + for _ in 0..2 { + manager.call("screenFrame", id.clone()).unwrap(); + } + assert_eq!( + manager + .call( + "screenInput", + input(&id, &first, json!({"kind":"key","key":"home"})) + ) + .unwrap_err() + .code, + "screen.stale_frame" + ); + let calls = fs::read_to_string(directory.path().join("hdc.calls")).unwrap(); + assert!(!calls + .lines() + .any(|line| line.contains("uiInput ") && !line.ends_with("help"))); +} + +#[test] +fn text_remains_one_remote_shell_argument_including_chinese_quotes_and_metacharacters() { + let (directory, manager) = phone(); + let id = start(&manager); + let frame = manager.call("screenFrame", id.clone()).unwrap(); + enable(&manager, &id, &frame); + let content = "你好,daniel's phone; $(echo not-executed) `echo not-executed`\n下一行"; + manager + .call( + "screenInput", + input(&id, &frame, json!({"kind":"text","text":content})), + ) + .unwrap(); + let calls = fs::read_to_string(directory.path().join("hdc.calls")).unwrap(); + let command = calls + .split("-t phone shell uitest uiInput text ") + .nth(1) + .unwrap() + .trim_end(); + // Parse the generated remote argument with an actual POSIX shell; it must remain literal data. + let parsed = std::process::Command::new("/bin/sh") + .arg("-c") + .arg(format!("printf '%s' {command}")) + .output() + .unwrap(); + assert!(parsed.status.success()); + assert_eq!(String::from_utf8(parsed.stdout).unwrap(), content); +} + +#[test] +fn stop_cancels_pending_capture_and_old_ids_cannot_control_a_restarted_session() { + let (directory, manager) = phone(); + let id = start(&manager); + fs::write(directory.path().join("hdc.slow"), b"").unwrap(); + let capturing = manager.clone(); + let pending_id = id.clone(); + let pending = thread::spawn(move || capturing.call("screenFrame", pending_id)); + let deadline = Instant::now() + Duration::from_secs(3); + while !directory.path().join("hdc.capturing").exists() { + assert!(Instant::now() < deadline); + thread::sleep(Duration::from_millis(10)); + } + let stopped = Instant::now(); + manager.call("screenStop", id.clone()).unwrap(); + assert!(stopped.elapsed() < Duration::from_millis(500)); + assert!(pending.join().unwrap().is_err()); + assert!(stopped.elapsed() < Duration::from_secs(2)); + fs::remove_file(directory.path().join("hdc.slow")).unwrap(); + let next = start(&manager); + assert_ne!(id, next); + assert_eq!( + manager.call("screenFrame", id).unwrap_err().code, + "screen.closed" + ); + assert!(manager.call("screenFrame", next).is_ok()); +} + +#[test] +fn invalid_capture_closes_the_session_and_remote_input_failure_disables_control() { + let (directory, manager) = phone(); + let id = start(&manager); + let frame = manager.call("screenFrame", id.clone()).unwrap(); + enable(&manager, &id, &frame); + fs::write(directory.path().join("hdc.fail-input"), b"").unwrap(); + assert!(manager + .call( + "screenInput", + input(&id, &frame, json!({"kind":"key","key":"back"})) + ) + .is_err()); + assert_eq!( + manager.call("screenFrame", id.clone()).unwrap()["control"], + false + ); + fs::write(directory.path().join("hdc.png"), b"success but no image").unwrap(); + assert_eq!( + manager.call("screenFrame", id.clone()).unwrap_err().code, + "screen.invalid_frame" + ); + assert_eq!( + manager + .call( + "screenControl", + json!({"session":id["session"],"enabled":true}) + ) + .unwrap_err() + .code, + "screen.closed" + ); +} diff --git a/curios/harmony-connector/native/src/hdc/tests.rs b/curios/harmony-connector/native/src/hdc/tests.rs index 4cee02e..9aaf02b 100644 --- a/curios/harmony-connector/native/src/hdc/tests.rs +++ b/curios/harmony-connector/native/src/hdc/tests.rs @@ -1,6 +1,9 @@ use super::*; use std::{io::Write, time::Instant}; +#[path = "screen_tests.rs"] +mod screen_tests; + fn fixture(debug: bool) -> (tempfile::TempDir, Arc) { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("hdc"); @@ -22,6 +25,7 @@ esac "#)).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); let manager = Arc::new(Manager { + screen: screen::Screen::default(), path: Mutex::new(Some(path)), settings: None, busy: Arc::new(AtomicBool::new(false)), diff --git a/curios/harmony-connector/native/src/main.rs b/curios/harmony-connector/native/src/main.rs index ee056af..be14190 100644 --- a/curios/harmony-connector/native/src/main.rs +++ b/curios/harmony-connector/native/src/main.rs @@ -19,6 +19,10 @@ fn main() -> anyhow::Result<()> { if std::env::args().any(|arg| arg == "--check-reconnect") { return diagnostics::reconnect(); } + let args: Vec<_> = std::env::args().collect(); + if args.get(1).is_some_and(|s| s == "--check-screen") { + return diagnostics::screen(&args[2..]); + } let _ = mtp::version(); wire::run(connector::Connector::new()) } diff --git a/curios/harmony-connector/ui/index.html b/curios/harmony-connector/ui/index.html index db51b20..956b5fa 100644 --- a/curios/harmony-connector/ui/index.html +++ b/curios/harmony-connector/ui/index.html @@ -161,6 +161,101 @@

             
           
+          
+
+
+

投屏与控制

+

未开始投屏

+
+
+ +
+
+
+
+ +
+ +

让手机,来到桌面。

+ 选择已授权的设备,开始投屏。 +
+
+
+ +
+ +
+ +

+ 拖动可滑动,滚轮可滚动,按住可长按。选中画面后,方向键、回车和退格作用于手机。 +

+ +
+ +
+

+ 默认仅观看,启用控制后才会发送操作。 +

+
+
+

+

+ 画面连续采集,刷新速度以设备响应为准。切换工作区、隐藏窗口或断开连接会停止投屏。 +

+

应用

diff --git a/curios/harmony-connector/ui/management.js b/curios/harmony-connector/ui/management.js index e263cfd..3e05946 100644 --- a/curios/harmony-connector/ui/management.js +++ b/curios/harmony-connector/ui/management.js @@ -1,9 +1,12 @@ +import { screen } from "./screen.js"; import { bytes } from "./format.js"; export function management({ native, host, notice }) { const $ = (id) => document.getElementById(id); const call = (method, params = {}) => native(`hdc.${method}`, params); + let screenPanel; const state = { + screenBusy: false, visible: false, available: false, monitorTimer: null, @@ -25,9 +28,9 @@ export function management({ native, host, notice }) { return el; }; function controls() { - const locked = state.working || state.busy; + const locked = state.working || state.busy || state.screenBusy; for (const button of $("management").querySelectorAll("button")) - button.disabled = locked; + if (!button.closest("#screen-panel")) button.disabled = locked; $("hdc-target").disabled = locked; const connected = state.devices.some( (d) => d.id === state.target && d.state === "Connected", @@ -53,11 +56,17 @@ export function management({ native, host, notice }) { $("app-uninstall").disabled = locked || !state.app || !!state.app.details?.applicationInfo?.isSystemApp; $("app-export").disabled = locked || !state.app?.debug; + screenPanel?.configure( + state.target, + connected && !!state.capabilities.screenshot?.available, + state.working || state.busy, + ); $("hdc-cancel").disabled = state.job?.state !== "running" || !!state.job?.cancelling; } async function action(fn) { - if (state.working || state.busy || state.disposed) return; + if (state.working || state.busy || state.screenBusy || state.disposed) + return; state.working = true; controls(); notice(""); @@ -190,7 +199,7 @@ export function management({ native, host, notice }) { } async function monitor() { if (state.disposed || !state.visible) return; - if (!state.working && !state.busy && state.available) { + if (!state.working && !state.busy && !state.screenBusy && state.available) { state.working = true; controls(); try { @@ -375,11 +384,23 @@ export function management({ native, host, notice }) { clearTimeout(state.timer); clearTimeout(state.monitorTimer); }); + screenPanel = screen({ + call, + host, + notice, + onBusy: (busy) => { + state.screenBusy = busy; + controls(); + }, + }); controls(); return (visible) => { state.visible = visible; clearTimeout(state.monitorTimer); - if (!visible) return; + if (!visible) { + screenPanel.hide(); + return; + } action(async () => { if (state.ready) return; const snapshot = await call("status"); diff --git a/curios/harmony-connector/ui/screen-geometry.js b/curios/harmony-connector/ui/screen-geometry.js new file mode 100644 index 0000000..73c7337 --- /dev/null +++ b/curios/harmony-connector/ui/screen-geometry.js @@ -0,0 +1,86 @@ +// Map the displayed, letterboxed image back to the phone's physical screen. +export function screenPoint(rect, width, height, clientX, clientY) { + if ( + ![rect.width, rect.height, width, height].every( + (n) => Number.isFinite(n) && n > 0, + ) + ) + return null; + const scale = Math.min(rect.width / width, rect.height / height); + const left = rect.left + (rect.width - width * scale) / 2; + const top = rect.top + (rect.height - height * scale) / 2; + const x = (clientX - left) / scale, + y = (clientY - top) / scale; + if ( + !Number.isFinite(x) || + !Number.isFinite(y) || + x < 0 || + y < 0 || + x >= width || + y >= height + ) + return null; + return { x: Math.floor(x), y: Math.floor(y) }; +} +export function gesture(from, to, elapsedMs, mode = "click") { + if (!from || !to) return null; + const distance = Math.hypot(to.x - from.x, to.y - from.y); + if (distance >= 12) + return { + kind: mode === "drag" ? "drag" : "swipe", + x: from.x, + y: from.y, + toX: to.x, + toY: to.y, + speed: Math.round( + Math.min( + 40000, + Math.max(200, distance / Math.max(0.1, elapsedMs / 1000)), + ), + ), + }; + return { + kind: + elapsedMs >= 550 || mode === "longClick" + ? "longClick" + : mode === "doubleClick" + ? "doubleClick" + : "click", + ...to, + }; +} +export function scrollGesture(point, dx, dy, width, height) { + if (!point || ![dx, dy].every(Number.isFinite)) return null; + const toX = Math.round( + Math.max( + 0, + Math.min( + width - 1, + point.x - Math.max(-width * 0.6, Math.min(width * 0.6, dx)), + ), + ), + ); + const toY = Math.round( + Math.max( + 0, + Math.min( + height - 1, + point.y - Math.max(-height * 0.6, Math.min(height * 0.6, dy)), + ), + ), + ); + if (toX === point.x && toY === point.y) return null; + return { kind: "swipe", ...point, toX, toY, speed: 1500 }; +} +export const screenKeys = { + Escape: "back", + Home: "home", + Enter: "enter", + Backspace: "backspace", + Delete: "delete", + Tab: "tab", + ArrowUp: "up", + ArrowDown: "down", + ArrowLeft: "left", + ArrowRight: "right", +}; diff --git a/curios/harmony-connector/ui/screen-geometry.test.js b/curios/harmony-connector/ui/screen-geometry.test.js new file mode 100644 index 0000000..8138b85 --- /dev/null +++ b/curios/harmony-connector/ui/screen-geometry.test.js @@ -0,0 +1,51 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { screenPoint, gesture, scrollGesture } from "./screen-geometry.js"; + +test("portrait phone maps the displayed image and ignores side letterboxes", () => { + const rect = { left: 20, top: 30, width: 800, height: 600 }; + assert.deepEqual(screenPoint(rect, 1200, 2400, 420, 330), { + x: 600, + y: 1200, + }); + assert.equal(screenPoint(rect, 1200, 2400, 100, 330), null); + assert.equal(screenPoint(rect, 1200, 2400, 571, 330), null); + assert.equal(screenPoint(rect, 1200, 2400, 420, 631), null); +}); +test("rotated phone uses physical landscape coordinates with top letterboxing", () => { + const rect = { left: 0, top: 0, width: 600, height: 800 }; + assert.deepEqual(screenPoint(rect, 2400, 1200, 300, 400), { + x: 1200, + y: 600, + }); + assert.equal(screenPoint(rect, 2400, 1200, 300, 200), null); + assert.equal(screenPoint(rect, 2400, 1200, Infinity, 400), null); +}); +test("gestures distinguish taps, long presses and drag paths with bounded speed", () => { + const from = { x: 200, y: 400 }, + to = { x: 500, y: 800 }; + assert.equal(gesture(from, from, 100).kind, "click"); + assert.equal(gesture(from, from, 700).kind, "longClick"); + assert.equal(gesture(from, from, 100, "doubleClick").kind, "doubleClick"); + assert.deepEqual(gesture(from, to, 1000), { + kind: "swipe", + x: 200, + y: 400, + toX: 500, + toY: 800, + speed: 500, + }); + assert.equal(gesture(from, to, 1000, "drag").kind, "drag"); + assert.equal(gesture(from, null, 100), null); +}); +test("wheel gestures remain inside the physical phone screen", () => { + assert.deepEqual(scrollGesture({ x: 10, y: 20 }, 9999, -9999, 600, 1200), { + kind: "swipe", + x: 10, + y: 20, + toX: 0, + toY: 740, + speed: 1500, + }); + assert.equal(scrollGesture({ x: 10, y: 20 }, 0, 0, 600, 1200), null); +}); diff --git a/curios/harmony-connector/ui/screen.js b/curios/harmony-connector/ui/screen.js new file mode 100644 index 0000000..9925d40 --- /dev/null +++ b/curios/harmony-connector/ui/screen.js @@ -0,0 +1,476 @@ +import { + screenPoint, + gesture, + scrollGesture, + screenKeys, +} from "./screen-geometry.js"; + +export function screen({ call, host, notice, onBusy }) { + const document = window.document; + const $ = (id) => document.getElementById(id); + const state = { + target: "", + available: false, + blocked: false, + session: null, + active: false, + starting: false, + stopping: false, + control: false, + controlRevision: 0, + supported: [], + frame: null, + revision: 0, + pending: null, + timer: null, + flight: null, + pointer: null, + wheel: null, + wheelTimer: null, + displayedAt: 0, + samples: [], + disposed: false, + }; + function controls() { + $("screen-start").disabled = + !state.available || + state.blocked || + state.active || + state.starting || + state.stopping; + $("screen-stop").disabled = !state.active || state.stopping; + $("screen-control").disabled = + !state.active || + !state.frame || + state.supported.length === 0 || + state.stopping; + $("screen-control").checked = state.control; + const enabled = + state.active && state.control && !!state.frame && !state.stopping; + for (const el of $("screen-inputs").querySelectorAll( + "button,textarea,select", + )) + el.disabled = !enabled; + $("screen-send-text").disabled = + !enabled || !state.supported.includes("text") || !$("screen-text").value; + $("screen-text").disabled = !enabled || !state.supported.includes("text"); + for (const el of $("screen-inputs").querySelectorAll("[data-screen-key]")) + el.disabled = !enabled || !state.supported.includes("keyEvent"); + for (const option of $("screen-gesture").options) + option.disabled = !state.supported.includes(option.value); + $("screen-save").disabled = !state.active || !state.frame || state.stopping; + $("screen-stage").classList.toggle("controlling", enabled); + $("screen-status").textContent = state.stopping + ? "正在停止…" + : state.starting + ? "正在连接屏幕…" + : state.active + ? state.frame + ? state.control + ? "正在投屏 · 可控制" + : "正在投屏 · 仅观看" + : "等待手机画面…" + : "未开始投屏"; + } + function clearPending() { + state.pending = null; + state.pointer = null; + state.wheel = null; + clearTimeout(state.wheelTimer); + } + function clearFrame() { + state.frame = null; + state.control = false; + clearPending(); + state.samples = []; + $("screen-image").removeAttribute("src"); + $("screen-image").hidden = true; + $("screen-placeholder").hidden = false; + $("screen-stats").textContent = ""; + $("screen-text").value = ""; + $("screen-input-status").textContent = "默认仅观看,启用控制后才会发送操作。"; + } + async function stop() { + const session = state.session; + state.revision++; + state.active = false; + state.session = null; + state.stopping = !!session; + clearTimeout(state.timer); + clearFrame(); + controls(); + try { + if (session) await call("screenStop", { session }); + } catch (error) { + if (!state.disposed) notice(error.message || String(error)); + } finally { + await state.flight?.catch(() => {}); + state.stopping = false; + if (state.disposed) return; + if (!state.starting) onBusy(false); + controls(); + } + } + function schedule(delay = 100) { + clearTimeout(state.timer); + if (state.active && !state.disposed) state.timer = setTimeout(pump, delay); + } + async function pump() { + if (!state.active || state.flight || state.disposed) return; + const revision = state.revision; + state.flight = (async () => { + if (state.pending) { + const pending = state.pending; + state.pending = null; + if (Date.now() - pending.created <= 1500 && state.control) { + const before = performance.now(); + try { + await call("screenInput", pending.request); + if (revision === state.revision && !state.disposed) + $("screen-input-status").textContent = + `操作已发送 · ${Math.round(performance.now() - before)} ms`; + } catch (error) { + if (revision !== state.revision || state.disposed) return; + state.control = false; + clearPending(); + controls(); + notice(error.message || String(error)); + } + } else + $("screen-input-status").textContent = + "画面响应较慢,已丢弃等待过久的操作。"; + } + if (!state.active || revision !== state.revision) return; + const controlRevision = state.controlRevision; + const frame = await call("screenFrame", { session: state.session }); + if (!state.active || revision !== state.revision) return; + if ( + !/^data:image\/jpeg;base64,/.test(frame.dataUrl) || + !Number.isInteger(frame.width) || + !Number.isInteger(frame.height) || + frame.width < 1 || + frame.height < 1 + ) + throw new Error("手机画面格式无效。"); + const image = $("screen-image"); + image.src = frame.dataUrl; + if (image.decode) await image.decode(); + if (!state.active || revision !== state.revision) { + image.removeAttribute("src"); + return; + } + const changed = state.frame && state.frame.geometry !== frame.geometry; + if (changed) { + state.controlRevision++; + state.control = false; + clearPending(); + $("screen-input-status").textContent = + "屏幕方向已变化,请重新启用控制。"; + } + state.frame = frame; + if (!changed && controlRevision === state.controlRevision) + state.control = !!frame.control; + state.displayedAt = Date.now(); + state.samples.push(performance.now()); + if (state.samples.length > 8) state.samples.shift(); + const elapsed = state.samples.at(-1) - state.samples[0]; + const fps = + elapsed > 0 + ? (((state.samples.length - 1) * 1000) / elapsed).toFixed(1) + : "—"; + $("screen-stats").textContent = + `${frame.width} × ${frame.height} · ${fps} 帧/秒 · 采集 ${Math.round(frame.captureMs)} ms`; + image.hidden = false; + $("screen-placeholder").hidden = true; + controls(); + })(); + let failure; + try { + await state.flight; + } catch (error) { + failure = error; + } finally { + state.flight = null; + } + if (revision !== state.revision || state.disposed) return; + if (failure) { + notice(failure.message || String(failure)); + await stop(); + } else schedule(state.pending ? 0 : 100); + } + async function start() { + if ( + state.starting || + state.active || + state.stopping || + state.blocked || + !state.available + ) + return; + const revision = ++state.revision; + state.starting = true; + onBusy(true); + clearFrame(); + controls(); + notice(""); + try { + const opened = await call("screenStart", { target: state.target }); + if (revision !== state.revision || state.disposed) { + await call("screenStop", { session: opened.session }); + return; + } + state.session = opened.session; + state.supported = opened.inputCapabilities; + state.active = true; + $("screen-input-status").textContent = state.supported.length + ? "启用控制后,可在画面中点击、长按或拖动。" + : "设备仅开放画面采集,未开放输入控制。"; + schedule(0); + } catch (error) { + if (!state.disposed && revision === state.revision) + notice(error.message || String(error)); + } finally { + state.starting = false; + if (!state.disposed) { + onBusy(state.active || state.stopping); + controls(); + } + } + } + function canControl() { + return ( + state.active && + state.control && + state.frame && + Date.now() - state.displayedAt < 3000 && + !state.stopping + ); + } + function send(input, frame = state.frame) { + if ( + !input || + !canControl() || + !frame || + frame.geometry !== state.frame.geometry + ) + return; + if ( + !state.supported.includes(input.kind === "key" ? "keyEvent" : input.kind) + ) + return; + if (state.pending) { + $("screen-input-status").textContent = "上一项操作仍在等待,请稍候。"; + return; + } + state.pending = { + created: Date.now(), + request: { + session: state.session, + geometry: frame.geometry, + sequence: frame.sequence, + input, + }, + }; + $("screen-input-status").textContent = "等待发送操作…"; + if (!state.flight) schedule(0); + } + function point(event) { + return state.frame + ? screenPoint( + $("screen-stage").getBoundingClientRect(), + state.frame.width, + state.frame.height, + event.clientX, + event.clientY, + ) + : null; + } + $("screen-start").onclick = start; + $("screen-stop").onclick = stop; + $("screen-control").onchange = async () => { + const revision = state.revision; + const controlRevision = ++state.controlRevision; + const enabled = $("screen-control").checked; + if (!enabled) { + state.control = false; + clearPending(); + controls(); + } + try { + const result = await call("screenControl", { + session: state.session, + enabled, + geometry: state.frame?.geometry, + }); + if ( + revision !== state.revision || + controlRevision !== state.controlRevision || + state.disposed + ) + return; + state.control = !!result.control; + } catch (error) { + if (revision !== state.revision || state.disposed) return; + state.control = false; + notice(error.message || String(error)); + } + controls(); + }; + const stage = $("screen-stage"); + stage.onpointerdown = (event) => { + if (!canControl() || ![0, 2].includes(event.button)) return; + const from = point(event); + if (!from) return; + event.preventDefault(); + stage.focus(); + stage.setPointerCapture?.(event.pointerId); + state.pointer = { + from, + frame: state.frame, + time: performance.now(), + id: event.pointerId, + mode: event.button === 2 ? "longClick" : $("screen-gesture").value, + }; + }; + stage.onpointerup = (event) => { + const pointer = state.pointer; + state.pointer = null; + if (!pointer || pointer.id !== event.pointerId) return; + send( + gesture( + pointer.from, + point(event), + performance.now() - pointer.time, + pointer.mode, + ), + pointer.frame, + ); + if (stage.hasPointerCapture?.(event.pointerId)) + stage.releasePointerCapture(event.pointerId); + }; + stage.onpointercancel = () => { + state.pointer = null; + }; + stage.oncontextmenu = (event) => { + if (canControl()) event.preventDefault(); + }; + stage.addEventListener( + "wheel", + (event) => { + if (!canControl() || !state.supported.includes("swipe")) return; + const at = point(event); + if (!at) return; + event.preventDefault(); + const scale = + event.deltaMode === 1 ? 20 : event.deltaMode === 2 ? 300 : 2; + const wheel = state.wheel || { + point: at, + frame: state.frame, + dx: 0, + dy: 0, + }; + wheel.dx += event.deltaX * scale; + wheel.dy += event.deltaY * scale; + state.wheel = wheel; + clearTimeout(state.wheelTimer); + state.wheelTimer = setTimeout(() => { + state.wheel = null; + send( + scrollGesture( + wheel.point, + wheel.dx, + wheel.dy, + wheel.frame.width, + wheel.frame.height, + ), + wheel.frame, + ); + }, 120); + }, + { passive: false }, + ); + stage.onkeydown = (event) => { + if ( + !canControl() || + event.isComposing || + event.ctrlKey || + event.metaKey || + event.altKey + ) + return; + if (screenKeys[event.key] && state.supported.includes("keyEvent")) { + event.preventDefault(); + if (!event.repeat) send({ kind: "key", key: screenKeys[event.key] }); + } else if (event.key.length === 1 && state.supported.includes("text")) { + event.preventDefault(); + $("screen-text").focus(); + $("screen-text").value += event.key; + controls(); + } + }; + for (const el of $("screen-inputs").querySelectorAll("[data-screen-key]")) + el.onclick = () => send({ kind: "key", key: el.dataset.screenKey }); + $("screen-text").oninput = controls; + $("screen-send-text").onclick = () => + send({ kind: "text", text: $("screen-text").value }); + $("screen-text").onkeydown = (event) => { + if ( + event.key === "Enter" && + (event.metaKey || event.ctrlKey) && + !event.isComposing + ) { + event.preventDefault(); + $("screen-send-text").click(); + } + }; + $("screen-expand").onclick = () => { + const expanded = $("screen-panel").classList.toggle("expanded"); + $("screen-expand").textContent = expanded ? "恢复大小" : "放大画面"; + }; + $("screen-save").onclick = async () => { + const revision = state.revision; + try { + const { path } = await host("openDirectory", { + title: "保存当前手机画面", + }); + if ( + path && + revision === state.revision && + state.active && + !state.disposed + ) { + const result = await call("screenSave", { + session: state.session, + directory: path, + }); + if (revision === state.revision && !state.disposed) + $("screen-input-status").textContent = `画面已保存到 ${result.path}`; + } + } catch (error) { + if (revision === state.revision && !state.disposed) + notice(error.message || String(error)); + } + }; + document.addEventListener("visibilitychange", () => { + if (document.hidden) stop(); + }); + window.addEventListener("pagehide", () => { + state.disposed = true; + stop(); + }); + controls(); + return { + configure(target, available, blocked) { + if (state.target !== target && (state.active || state.starting)) stop(); + state.target = target; + state.available = available; + state.blocked = blocked; + controls(); + }, + hide() { + if (state.active || state.starting) stop(); + $("screen-panel").classList.remove("expanded"); + $("screen-expand").textContent = "放大画面"; + }, + }; +} diff --git a/curios/harmony-connector/ui/style.css b/curios/harmony-connector/ui/style.css index 941ecad..3c3001d 100644 --- a/curios/harmony-connector/ui/style.css +++ b/curios/harmony-connector/ui/style.css @@ -933,3 +933,119 @@ dialog li { justify-content: flex-end; } } + +.screen-card h2 { + text-wrap: balance; +} +.screen-card.expanded { + position: fixed; + inset: 12px; + z-index: 20; + overflow: auto; + background: var(--bg); + margin: 0; + box-shadow: 0 8px 70px #0006; +} +.screen-layout { + display: grid; + grid-template-columns: minmax(0, 1fr) 220px; + gap: 22px; +} +.screen-stage { + position: relative; + height: 510px; + min-height: 240px; + background: #111318; + border-radius: 12px; + display: grid; + place-items: center; + overflow: hidden; + outline: 1px solid rgba(0, 0, 0, 0.1); +} +.screen-stage.controlling { + cursor: crosshair; + touch-action: none; + user-select: none; +} +.screen-stage:focus-visible { + outline: 3px solid var(--blue); + outline-offset: 3px; +} +#screen-image { + position: absolute; + width: 100%; + height: 100%; + object-fit: contain; + pointer-events: none; +} +#screen-placeholder { + text-align: center; + color: #a4afc3; + padding: 24px; +} +#screen-placeholder > span { + font-size: 64px; + font-weight: 200; +} +#screen-placeholder p { + font-size: 18px; + color: #d6deeb; +} +.screen-inputs { + display: flex; + flex-direction: column; + gap: 10px; + align-self: start; +} +.screen-inputs label { + font-size: 12px; +} +.control-switch { + display: flex; + align-items: center; + min-height: 40px; + gap: 6px; +} +.screen-inputs textarea { + width: 100%; + resize: vertical; + min-height: 90px; + font: inherit; + line-height: 1.6; + padding: 10px; + color: var(--ink); + background: var(--bg); + border: 1px solid var(--line); + border-radius: 9px; +} +.screen-nav { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} +.screen-inputs p { + margin: 0; +} +.screen-card button:active:not(:disabled) { + transform: scale(0.96); +} +#screen-stats { + font-variant-numeric: tabular-nums; + min-height: 22px; +} +@media (max-width: 960px) { + .screen-layout { + grid-template-columns: 1fr; + } + .screen-inputs { + max-width: 100%; + } + .screen-stage { + height: 440px; + } +} +@media (prefers-color-scheme: dark) { + .screen-stage { + outline-color: rgba(255, 255, 255, 0.1); + } +} diff --git a/docs/harmony-connector.md b/docs/harmony-connector.md index fd3fcd0..d2cd888 100644 --- a/docs/harmony-connector.md +++ b/docs/harmony-connector.md @@ -1,6 +1,6 @@ # Harmony Connector -插件 ID:`dev.trove.harmony-connector`。当前版本:`0.2.0`。 +插件 ID:`dev.trove.harmony-connector`。当前版本:`0.3.0`。 macOS 上的鸿蒙设备连接工具。通过 USB 收发文件与完整文件夹;使用官方 HDC 管理应用、保存截图、导出日志和调试应用数据。功能按连接工具、设备实际能力和授权开放,不按 HarmonyOS 大版本或机型名称限制。 @@ -57,17 +57,27 @@ macOS 上的鸿蒙设备连接工具。通过 USB 收发文件与完整文件夹 管理操作使用独立串行任务,支持取消与超时;取消不回滚设备端已经完成的更改。操作开始前重新确认所选设备在线。功能有命令支持仍可能被单个应用、对象或系统权限拒绝,失败会显示设备返回的原因。 -## 投屏和控制调研 +## 投屏与控制 -**本版本尚未实现实时投屏或键鼠控制。** 官方 scrcpy 的设备端依赖 Android、ADB、`app_process` 和 Android 屏幕/输入 API,不能直接连接原生鸿蒙。仅能运行 APK 并不足以证明 scrcpy 兼容。[scrcpy 官方说明](https://github.com/Genymobile/scrcpy) +在「设备与应用」选择已授权的 HDC 目标,点击「开始投屏」。默认仅观看;画面出现后勾选「启用鼠标与键盘控制」。功能依设备的 `uitest` 帮助和实际操作结果开放。 -最新 DevEco Studio 与 DevEco Testing 已提供官方 Mac 投屏功能;目前未找到可嵌入本插件的公开投屏协议。可行的插件路线是先以 HDC + UITest 实现截图预览和明确的点击/滑动输入,再单独验证用户授权的鸿蒙采集端、视频编码和传输。连续视频方案需要设备端适配,不能把截图轮询宣传成 scrcpy 视频流。详见[投屏与键鼠控制调研](research/harmony-screen-control.md)。 +- 连续显示手机画面,显示真实分辨率、实测刷新帧率和采集耗时。可放大面板,将当前画面保存为 JPEG。 +- 点击、双击、长按、滑动、拖拽与滚轮滚动;坐标按实际手机尺寸映射,忽略画面周围留白。 +- 返回、主页、音量、方向键、回车、退格等按键。快捷键只在手机画面获得焦点时生效。 +- 用专用文字框发送文本,支持输入法;不读取系统剪贴板。文字能否进入目标输入框,以手机焦点和输入权限为准。 +- 一个屏幕会话只保留一项待发送操作。停止、切换工作区、页面隐藏、断线时清空队列和旧画面,不在重连后重放输入。横竖屏变化后需重新启用控制。 + +当前实现通过 HDC + UITest 连续采集截图,完成一帧再开始下一帧;不是 H.264/H.265 视频流,不承诺 scrcpy 的 30/60 FPS、音频或低延迟多点触控。拖动在松开时发送完整手势。采集临时文件使用每次唯一名称并清理;远端清理被断线阻断时,不扫描删除其他文件。 + +图像解码限制内存与尺寸,传给 UI 的 JPEG 最长边不超过 1600 像素;坐标仍按手机原始尺寸计算。画面过旧、方向变化、会话过期或输入失败时会阻止控制。受保护画面和锁屏遵循系统限制,不自动解锁。自动化测试和真机结果分别记录,详见[投屏能力调研](research/harmony-screen-control.md)。 ## 重连问题与 debug 0.1.1 真机记录:HUAWEI Pura 70 Ultra,USB `12d1:1101`,USB 名称 `HDC Device`,MTP 友好名称 `HUAWEI Pura 70 Ultra`,型号代码 `HBP-AL00`。设备未通过 MTP 返回自定义名称 `daniel`。连接后可读取存储,断开请求约 0.27 毫秒返回,USB 释放约 61 秒后结束;**这不证明随后能重新打开会话**。用户报告仍需拔线。 -0.2.0 没有将未经真机验证的 USB reset 或超时补丁当作重连修复。调研确认当前 libmtp 标志已经执行 reset,新增同样 reset 不能作为修复依据。须区分 CloseSession、接口释放、USB reset、重新枚举和设备端会话状态。见[会话恢复调研](research/harmony-mtp-session.md)。本轮诊断没有发现 USB 设备,故重连仍待真机复现和修复。 +0.3.0 真机复现了 CloseSession 等待 60 秒的问题。固定 libmtp 依赖增加一个小补丁,仅把 CloseSession 的超时设为 2 秒,文件传输超时、接口释放、现有 USB reset 与关闭顺序保持不变。macOS 重枚举有短暂离线窗口,诊断程序最多等 10 秒重新发现设备,再实际打开并读取存储和目录。 + +2026-09-10,在同一 USB 插接周期,开发构建与发布构建各完成 3 轮只读重连,每轮返回 1 个存储和 9 个根目录条目。开发构建关闭约 2.28–2.33 秒;发布构建约 2.13–7.28 秒,包含 USB 重枚举耗时。设备仍不正常响应 CloseSession,恢复由随后的 USB 清理/reset 完成;不能将 2 秒超时宣传为所有设备都固定 2 秒完成断开。该结果覆盖这台手机,不代表全部固件。见[会话恢复调研](research/harmony-mtp-session.md)。 关闭插件窗口,保留一台解锁且处于「传输文件」模式的手机,然后执行: @@ -79,7 +89,19 @@ curios/harmony-connector/backend/harmony-connector --diagnose curios/harmony-connector/backend/harmony-connector --check-reconnect ``` -`--check-reconnect` 逐阶段输出 JSON、耗时和实际目录读取结果,不修改手机文件。没有恰好一台设备或任何一轮失败时以非零状态结束;原生 CloseSession 仍可能等待 USB 超时。不能仅凭扫描发现设备、按钮可用或 reset 返回成功判定验收通过。 +`--check-reconnect` 逐阶段输出 JSON、耗时和实际目录读取结果,不修改手机文件。没有恰好一台设备或任何一轮失败时以非零状态结束;CloseSession 时限为 2 秒,USB 清理和重枚举仍可能增加耗时。不能仅凭扫描发现设备、按钮可用或 reset 返回成功判定验收通过。 + +只读验证投屏(先完成手机 USB 调试授权): + +```sh +curios/harmony-connector/backend/harmony-connector --check-screen /absolute/path/to/hdc +# 多台设备时可指定目标;最后的可选目录用于保存最终画面 +curios/harmony-connector/backend/harmony-connector --check-screen /absolute/path/to/hdc DEVICE_ID /absolute/save/directory +``` + +该命令走 UI 相同的后端接口,采集并解码 3 帧,输出耗时,最后关闭会话;不启用控制、不发送输入。首次 HDC 连接显示 Unauthorized 时,在手机上允许 USB 调试。`hdc -v` 能运行但管理失败时,先用 `hdc list targets -v` 检查授权状态。 + +本次自动化验证通过 39 项原生测试、27 项 UI/格式/坐标测试、3 项 USB C 桥接检查及 3 项底座协议检查。浏览器另外验证了持续显示模拟画面、启用控制、中文输入、放大和停止清空;模拟数据不会包含在发布包内。官方 Command Line Tools 26.0.0.821(Mac ARM)的 HDC 返回 `Ver: 3.2.0f`,已识别真机,但当次状态为 Unauthorized,故投屏和输入的真机效果尚未验收,不能将上述替身测试当作真机通过。 ## 构建与发布 @@ -95,7 +117,7 @@ pnpm run release:prepare --entries dist/harmony-release.json 发布创建新的不可变版本产物,保留旧版本、哈希和生产源的已有条目。推送后必须以 Git 仓库为源执行安装验证。生产 README 保持产品介绍;技术约束与验收记录在本文和 `docs/research/`。 -代码职责:`native/src/file_transfer.rs` 管理目录计划与落盘,`progress.rs` 统计进度,`connector.rs` 管理 MTP 会话与任务,`mtp.rs` / `mtp_bridge.c` 封装固定依赖,`hdc.rs` / `hdc_data.rs` / `hdc/files.rs` 管理官方工具与输出校验,`command.rs` 控制进程、输出、取消和时限。UI 的文件与管理工作区分开实现。 +代码职责:`native/src/file_transfer.rs` 管理目录计划与落盘,`progress.rs` 统计进度,`connector.rs` 管理 MTP 会话与任务,`mtp.rs` / `mtp_bridge.c` 封装固定依赖,`hdc.rs` / `hdc_data.rs` / `hdc/files.rs` 管理官方工具与输出校验,`hdc/screen.rs` / `screen_input.rs` 管理屏幕会话与输入,`command.rs` 控制进程、输出、取消和时限。UI 的文件与管理工作区分开实现。 验证包括文件夹往返与失败清理、会话状态、工具命令和解析、安装确认及后置条件、页面交互、USB 桥接和底座协议。HDC 测试使用明确的工具替身,不代表商业手机已经完成安装、备份或截图验收。实际文件吞吐量和 USB 往返 SHA-256 仍需真机记录。 diff --git a/source/index.json b/source/index.json index 0d12ebe..cfdf43d 100644 --- a/source/index.json +++ b/source/index.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "id": "trove-official", "name": "Trove 官方软件源", - "generatedAt": "2026-09-10T03:55:38.730Z", + "generatedAt": "2026-09-10T06:14:13.482Z", "curios": [ { "id": "dev.trove.qa.host", @@ -198,8 +198,23 @@ { "id": "dev.trove.harmony-connector", "name": "Harmony Connector", - "description": "连接鸿蒙设备,收发完整文件夹、管理应用,轻松保存截图和日志。", + "description": "连接鸿蒙设备,收发完整文件夹、管理应用,查看手机画面并通过键鼠操作。", "versions": [ + { + "version": "0.3.0", + "engine": { + "trove": ">=0.1.0 <1.0.0" + }, + "releasedAt": "2026-09-10T06:14:13.482Z", + "artifacts": [ + { + "target": "aarch64-apple-darwin", + "url": "packages/dev.trove.harmony-connector/0.3.0/aarch64-apple-darwin.curio", + "sha256": "e773d5583fc6f51508fd41926ba455a35147e8fdf7833a1ff9cc2233f4d93c03", + "size": 3030997 + } + ] + }, { "version": "0.2.0", "engine": { diff --git a/source/packages/dev.trove.harmony-connector/0.3.0/aarch64-apple-darwin.curio b/source/packages/dev.trove.harmony-connector/0.3.0/aarch64-apple-darwin.curio new file mode 100644 index 0000000..5257812 Binary files /dev/null and b/source/packages/dev.trove.harmony-connector/0.3.0/aarch64-apple-darwin.curio differ diff --git a/tools/fixtures/harmony-ui.mjs b/tools/fixtures/harmony-ui.mjs index 9d31bc4..1ec20ef 100644 --- a/tools/fixtures/harmony-ui.mjs +++ b/tools/fixtures/harmony-ui.mjs @@ -61,13 +61,80 @@ const entries = [ modified: 1788998400, }, ]; +let screenId = 0, + screenSequence = 0, + screenControl = false; +let lastInput = "等待操作"; +function screenFrame() { + if (!attached) throw new Error("设备已断开(模拟)"); + const canvas = document.createElement("canvas"); + canvas.width = 600; + canvas.height = 1200; + const context = canvas.getContext("2d"); + const gradient = context.createLinearGradient(0, 0, 600, 1200); + gradient.addColorStop(0, "#183958"); + gradient.addColorStop(1, "#3d1d52"); + context.fillStyle = gradient; + context.fillRect(0, 0, 600, 1200); + context.fillStyle = "#fff"; + context.textAlign = "center"; + context.font = "bold 70px sans-serif"; + context.fillText("Harmony", 300, 260); + context.font = "26px sans-serif"; + context.fillText("投屏界面预览 · 模拟画面", 300, 320); + context.fillStyle = "#e9dcff"; + context.fillText(lastInput, 300, 600, 550); + context.fillText(`画面 ${++screenSequence}`, 300, 670); + context.fillStyle = "#fff"; + context.fillRect(225, 1150, 150, 6); + return { + dataUrl: canvas.toDataURL("image/jpeg", 0.8), + width: 600, + height: 1200, + geometry: 1, + sequence: screenSequence, + captureMs: 180, + control: screenControl, + }; +} window.__TROVE_CONTEXT__ = { curioId: "dev.trove.harmony-connector", - version: "0.2.0", + version: "0.3.0", instanceId: "ui-fixture", platform: "macos", }; window.__TROVE_BRIDGE__ = createMockBridge({ + "@self.hdc.screenStart": () => { + screenControl = false; + return { + session: ++screenId, + control: false, + inputCapabilities: [ + "click", + "doubleClick", + "longClick", + "swipe", + "drag", + "text", + "keyEvent", + ], + }; + }, + "@self.hdc.screenFrame": screenFrame, + "@self.hdc.screenStop": () => ({ stopped: true }), + "@self.hdc.screenControl": ({ enabled }) => ({ + control: (screenControl = enabled), + }), + "@self.hdc.screenInput": ({ input }) => { + lastInput = + input.kind === "text" + ? input.text + : input.kind === "key" + ? input.key + : `${input.kind} · ${input.x}, ${input.y}`; + return { executed: true }; + }, + "@self.hdc.screenSave": () => ({ path: "/ui-fixture/screen.jpg" }), "@self.hdc.status": () => ({ busy: false, job: null }), "@self.hdc.tool": () => ({ available: true, diff --git a/tools/harmony-ui.test.mjs b/tools/harmony-ui.test.mjs index cae52e1..db19e7b 100644 --- a/tools/harmony-ui.test.mjs +++ b/tools/harmony-ui.test.mjs @@ -53,6 +53,12 @@ async function app(t) { job: null, busy: false, monitorWait: null, + screenSession: 0, + screenSequence: 0, + screenGeometry: 1, + screenControl: false, + screenFrameWait: null, + screenFrameError: null, }; const { createMockBridge } = await import( pathToFileURL(path.join(directory, "sdk/index.js")) @@ -73,6 +79,41 @@ async function app(t) { }; window.__TROVE_BRIDGE__ = createMockBridge( Object.fromEntries([ + handler("hdc.screenStart", () => ({ + session: ++model.screenSession, + inputCapabilities: [ + "click", + "doubleClick", + "longClick", + "swipe", + "drag", + "text", + "keyEvent", + ], + control: false, + })), + handler("hdc.screenStop", () => { + model.screenControl = false; + return { stopped: true }; + }), + handler("hdc.screenControl", ({ enabled }) => ({ + control: (model.screenControl = enabled), + })), + handler("hdc.screenFrame", async () => { + if (model.screenFrameWait) await model.screenFrameWait; + if (model.screenFrameError) throw model.screenFrameError; + return { + dataUrl: "data:image/jpeg;base64,/9j/2Q==", + width: 1200, + height: 2400, + captureMs: 300, + sequence: ++model.screenSequence, + geometry: model.screenGeometry, + control: model.screenControl, + }; + }), + handler("hdc.screenInput", () => ({ executed: true })), + handler("hdc.screenSave", () => ({ path: "/fixture/screen.jpg" })), handler("hdc.status", () => ({ busy: false, job: null })), handler("hdc.tool", () => ({ available: model.hdcAvailable, @@ -365,3 +406,113 @@ test("device disconnect clears app details and prevents stale target operations" assert.equal($("app-install").disabled, true); assert.equal($("app-launch").disabled, true); }); + +test("screen viewing starts read-only, keeps refreshing and reports measured capture time", async (t) => { + const { $, click, tick, model } = await app(t); + model.hdcAvailable = true; + await click("tab-management"); + assert.equal($("screen-start").disabled, false); + await click("screen-start"); + await tick(); + assert.equal($("screen-image").hidden, false); + assert.match($("screen-status").textContent, /仅观看/); + assert.match($("screen-stats").textContent, /1200 × 2400.*300 ms/); + assert.equal($("screen-control").checked, false); + assert.equal($("app-install").disabled, true); + const before = model.screenSequence; + await tick(); + assert.ok(model.screenSequence > before); + assert.ok(!model.calls.includes("hdc.screenInput")); +}); + +test("screen text and navigation require enabled control and use the displayed session", async (t) => { + const { $, click, tick, model } = await app(t); + model.hdcAvailable = true; + await click("tab-management"); + await click("screen-start"); + await tick(); + await click("screen-control"); + $("screen-text").value = "你好,daniel's phone"; + $("screen-text").dispatchEvent(new window.Event("input")); + await click("screen-send-text"); + await tick(); + const sent = model.parameters.find(([name]) => name === "hdc.screenInput")[1]; + assert.deepEqual(sent, { + session: 1, + sequence: 1, + geometry: 1, + input: { kind: "text", text: "你好,daniel's phone" }, + }); + await click("screen-stop"); + assert.equal($("screen-image").hidden, true); + assert.equal($("screen-image").hasAttribute("src"), false); + assert.equal($("screen-text").disabled, true); + assert.equal($("app-install").disabled, false); +}); + +test("stopping during a pending frame discards that response and permits a fresh screen session", async (t) => { + const { $, click, tick, model } = await app(t); + model.hdcAvailable = true; + await click("tab-management"); + let release; + model.screenFrameWait = new Promise((resolve) => { + release = resolve; + }); + await click("screen-start"); + await tick(); + await click("screen-stop"); + assert.equal($("screen-image").hidden, true); + assert.ok(model.calls.includes("hdc.screenStop")); + release(); + model.screenFrameWait = null; + await settle(); + assert.equal($("screen-image").hasAttribute("src"), false); + await click("screen-start"); + await tick(); + assert.equal(model.screenSession, 2); + assert.equal($("screen-image").hidden, false); +}); + +test("capture failure clears the screen and queued input instead of showing an old live frame", async (t) => { + const { $, click, tick, model } = await app(t); + model.hdcAvailable = true; + await click("tab-management"); + await click("screen-start"); + await tick(); + model.screenFrameError = new Error("Device disconnected"); + await tick(); + assert.equal($("screen-image").hidden, true); + assert.match($("screen-status").textContent, /未开始/); + assert.equal($("screen-control").checked, false); +}); + +test("leaving management stops refresh and control; returning does not resume them", async (t) => { + const { $, click, tick, model } = await app(t); + model.hdcAvailable = true; + await click("tab-management"); + await click("screen-start"); + await tick(); + await click("screen-control"); + await click("tab-files"); + const before = model.screenSequence; + await tick(); + await click("tab-management"); + await tick(); + assert.equal(model.screenSequence, before); + assert.equal($("screen-image").hidden, true); + assert.equal($("screen-control").checked, false); +}); + +test("rotation disables control and clears a gesture waiting to be sent", async (t) => { + const { $, click, tick, model } = await app(t); + model.hdcAvailable = true; + await click("tab-management"); + await click("screen-start"); + await tick(); + await click("screen-control"); + model.screenGeometry++; + model.screenControl = false; + await tick(); + assert.equal($("screen-control").checked, false); + assert.match($("screen-input-status").textContent, /方向已变化/); +}); diff --git a/tools/test-harmony.mjs b/tools/test-harmony.mjs index 0ba61da..9905fc0 100644 --- a/tools/test-harmony.mjs +++ b/tools/test-harmony.mjs @@ -19,6 +19,7 @@ run( }, ); run(process.execPath, ["--test", path.join(plugin, "ui/format.test.js")]); +run(process.execPath, ["--test", path.join(plugin, "ui/screen-geometry.test.js")]); run(process.execPath, ["--test", path.join(root, "tools/harmony-ui.test.mjs")]); const prefix = path.join(plugin, "dist/native-deps/install"); const bridgeTest = path.join(plugin, "dist/mtp-bridge-test"); diff --git a/vendor/harmony-connector/libmtp-close-timeout.patch b/vendor/harmony-connector/libmtp-close-timeout.patch new file mode 100644 index 0000000..d812af3 --- /dev/null +++ b/vendor/harmony-connector/libmtp-close-timeout.patch @@ -0,0 +1,12 @@ +--- a/src/libusb1-glue.c ++++ b/src/libusb1-glue.c +@@ -2358,6 +2358,9 @@ + + void close_device (PTP_USB *ptp_usb, PTPParams *params) + { ++ /* Closing is a small command, not a file transfer. A non-responsive ++ * CloseSession must not consume the device's long transfer timeout. */ ++ set_usb_device_timeout(ptp_usb, 2000); + if (ptp_closesession(params)!=PTP_RC_OK) + LIBMTP_ERROR("ERROR: Could not close session!\n"); + close_usb(ptp_usb);