feat(harmony): add screen viewing and control with USB reconnection recovery

2026-09-10 14:15:43 +08:00
parent 44d8cf5c92
commit 170ea4d112
26 changed files with 2085 additions and 26 deletions
@@ -53,6 +53,12 @@ async function app(t) {
    job: null,
    busy: false,
    monitorWait: null,
+    screenSession: 0,
+    screenSequence: 0,
+    screenGeometry: 1,
+    screenControl: false,
+    screenFrameWait: null,
+    screenFrameError: null,
  };
  const { createMockBridge } = await import(
    pathToFileURL(path.join(directory, "sdk/index.js"))
@@ -73,6 +79,41 @@ async function app(t) {
  };
  window.__TROVE_BRIDGE__ = createMockBridge(
    Object.fromEntries([
+      handler("hdc.screenStart", () => ({
+        session: ++model.screenSession,
+        inputCapabilities: [
+          "click",
+          "doubleClick",
+          "longClick",
+          "swipe",
+          "drag",
+          "text",
+          "keyEvent",
+        ],
+        control: false,
+      })),
+      handler("hdc.screenStop", () => {
+        model.screenControl = false;
+        return { stopped: true };
+      }),
+      handler("hdc.screenControl", ({ enabled }) => ({
+        control: (model.screenControl = enabled),
+      })),
+      handler("hdc.screenFrame", async () => {
+        if (model.screenFrameWait) await model.screenFrameWait;
+        if (model.screenFrameError) throw model.screenFrameError;
+        return {
+          dataUrl: "data:image/jpeg;base64,/9j/2Q==",
+          width: 1200,
+          height: 2400,
+          captureMs: 300,
+          sequence: ++model.screenSequence,
+          geometry: model.screenGeometry,
+          control: model.screenControl,
+        };
+      }),
+      handler("hdc.screenInput", () => ({ executed: true })),
+      handler("hdc.screenSave", () => ({ path: "/fixture/screen.jpg" })),
      handler("hdc.status", () => ({ busy: false, job: null })),
      handler("hdc.tool", () => ({
        available: model.hdcAvailable,
@@ -365,3 +406,113 @@ test("device disconnect clears app details and prevents stale target operations"
  assert.equal($("app-install").disabled, true);
  assert.equal($("app-launch").disabled, true);
 });
+
+test("screen viewing starts read-only, keeps refreshing and reports measured capture time", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  assert.equal($("screen-start").disabled, false);
+  await click("screen-start");
+  await tick();
+  assert.equal($("screen-image").hidden, false);
+  assert.match($("screen-status").textContent, /仅观看/);
+  assert.match($("screen-stats").textContent, /1200 × 2400.*300 ms/);
+  assert.equal($("screen-control").checked, false);
+  assert.equal($("app-install").disabled, true);
+  const before = model.screenSequence;
+  await tick();
+  assert.ok(model.screenSequence > before);
+  assert.ok(!model.calls.includes("hdc.screenInput"));
+});
+
+test("screen text and navigation require enabled control and use the displayed session", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  await click("screen-control");
+  $("screen-text").value = "你好，daniel's phone";
+  $("screen-text").dispatchEvent(new window.Event("input"));
+  await click("screen-send-text");
+  await tick();
+  const sent = model.parameters.find(([name]) => name === "hdc.screenInput")[1];
+  assert.deepEqual(sent, {
+    session: 1,
+    sequence: 1,
+    geometry: 1,
+    input: { kind: "text", text: "你好，daniel's phone" },
+  });
+  await click("screen-stop");
+  assert.equal($("screen-image").hidden, true);
+  assert.equal($("screen-image").hasAttribute("src"), false);
+  assert.equal($("screen-text").disabled, true);
+  assert.equal($("app-install").disabled, false);
+});
+
+test("stopping during a pending frame discards that response and permits a fresh screen session", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  let release;
+  model.screenFrameWait = new Promise((resolve) => {
+    release = resolve;
+  });
+  await click("screen-start");
+  await tick();
+  await click("screen-stop");
+  assert.equal($("screen-image").hidden, true);
+  assert.ok(model.calls.includes("hdc.screenStop"));
+  release();
+  model.screenFrameWait = null;
+  await settle();
+  assert.equal($("screen-image").hasAttribute("src"), false);
+  await click("screen-start");
+  await tick();
+  assert.equal(model.screenSession, 2);
+  assert.equal($("screen-image").hidden, false);
+});
+
+test("capture failure clears the screen and queued input instead of showing an old live frame", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  model.screenFrameError = new Error("Device disconnected");
+  await tick();
+  assert.equal($("screen-image").hidden, true);
+  assert.match($("screen-status").textContent, /未开始/);
+  assert.equal($("screen-control").checked, false);
+});
+
+test("leaving management stops refresh and control; returning does not resume them", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  await click("screen-control");
+  await click("tab-files");
+  const before = model.screenSequence;
+  await tick();
+  await click("tab-management");
+  await tick();
+  assert.equal(model.screenSequence, before);
+  assert.equal($("screen-image").hidden, true);
+  assert.equal($("screen-control").checked, false);
+});
+
+test("rotation disables control and clears a gesture waiting to be sent", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  await click("screen-control");
+  model.screenGeometry++;
+  model.screenControl = false;
+  await tick();
+  assert.equal($("screen-control").checked, false);
+  assert.match($("screen-input-status").textContent, /方向已变化/);
+});