feat(harmony): add screen viewing and control with USB reconnection recovery

tools/fixtures/harmony-ui.mjs

@@ -61,13 +61,80 @@ const entries = [
     modified: 1788998400,
   },
 ];
+let screenId = 0,
+  screenSequence = 0,
+  screenControl = false;
+let lastInput = "等待操作";
+function screenFrame() {
+  if (!attached) throw new Error("设备已断开（模拟）");
+  const canvas = document.createElement("canvas");
+  canvas.width = 600;
+  canvas.height = 1200;
+  const context = canvas.getContext("2d");
+  const gradient = context.createLinearGradient(0, 0, 600, 1200);
+  gradient.addColorStop(0, "#183958");
+  gradient.addColorStop(1, "#3d1d52");
+  context.fillStyle = gradient;
+  context.fillRect(0, 0, 600, 1200);
+  context.fillStyle = "#fff";
+  context.textAlign = "center";
+  context.font = "bold 70px sans-serif";
+  context.fillText("Harmony", 300, 260);
+  context.font = "26px sans-serif";
+  context.fillText("投屏界面预览 · 模拟画面", 300, 320);
+  context.fillStyle = "#e9dcff";
+  context.fillText(lastInput, 300, 600, 550);
+  context.fillText(`画面 ${++screenSequence}`, 300, 670);
+  context.fillStyle = "#fff";
+  context.fillRect(225, 1150, 150, 6);
+  return {
+    dataUrl: canvas.toDataURL("image/jpeg", 0.8),
+    width: 600,
+    height: 1200,
+    geometry: 1,
+    sequence: screenSequence,
+    captureMs: 180,
+    control: screenControl,
+  };
+}
 window.__TROVE_CONTEXT__ = {
   curioId: "dev.trove.harmony-connector",
-  version: "0.2.0",
+  version: "0.3.0",
   instanceId: "ui-fixture",
   platform: "macos",
 };
 window.__TROVE_BRIDGE__ = createMockBridge({
+  "@self.hdc.screenStart": () => {
+    screenControl = false;
+    return {
+      session: ++screenId,
+      control: false,
+      inputCapabilities: [
+        "click",
+        "doubleClick",
+        "longClick",
+        "swipe",
+        "drag",
+        "text",
+        "keyEvent",
+      ],
+    };
+  },
+  "@self.hdc.screenFrame": screenFrame,
+  "@self.hdc.screenStop": () => ({ stopped: true }),
+  "@self.hdc.screenControl": ({ enabled }) => ({
+    control: (screenControl = enabled),
+  }),
+  "@self.hdc.screenInput": ({ input }) => {
+    lastInput =
+      input.kind === "text"
+        ? input.text
+        : input.kind === "key"
+          ? input.key
+          : `${input.kind} · ${input.x}, ${input.y}`;
+    return { executed: true };
+  },
+  "@self.hdc.screenSave": () => ({ path: "/ui-fixture/screen.jpg" }),
   "@self.hdc.status": () => ({ busy: false, job: null }),
   "@self.hdc.tool": () => ({
     available: true,

tools/harmony-ui.test.mjs

@@ -53,6 +53,12 @@ async function app(t) {
     job: null,
     busy: false,
     monitorWait: null,
+    screenSession: 0,
+    screenSequence: 0,
+    screenGeometry: 1,
+    screenControl: false,
+    screenFrameWait: null,
+    screenFrameError: null,
   };
   const { createMockBridge } = await import(
     pathToFileURL(path.join(directory, "sdk/index.js"))
@@ -73,6 +79,41 @@ async function app(t) {
   };
   window.__TROVE_BRIDGE__ = createMockBridge(
     Object.fromEntries([
+      handler("hdc.screenStart", () => ({
+        session: ++model.screenSession,
+        inputCapabilities: [
+          "click",
+          "doubleClick",
+          "longClick",
+          "swipe",
+          "drag",
+          "text",
+          "keyEvent",
+        ],
+        control: false,
+      })),
+      handler("hdc.screenStop", () => {
+        model.screenControl = false;
+        return { stopped: true };
+      }),
+      handler("hdc.screenControl", ({ enabled }) => ({
+        control: (model.screenControl = enabled),
+      })),
+      handler("hdc.screenFrame", async () => {
+        if (model.screenFrameWait) await model.screenFrameWait;
+        if (model.screenFrameError) throw model.screenFrameError;
+        return {
+          dataUrl: "data:image/jpeg;base64,/9j/2Q==",
+          width: 1200,
+          height: 2400,
+          captureMs: 300,
+          sequence: ++model.screenSequence,
+          geometry: model.screenGeometry,
+          control: model.screenControl,
+        };
+      }),
+      handler("hdc.screenInput", () => ({ executed: true })),
+      handler("hdc.screenSave", () => ({ path: "/fixture/screen.jpg" })),
       handler("hdc.status", () => ({ busy: false, job: null })),
       handler("hdc.tool", () => ({
         available: model.hdcAvailable,
@@ -365,3 +406,113 @@ test("device disconnect clears app details and prevents stale target operations"
   assert.equal($("app-install").disabled, true);
   assert.equal($("app-launch").disabled, true);
 });
+
+test("screen viewing starts read-only, keeps refreshing and reports measured capture time", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  assert.equal($("screen-start").disabled, false);
+  await click("screen-start");
+  await tick();
+  assert.equal($("screen-image").hidden, false);
+  assert.match($("screen-status").textContent, /仅观看/);
+  assert.match($("screen-stats").textContent, /1200 × 2400.*300 ms/);
+  assert.equal($("screen-control").checked, false);
+  assert.equal($("app-install").disabled, true);
+  const before = model.screenSequence;
+  await tick();
+  assert.ok(model.screenSequence > before);
+  assert.ok(!model.calls.includes("hdc.screenInput"));
+});
+
+test("screen text and navigation require enabled control and use the displayed session", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  await click("screen-control");
+  $("screen-text").value = "你好，daniel's phone";
+  $("screen-text").dispatchEvent(new window.Event("input"));
+  await click("screen-send-text");
+  await tick();
+  const sent = model.parameters.find(([name]) => name === "hdc.screenInput")[1];
+  assert.deepEqual(sent, {
+    session: 1,
+    sequence: 1,
+    geometry: 1,
+    input: { kind: "text", text: "你好，daniel's phone" },
+  });
+  await click("screen-stop");
+  assert.equal($("screen-image").hidden, true);
+  assert.equal($("screen-image").hasAttribute("src"), false);
+  assert.equal($("screen-text").disabled, true);
+  assert.equal($("app-install").disabled, false);
+});
+
+test("stopping during a pending frame discards that response and permits a fresh screen session", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  let release;
+  model.screenFrameWait = new Promise((resolve) => {
+    release = resolve;
+  });
+  await click("screen-start");
+  await tick();
+  await click("screen-stop");
+  assert.equal($("screen-image").hidden, true);
+  assert.ok(model.calls.includes("hdc.screenStop"));
+  release();
+  model.screenFrameWait = null;
+  await settle();
+  assert.equal($("screen-image").hasAttribute("src"), false);
+  await click("screen-start");
+  await tick();
+  assert.equal(model.screenSession, 2);
+  assert.equal($("screen-image").hidden, false);
+});
+
+test("capture failure clears the screen and queued input instead of showing an old live frame", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  model.screenFrameError = new Error("Device disconnected");
+  await tick();
+  assert.equal($("screen-image").hidden, true);
+  assert.match($("screen-status").textContent, /未开始/);
+  assert.equal($("screen-control").checked, false);
+});
+
+test("leaving management stops refresh and control; returning does not resume them", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  await click("screen-control");
+  await click("tab-files");
+  const before = model.screenSequence;
+  await tick();
+  await click("tab-management");
+  await tick();
+  assert.equal(model.screenSequence, before);
+  assert.equal($("screen-image").hidden, true);
+  assert.equal($("screen-control").checked, false);
+});
+
+test("rotation disables control and clears a gesture waiting to be sent", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  await click("screen-control");
+  model.screenGeometry++;
+  model.screenControl = false;
+  await tick();
+  assert.equal($("screen-control").checked, false);
+  assert.match($("screen-input-status").textContent, /方向已变化/);
+});

tools/test-harmony.mjs

@@ -19,6 +19,7 @@ run(
   },
 );
 run(process.execPath, ["--test", path.join(plugin, "ui/format.test.js")]);
+run(process.execPath, ["--test", path.join(plugin, "ui/screen-geometry.test.js")]);
 run(process.execPath, ["--test", path.join(root, "tools/harmony-ui.test.mjs")]);
 const prefix = path.join(plugin, "dist/native-deps/install");
 const bridgeTest = path.join(plugin, "dist/mtp-bridge-test");