feat(harmony): add native H.264 mirroring window with direct touch control

tools/fixtures/harmony-ui.mjs

@@ -61,42 +61,7 @@ const entries = [
     modified: 1788998400,
   },
 ];
-let screenId = 0,
-  screenSequence = 0,
-  screenControl = false;
-let lastInput = "等待操作";
-function screenFrame() {
-  if (!attached) throw new Error("设备已断开（模拟）");
-  const canvas = document.createElement("canvas");
-  canvas.width = 600;
-  canvas.height = 1200;
-  const context = canvas.getContext("2d");
-  const gradient = context.createLinearGradient(0, 0, 600, 1200);
-  gradient.addColorStop(0, "#183958");
-  gradient.addColorStop(1, "#3d1d52");
-  context.fillStyle = gradient;
-  context.fillRect(0, 0, 600, 1200);
-  context.fillStyle = "#fff";
-  context.textAlign = "center";
-  context.font = "bold 70px sans-serif";
-  context.fillText("Harmony", 300, 260);
-  context.font = "26px sans-serif";
-  context.fillText("投屏界面预览 · 模拟画面", 300, 320);
-  context.fillStyle = "#e9dcff";
-  context.fillText(lastInput, 300, 600, 550);
-  context.fillText(`画面 ${++screenSequence}`, 300, 670);
-  context.fillStyle = "#fff";
-  context.fillRect(225, 1150, 150, 6);
-  return {
-    dataUrl: canvas.toDataURL("image/jpeg", 0.8),
-    width: 600,
-    height: 1200,
-    geometry: 1,
-    sequence: screenSequence,
-    captureMs: 180,
-    control: screenControl,
-  };
-}
+let screenActive = false;
 window.__TROVE_CONTEXT__ = {
   curioId: "dev.trove.harmony-connector",
   version: "0.3.0",
@@ -104,37 +69,9 @@ window.__TROVE_CONTEXT__ = {
   platform: "macos",
 };
 window.__TROVE_BRIDGE__ = createMockBridge({
-  "@self.hdc.screenStart": () => {
-    screenControl = false;
-    return {
-      session: ++screenId,
-      control: false,
-      inputCapabilities: [
-        "click",
-        "doubleClick",
-        "longClick",
-        "swipe",
-        "drag",
-        "text",
-        "keyEvent",
-      ],
-    };
-  },
-  "@self.hdc.screenFrame": screenFrame,
-  "@self.hdc.screenStop": () => ({ stopped: true }),
-  "@self.hdc.screenControl": ({ enabled }) => ({
-    control: (screenControl = enabled),
-  }),
-  "@self.hdc.screenInput": ({ input }) => {
-    lastInput =
-      input.kind === "text"
-        ? input.text
-        : input.kind === "key"
-          ? input.key
-          : `${input.kind} · ${input.x}, ${input.y}`;
-    return { executed: true };
-  },
-  "@self.hdc.screenSave": () => ({ path: "/ui-fixture/screen.jpg" }),
+  "@self.hdc.screenWindowOpen": () => ({ active: (screenActive = true) }),
+  "@self.hdc.screenWindowStop": () => ({ active: (screenActive = false) }),
+  "@self.hdc.screenWindowStatus": () => ({ active: screenActive }),
   "@self.hdc.status": () => ({ busy: false, job: null }),
   "@self.hdc.tool": () => ({
     available: true,

tools/harmony-ui.test.mjs

@@ -53,13 +53,10 @@ async function app(t) {
     job: null,
     busy: false,
     monitorWait: null,
-    screenSession: 0,
-    screenSequence: 0,
-    screenGeometry: 1,
-    screenControl: false,
-    screenFrameWait: null,
-    screenFrameError: null,
-    screenUnchanged: false,
+    screenActive: false,
+    screenTarget: "",
+    screenError: null,
+    screenOpenWait: null,
     openedUrls: [],
     selectedDirectory: "/fixture/folder",
     configureError: null,
@@ -83,53 +80,15 @@ async function app(t) {
   };
   window.__TROVE_BRIDGE__ = createMockBridge(
     Object.fromEntries([
-      handler("hdc.screenStart", () => ({
-        session: ++model.screenSession,
-        inputCapabilities: [
-          "click",
-          "doubleClick",
-          "longClick",
-          "swipe",
-          "drag",
-          "text",
-          "keyEvent",
-        ],
-        control: false,
-      })),
-      handler("hdc.screenStop", () => {
-        model.screenControl = false;
-        return { stopped: true };
+      handler("hdc.screenWindowOpen", async ({target}) => {
+        if (model.screenOpenWait) await model.screenOpenWait;
+        if (model.screenError) throw model.screenError;
+        model.screenActive = true;
+        model.screenTarget = target;
+        return { active: true, target };
       }),
-      handler("hdc.screenControl", ({ enabled }) => ({
-        control: (model.screenControl = enabled),
-      })),
-      handler("hdc.screenFrame", async () => {
-        if (model.screenFrameWait) await model.screenFrameWait;
-        if (model.screenFrameError) throw model.screenFrameError;
-        if (model.screenUnchanged && model.screenSequence)
-          return {
-            unchanged: true,
-            width: 1200,
-            height: 2400,
-            sequence: model.screenSequence,
-            geometry: model.screenGeometry,
-            control: model.screenControl,
-            receivedFps: 0,
-            mbps: 0,
-          };
-        return {
-          dataUrl: "data:image/jpeg;base64,/9j/2Q==",
-          width: 1200,
-          height: 2400,
-          receivedFps: 27,
-          mbps: 3.2,
-          sequence: ++model.screenSequence,
-          geometry: model.screenGeometry,
-          control: model.screenControl,
-        };
-      }),
-      handler("hdc.screenInput", () => ({ executed: true })),
-      handler("hdc.screenSave", () => ({ path: "/fixture/screen.jpg" })),
+      handler("hdc.screenWindowStop", () => ({ active: (model.screenActive = false) })),
+      handler("hdc.screenWindowStatus", () => ({ active: model.screenActive, target: model.screenTarget, error: model.screenError })),
       handler("hdc.status", () => ({ busy: false, job: null })),
       handler("hdc.tool", () => ({
         available: model.hdcAvailable,
@@ -486,161 +445,62 @@ test("device disconnect clears app details and prevents stale target operations"
   assert.equal($("app-launch").disabled, true);
 });
 
-test("screen viewing starts read-only, keeps refreshing and reports stream throughput", async (t) => {
-  const { $, click, tick, model } = await app(t);
+test("screen opens a separate window with default pointer control", async (t) => {
+  const { $, click, model } = await app(t);
   model.hdcAvailable = true;
   await click("tab-management");
+  await click("screen-start");
+  assert.equal(model.screenActive, true);
+  assert.equal($("screen-start").textContent, "显示投屏窗口");
+  assert.match($("screen-status").textContent, /默认鼠标控制/);
+  assert.equal($("screen-stop").disabled, false);
+  assert.ok(!model.calls.includes("hdc.screenFrame"));
+});
+
+test("switching management tabs and hiding the page leaves the independent window open", async (t) => {
+  const { click, model, tick } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await click("tab-files");
+  window.document.dispatchEvent(new window.Event("visibilitychange"));
+  await tick();
+  assert.equal(model.screenActive, true);
+  assert.ok(!model.calls.includes("hdc.screenWindowStop"));
+});
+
+test("explicitly stopping a window permits reconnecting without a USB replug", async (t) => {
+  const { $, click, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await click("screen-stop");
+  assert.equal(model.screenActive, false);
   assert.equal($("screen-start").disabled, false);
   await click("screen-start");
-  await tick();
-  assert.equal($("screen-image").hidden, false);
-  assert.match($("screen-status").textContent, /仅观看/);
-  assert.match(
-    $("screen-stats").textContent,
-    /1200 × 2400.*接收 27 帧\/秒.*3.2 Mbps/,
-  );
-  assert.equal($("screen-control").checked, false);
-  assert.equal($("app-install").disabled, true);
-  const before = model.screenSequence;
-  await tick();
-  assert.ok(model.screenSequence > before);
-  assert.ok(!model.calls.includes("hdc.screenInput"));
+  assert.equal(model.screenActive, true);
 });
 
-test("screen text and navigation require enabled control and use the displayed session", async (t) => {
-  const { $, click, tick, model } = await app(t);
+test("closing the native window is reflected by the launcher", async (t) => {
+  const { $, click, model, tick } = await app(t);
   model.hdcAvailable = true;
   await click("tab-management");
   await click("screen-start");
+  model.screenActive = false;
   await tick();
-  await click("screen-control");
-  $("screen-text").value = "你好，daniel's phone";
-  $("screen-text").dispatchEvent(new window.Event("input"));
-  await click("screen-send-text");
-  await tick();
-  const sent = model.parameters.find(([name]) => name === "hdc.screenInput")[1];
-  assert.deepEqual(sent, {
-    session: 1,
-    sequence: 1,
-    geometry: 1,
-    input: { kind: "text", text: "你好，daniel's phone" },
-  });
-  await click("screen-stop");
-  assert.equal($("screen-image").hidden, true);
-  assert.equal($("screen-image").hasAttribute("src"), false);
-  assert.equal($("screen-text").disabled, true);
-  assert.equal($("app-install").disabled, false);
+  assert.equal($("screen-stop").disabled, true);
+  assert.equal($("screen-start").textContent, "打开投屏窗口");
 });
 
-test("an unchanged live screen preserves its image and does not count repeated frames", async (t) => {
-  const { $, click, tick, model } = await app(t);
+test("a failed window start leaves the launcher available to retry", async (t) => {
+  const { $, click, model } = await app(t);
   model.hdcAvailable = true;
   await click("tab-management");
+  model.screenError = new Error("视频连接失败");
   await click("screen-start");
-  await tick();
-  await click("screen-control");
-  const source = $("screen-image").src;
-  const sequence = model.screenSequence;
-  model.screenUnchanged = true;
-  await tick();
-  await tick();
-  assert.equal($("screen-image").src, source);
-  assert.equal($("screen-image").hidden, false);
-  assert.equal(model.screenSequence, sequence);
-  assert.equal($("screen-control").checked, true);
-  assert.match($("screen-stats").textContent, /接收 0 帧\/秒/);
-  assert.ok(!model.calls.includes("hdc.screenStop"));
-});
-
-test("control does not wait behind a pending video frame", async (t) => {
-  const { $, click, tick, model } = await app(t);
-  model.hdcAvailable = true;
-  await click("tab-management");
+  assert.equal($("screen-start").disabled, false);
+  assert.equal($("screen-stop").disabled, true);
+  model.screenError = null;
   await click("screen-start");
-  await tick();
-  await click("screen-control");
-  let release;
-  model.screenFrameWait = new Promise((resolve) => {
-    release = resolve;
-  });
-  await tick();
-  $("screen-inputs").querySelector('[data-screen-key="home"]').click();
-  await settle();
-  assert.ok(
-    model.parameters.some(
-      ([name, params]) =>
-        name === "hdc.screenInput" && params.input.key === "home",
-    ),
-  );
-  release();
-  model.screenFrameWait = null;
-  await settle();
-  await click("screen-stop");
-});
-
-test("stopping during a pending frame discards that response and permits a fresh screen session", async (t) => {
-  const { $, click, tick, model } = await app(t);
-  model.hdcAvailable = true;
-  await click("tab-management");
-  let release;
-  model.screenFrameWait = new Promise((resolve) => {
-    release = resolve;
-  });
-  await click("screen-start");
-  await tick();
-  await click("screen-stop");
-  assert.equal($("screen-image").hidden, true);
-  assert.ok(model.calls.includes("hdc.screenStop"));
-  release();
-  model.screenFrameWait = null;
-  await settle();
-  assert.equal($("screen-image").hasAttribute("src"), false);
-  await click("screen-start");
-  await tick();
-  assert.equal(model.screenSession, 2);
-  assert.equal($("screen-image").hidden, false);
-});
-
-test("capture failure clears the screen and queued input instead of showing an old live frame", async (t) => {
-  const { $, click, tick, model } = await app(t);
-  model.hdcAvailable = true;
-  await click("tab-management");
-  await click("screen-start");
-  await tick();
-  model.screenFrameError = new Error("Device disconnected");
-  await tick();
-  assert.equal($("screen-image").hidden, true);
-  assert.match($("screen-status").textContent, /未开始/);
-  assert.equal($("screen-control").checked, false);
-});
-
-test("leaving management stops refresh and control; returning does not resume them", async (t) => {
-  const { $, click, tick, model } = await app(t);
-  model.hdcAvailable = true;
-  await click("tab-management");
-  await click("screen-start");
-  await tick();
-  await click("screen-control");
-  await click("tab-files");
-  const before = model.screenSequence;
-  await tick();
-  await click("tab-management");
-  await tick();
-  assert.equal(model.screenSequence, before);
-  assert.equal($("screen-image").hidden, true);
-  assert.equal($("screen-control").checked, false);
-});
-
-test("rotation disables control and clears a gesture waiting to be sent", async (t) => {
-  const { $, click, tick, model } = await app(t);
-  model.hdcAvailable = true;
-  await click("tab-management");
-  await click("screen-start");
-  await tick();
-  await click("screen-control");
-  model.screenGeometry++;
-  model.screenControl = false;
-  await tick();
-  assert.equal($("screen-control").checked, false);
-  assert.match($("screen-input-status").textContent, /方向已变化/);
+  assert.equal(model.screenActive, true);
 });