feat(harmony): stream screens over HDC without a companion app

Replace per-frame file capture with the unchanged official Hypium agent, bounded JPEG decoding, independent control RPC and scoped session cleanup. Remove the HAP build and account-signing experiment. All 85 automated checks pass; moving-screen performance and integrated phone input remain unverified because the USB device went offline. Keep published 0.3.2 artifacts unchanged.

tools/harmony-stream.py

@@ -1,233 +0,0 @@
-#!/usr/bin/env python3
-"""Measure the companion's HCS1 video stream; no screenshots/files are retained.
-
-Development tool, not the plugin's production player. Requires official HDC,
-the signed companion on the phone, and ffmpeg (or ffplay with --view).
-"""
-import argparse
-import json
-import os
-from pathlib import Path
-import re
-import secrets
-import select
-import shutil
-import socket
-import struct
-import subprocess
-import threading
-import time
-
-BUNDLE = "ink.render.trove.harmonyconnector"
-MAX_PACKET = 2 * 1024 * 1024
-
-
-def read_exact(sock, size, deadline):
-    result = bytearray()
-    while len(result) < size:
-        remaining = deadline - time.monotonic()
-        if remaining <= 0:
-            raise TimeoutError("Video stream timed out")
-        sock.settimeout(min(remaining, 0.2))
-        try:
-            part = sock.recv(size - len(result))
-        except socket.timeout:
-            continue
-        if not part:
-            raise EOFError("Screen sharing stopped or USB disconnected")
-        result.extend(part)
-    return bytes(result)
-
-
-def read_hello(sock, deadline):
-    magic, width, height, display_width, display_height, fps = struct.unpack(
-        ">4s5I", read_exact(sock, 24, deadline))
-    if (magic != b"HCS1" or not 1 <= fps <= 60 or
-            not all(2 <= d <= 8192 for d in (width, height, display_width, display_height)) or
-            width % 2 or height % 2 or max(width, height) > 1280):
-        raise ValueError("Unsupported stream header")
-    return {"width": width, "height": height, "displayWidth": display_width,
-            "displayHeight": display_height, "configuredFps": fps}
-
-
-def read_packet(sock, deadline):
-    size, flags, pts = struct.unpack(
-        ">IIq", read_exact(sock, 16, deadline))
-    if not 0 < size <= MAX_PACKET or flags & ~0x7f or pts < 0:
-        raise ValueError("Invalid video packet")
-    data = read_exact(sock, size, deadline)
-    if not data.startswith((b"\0\0\1", b"\0\0\0\1")):
-        raise ValueError("Encoder did not return Annex B H.264")
-    return flags, pts, data
-
-
-class Decoder:
-    def __init__(self, view):
-        name = "ffplay" if view else "ffmpeg"
-        executable = shutil.which(name)
-        if not executable:
-            raise RuntimeError(f"Development measurement requires {name} on PATH")
-        args = [executable, "-hide_banner", "-loglevel", "error",
-                "-flags", "low_delay", "-probesize", "32", "-analyzeduration", "0",
-                "-f", "h264", "-i", "pipe:0"]
-        if view:
-            args += ["-window_title", "Harmony Connector · H.264 preview", "-autoexit"]
-        else:
-            args += ["-progress", "pipe:1", "-stats_period", "0.25", "-f", "null", "-"]
-        self.frames = 0
-        self.error = bytearray()
-        self.process = subprocess.Popen(args, stdin=subprocess.PIPE,
-                                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        os.set_blocking(self.process.stdin.fileno(), False)
-        self.threads = [threading.Thread(target=self.progress, daemon=True),
-                        threading.Thread(target=self.errors, daemon=True)]
-        for thread in self.threads:
-            thread.start()
-
-    def progress(self):
-        for line in self.process.stdout:
-            if line.startswith(b"frame="):
-                self.frames = int(line.split(b"=", 1)[1])
-
-    def errors(self):
-        for line in self.process.stderr:
-            if len(self.error) < 8192:
-                self.error.extend(line[:8192 - len(self.error)])
-
-    def write(self, data):
-        deadline = time.monotonic() + 1
-        view = memoryview(data)
-        while view:
-            if self.process.poll() is not None:
-                raise RuntimeError("Video decoder exited: " + self.error.decode(errors="replace"))
-            if time.monotonic() >= deadline:
-                raise TimeoutError("Decoder is falling behind; stopped instead of queuing video")
-            if not select.select([], [self.process.stdin], [], 0.05)[1]:
-                continue
-            try:
-                count = os.write(self.process.stdin.fileno(), view)
-                view = view[count:]
-            except BlockingIOError:
-                continue
-
-    def close(self):
-        self.process.stdin.close()
-        try:
-            self.process.wait(timeout=3)
-        except subprocess.TimeoutExpired:
-            self.process.terminate()
-            try:
-                self.process.wait(timeout=1)
-            except subprocess.TimeoutExpired:
-                self.process.kill()
-                self.process.wait()
-        for thread in self.threads:
-            thread.join(timeout=1)
-        self.process.stdout.close()
-        self.process.stderr.close()
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--hdc", required=True, type=Path)
-    parser.add_argument("--target")
-    parser.add_argument("--seconds", type=int, default=20)
-    parser.add_argument("--view", action="store_true")
-    args = parser.parse_args()
-    if not 5 <= args.seconds <= 120 or not args.hdc.is_file():
-        parser.error("Use an existing HDC executable and 5–120 measurement seconds")
-
-    def hdc(*command):
-        try:
-            result = subprocess.run([str(args.hdc.resolve()), *command], capture_output=True,
-                                    text=True, timeout=10, check=False)
-        except subprocess.TimeoutExpired:
-            raise RuntimeError("HDC command timed out") from None
-        if result.returncode:
-            raise RuntimeError("HDC command did not complete successfully")
-        if re.search(r"\b(fail(?:ed)?|error)\b", result.stdout + result.stderr, re.I):
-            raise RuntimeError("HDC command failed: " + result.stdout + result.stderr)
-        return result.stdout
-
-    targets = [line.split()[0] for line in hdc("list", "targets", "-v").splitlines()
-               if "Connected" in line.split()]
-    target = args.target or (targets[0] if len(targets) == 1 else None)
-    if target not in targets:
-        raise RuntimeError("Select one connected, authorized phone with --target")
-    installed = hdc("-t", target, "shell", f"bm dump -n {BUNDLE}")
-    if BUNDLE not in installed:
-        raise RuntimeError("Install the signed Harmony Connector companion before measuring")
-    decoder = Decoder(args.view)
-    sock = None
-    forward = False
-    with socket.socket() as reservation:
-        reservation.bind(("127.0.0.1", 0))
-        local_port = reservation.getsockname()[1]
-    remote_port = 20000 + secrets.randbelow(40000)
-    local, remote = f"tcp:{local_port}", f"tcp:{remote_port}"
-    token = secrets.token_hex(32)
-    frames = total_bytes = 0
-    started = first_frame = None
-    try:
-        hdc("-t", target, "fport", local, remote)
-        forward = True
-        hdc("-t", target, "shell", f"aa start -b {BUNDLE} -a EntryAbility -m entry "
-            f"--ps token {token} --pi port {remote_port}")
-        print("请在手机点击「共享屏幕」，并同意系统授权。", flush=True)
-        deadline = time.monotonic() + 90
-        while time.monotonic() < deadline:
-            try:
-                sock = socket.create_connection(("127.0.0.1", local_port), timeout=1)
-                sock.sendall(token.encode("ascii"))
-                hello = read_hello(sock, min(deadline, time.monotonic() + 2))
-                break
-            except (OSError, EOFError, TimeoutError):
-                if sock:
-                    sock.close()
-                sock = None
-                time.sleep(0.2)
-        if not sock:
-            raise TimeoutError("Phone did not start screen sharing")
-        print(json.dumps({"stage": "connected", **hello}), flush=True)
-        started = time.monotonic()
-        last_report = started
-        while first_frame is None or time.monotonic() - first_frame < args.seconds:
-            flags, _, data = read_packet(sock, time.monotonic() + (45 if first_frame is None else 3))
-            decoder.write(data)
-            total_bytes += len(data)
-            now = time.monotonic()
-            if not flags & 8:
-                frames += 1
-                if first_frame is None:
-                    first_frame = now
-                    print(json.dumps({"stage": "first-frame", "waitMs": round((now - started) * 1000)}), flush=True)
-            if first_frame and now - last_report >= 1:
-                elapsed = now - first_frame
-                print(json.dumps({"stage": "receiving", "frames": frames,
-                    "receivedFps": round((frames - 1) / elapsed, 1),
-                    "decodedFrames": decoder.frames,
-                    "mbps": round(total_bytes * 8 / elapsed / 1000000, 2)}), flush=True)
-                last_report = now
-        elapsed = time.monotonic() - first_frame
-    finally:
-        if sock:
-            try:
-                sock.sendall(b"\0")
-            except OSError:
-                pass
-            sock.close()
-        try:
-            if forward:
-                hdc("-t", target, "fport", "rm", local, remote)
-        finally:
-            decoder.close()
-    passed = (frames - 1) / elapsed >= 20 and (args.view or decoder.frames >= frames * 0.95)
-    print(json.dumps({"passed": passed, "frames": frames, "decodedFrames": decoder.frames,
-        "receivedFps": round((frames - 1) / elapsed, 1), "durationSeconds": round(elapsed, 2),
-        "endToEndLatencyMeasured": False, "inputSent": False, "screenFilesSaved": False}), flush=True)
-    if not passed:
-        raise SystemExit(1)
-
-
-if __name__ == "__main__":
-    main()

tools/harmony-stream.test.py

@@ -1,87 +0,0 @@
-import importlib.util
-from pathlib import Path
-import re
-import shutil
-import socket
-import struct
-import subprocess
-import time
-import unittest
-
-spec = importlib.util.spec_from_file_location("harmony_stream", Path(__file__).with_name("harmony-stream.py"))
-stream = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(stream)
-
-
-class StreamProtocolTests(unittest.TestCase):
-    def setUp(self):
-        self.sender, self.receiver = socket.socketpair()
-        self.deadline = time.monotonic() + 1
-
-    def tearDown(self):
-        self.sender.close()
-        self.receiver.close()
-
-    def test_negotiated_dimensions(self):
-        self.sender.sendall(struct.pack(">4s5I", b"HCS1", 566, 1280, 1260, 2844, 30))
-        hello = stream.read_hello(self.receiver, self.deadline)
-        self.assertEqual((hello["width"], hello["displayHeight"], hello["configuredFps"]), (566, 2844, 30))
-
-    def test_unknown_version_rejected(self):
-        self.sender.sendall(struct.pack(">4s5I", b"HCS2", 640, 480, 1280, 960, 30))
-        with self.assertRaises(ValueError):
-            stream.read_hello(self.receiver, self.deadline)
-
-    def test_dimensions_bounded_before_video_allocations(self):
-        self.sender.sendall(struct.pack(">4s5I", b"HCS1", 65536, 480, 1280, 960, 30))
-        with self.assertRaises(ValueError):
-            stream.read_hello(self.receiver, self.deadline)
-
-    def test_oversized_packet_rejected_without_reading_payload(self):
-        self.sender.sendall(struct.pack(">IIq", stream.MAX_PACKET + 1, 2, 0))
-        with self.assertRaises(ValueError):
-            stream.read_packet(self.receiver, self.deadline)
-
-    def test_disconnect_in_packet_is_not_a_frame(self):
-        self.sender.sendall(struct.pack(">IIq", 32, 2, 42) + b"\0\0\1\x65")
-        self.sender.shutdown(socket.SHUT_WR)
-        with self.assertRaises(EOFError):
-            stream.read_packet(self.receiver, self.deadline)
-
-    def test_silent_connection_has_bounded_wait(self):
-        with self.assertRaises(TimeoutError):
-            stream.read_packet(self.receiver, time.monotonic() + 0.02)
-
-    def test_non_h264_payload_rejected(self):
-        self.sender.sendall(struct.pack(">IIq", 8, 2, 42) + b"JPEGdata")
-        with self.assertRaises(ValueError):
-            stream.read_packet(self.receiver, self.deadline)
-
-    @unittest.skipUnless(shutil.which("ffmpeg"), "ffmpeg required for actual decoder validation")
-    def test_twenty_generated_frames_survive_transport_and_decode(self):
-        encoded = subprocess.run([
-            "ffmpeg", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i",
-            "testsrc2=size=160x96:rate=30", "-frames:v", "20", "-c:v", "libx264",
-            "-preset", "ultrafast", "-tune", "zerolatency", "-profile:v", "baseline",
-            "-g", "10", "-bf", "0", "-x264-params", "aud=1:repeat-headers=1",
-            "-f", "h264", "pipe:1"], capture_output=True, check=True, timeout=10).stdout
-        # Access unit delimiters separate independently timestamped encoder callbacks.
-        boundaries = [match.start() for match in re.finditer(b"\x00\x00\x00\x01\x09", encoded)]
-        self.assertEqual(len(boundaries), 20)
-        decoder = stream.Decoder(False)
-        try:
-            for index, (start, end) in enumerate(zip(boundaries, boundaries[1:] + [len(encoded)])):
-                data = encoded[start:end]
-                flags = 2 if index % 10 == 0 else 0
-                self.sender.sendall(struct.pack(">IIq", len(data), flags, index * 33333) + data)
-                actual_flags, pts, actual_data = stream.read_packet(self.receiver, time.monotonic() + 1)
-                self.assertEqual((actual_flags, pts), (flags, index * 33333))
-                decoder.write(actual_data)
-        finally:
-            decoder.close()
-        self.assertEqual(decoder.process.returncode, 0, decoder.error.decode())
-        self.assertEqual(decoder.frames, 20, decoder.error.decode())
-
-
-if __name__ == "__main__":
-    unittest.main()

tools/harmony-ui.test.mjs

@@ -59,6 +59,7 @@ async function app(t) {
     screenControl: false,
     screenFrameWait: null,
     screenFrameError: null,
+    screenUnchanged: false,
     openedUrls: [],
     selectedDirectory: "/fixture/folder",
     configureError: null,
@@ -105,11 +106,23 @@ async function app(t) {
       handler("hdc.screenFrame", async () => {
         if (model.screenFrameWait) await model.screenFrameWait;
         if (model.screenFrameError) throw model.screenFrameError;
+        if (model.screenUnchanged && model.screenSequence)
+          return {
+            unchanged: true,
+            width: 1200,
+            height: 2400,
+            sequence: model.screenSequence,
+            geometry: model.screenGeometry,
+            control: model.screenControl,
+            receivedFps: 0,
+            mbps: 0,
+          };
         return {
           dataUrl: "data:image/jpeg;base64,/9j/2Q==",
           width: 1200,
           height: 2400,
-          captureMs: 300,
+          receivedFps: 27,
+          mbps: 3.2,
           sequence: ++model.screenSequence,
           geometry: model.screenGeometry,
           control: model.screenControl,
@@ -473,7 +486,7 @@ test("device disconnect clears app details and prevents stale target operations"
   assert.equal($("app-launch").disabled, true);
 });
 
-test("screen viewing starts read-only, keeps refreshing and reports measured capture time", async (t) => {
+test("screen viewing starts read-only, keeps refreshing and reports stream throughput", async (t) => {
   const { $, click, tick, model } = await app(t);
   model.hdcAvailable = true;
   await click("tab-management");
@@ -482,7 +495,10 @@ test("screen viewing starts read-only, keeps refreshing and reports measured cap
   await tick();
   assert.equal($("screen-image").hidden, false);
   assert.match($("screen-status").textContent, /仅观看/);
-  assert.match($("screen-stats").textContent, /1200 × 2400.*300 ms/);
+  assert.match(
+    $("screen-stats").textContent,
+    /1200 × 2400.*接收 27 帧\/秒.*3.2 Mbps/,
+  );
   assert.equal($("screen-control").checked, false);
   assert.equal($("app-install").disabled, true);
   const before = model.screenSequence;
@@ -516,6 +532,52 @@ test("screen text and navigation require enabled control and use the displayed s
   assert.equal($("app-install").disabled, false);
 });
 
+test("an unchanged live screen preserves its image and does not count repeated frames", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  await click("screen-control");
+  const source = $("screen-image").src;
+  const sequence = model.screenSequence;
+  model.screenUnchanged = true;
+  await tick();
+  await tick();
+  assert.equal($("screen-image").src, source);
+  assert.equal($("screen-image").hidden, false);
+  assert.equal(model.screenSequence, sequence);
+  assert.equal($("screen-control").checked, true);
+  assert.match($("screen-stats").textContent, /接收 0 帧\/秒/);
+  assert.ok(!model.calls.includes("hdc.screenStop"));
+});
+
+test("control does not wait behind a pending video frame", async (t) => {
+  const { $, click, tick, model } = await app(t);
+  model.hdcAvailable = true;
+  await click("tab-management");
+  await click("screen-start");
+  await tick();
+  await click("screen-control");
+  let release;
+  model.screenFrameWait = new Promise((resolve) => {
+    release = resolve;
+  });
+  await tick();
+  $("screen-inputs").querySelector('[data-screen-key="home"]').click();
+  await settle();
+  assert.ok(
+    model.parameters.some(
+      ([name, params]) =>
+        name === "hdc.screenInput" && params.input.key === "home",
+    ),
+  );
+  release();
+  model.screenFrameWait = null;
+  await settle();
+  await click("screen-stop");
+});
+
 test("stopping during a pending frame discards that response and permits a fresh screen session", async (t) => {
   const { $, click, tick, model } = await app(t);
   model.hdcAvailable = true;