diff --git a/.gitignore b/.gitignore index c26ba5a..e7c9ba2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ target/ /curios/harmony-connector/legal/*-COPYING /curios/harmony-connector/legal/build.mjs /curios/harmony-connector/legal/rust/ +/curios/harmony-connector/legal/hypium/ +# Keep local credentials from the retired companion experiment private. +/curios/harmony-connector/companion/ diff --git a/curios/harmony-connector/build.mjs b/curios/harmony-connector/build.mjs index 1ba4dcb..6bba945 100644 --- a/curios/harmony-connector/build.mjs +++ b/curios/harmony-connector/build.mjs @@ -94,6 +94,24 @@ for (const source of sources) { await copyFile(patchFile, path.join(plugin, "legal/sources", source.patch)); } if (process.argv.includes("--dependencies-only")) process.exit(0); +const agentDirectory = path.join(root, "vendor/harmony-connector/hypium"); +const agentProvenance = JSON.parse( + await readFile(path.join(agentDirectory, "provenance.json"), "utf8"), +); +const agentBytes = await readFile( + path.join(agentDirectory, "uitest_agent_v1.2.2.so"), +); +if ( + createHash("sha256").update(agentBytes).digest("hex") !== + agentProvenance.sha256 +) + throw new Error("Hypium device agent checksum mismatch"); +await mkdir(path.join(plugin, "legal/hypium"), { recursive: true }); +for (const name of ["LICENSE", "provenance.json"]) + await copyFile( + path.join(agentDirectory, name), + path.join(plugin, "legal/hypium", name), + ); run( "cargo", ["build", "--release", "--locked", "--manifest-path", "native/Cargo.toml"], diff --git a/curios/harmony-connector/companion/.gitignore b/curios/harmony-connector/companion/.gitignore deleted file mode 100644 index ab6f752..0000000 --- a/curios/harmony-connector/companion/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -/node_modules -/oh_modules -/local.properties -/.idea -**/build -/.hvigor -.cxx -/.clangd -/.clang-format -/.clang-tidy -**/.test -/.appanalyzer -# Local signing configuration and credentials -/build-profile.json5 -/signing/ -/material/ -*.p12 -*.p7b -*.cer -*.csr -*.pem -*.key diff --git a/curios/harmony-connector/companion/AppScope/app.json5 b/curios/harmony-connector/companion/AppScope/app.json5 deleted file mode 100644 index 3fc5fb5..0000000 --- a/curios/harmony-connector/companion/AppScope/app.json5 +++ /dev/null @@ -1,10 +0,0 @@ -{ - "app": { - "bundleName": "ink.render.trove.harmonyconnector", - "vendor": "Trove", - "versionCode": 1, - "versionName": "0.1.0", - "icon": "$media:layered_image", - "label": "$string:app_name" - } -} diff --git a/curios/harmony-connector/companion/AppScope/resources/base/element/string.json b/curios/harmony-connector/companion/AppScope/resources/base/element/string.json deleted file mode 100644 index bdd890c..0000000 --- a/curios/harmony-connector/companion/AppScope/resources/base/element/string.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "string": [ - { - "name": "app_name", - "value": "Harmony Connector" - } - ] -} diff --git a/curios/harmony-connector/companion/AppScope/resources/base/media/background.png b/curios/harmony-connector/companion/AppScope/resources/base/media/background.png deleted file mode 100644 index e129e5a..0000000 Binary files a/curios/harmony-connector/companion/AppScope/resources/base/media/background.png and /dev/null differ diff --git a/curios/harmony-connector/companion/AppScope/resources/base/media/foreground.png b/curios/harmony-connector/companion/AppScope/resources/base/media/foreground.png deleted file mode 100644 index e129e5a..0000000 Binary files a/curios/harmony-connector/companion/AppScope/resources/base/media/foreground.png and /dev/null differ diff --git a/curios/harmony-connector/companion/AppScope/resources/base/media/layered_image.json b/curios/harmony-connector/companion/AppScope/resources/base/media/layered_image.json deleted file mode 100644 index fb49920..0000000 --- a/curios/harmony-connector/companion/AppScope/resources/base/media/layered_image.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "layered-image": - { - "background" : "$media:background", - "foreground" : "$media:foreground" - } -} \ No newline at end of file diff --git a/curios/harmony-connector/companion/build-profile.template.json5 b/curios/harmony-connector/companion/build-profile.template.json5 deleted file mode 100644 index ff7128a..0000000 --- a/curios/harmony-connector/companion/build-profile.template.json5 +++ /dev/null @@ -1,41 +0,0 @@ -{ - "app": { - "signingConfigs": [], - "products": [ - { - "name": "default", - "targetSdkVersion": "26.0.0", - "compatibleSdkVersion": "5.0.5(17)", - "runtimeOS": "HarmonyOS", - "buildOption": { - "strictMode": { - "caseSensitiveCheck": true, - "useNormalizedOHMUrl": true - } - } - } - ], - "buildModeSet": [ - { - "name": "debug" - }, - { - "name": "release" - } - ] - }, - "modules": [ - { - "name": "entry", - "srcPath": "./entry", - "targets": [ - { - "name": "default", - "applyToProducts": [ - "default" - ] - } - ] - } - ] -} diff --git a/curios/harmony-connector/companion/code-linter.json5 b/curios/harmony-connector/companion/code-linter.json5 deleted file mode 100644 index 073990f..0000000 --- a/curios/harmony-connector/companion/code-linter.json5 +++ /dev/null @@ -1,32 +0,0 @@ -{ - "files": [ - "**/*.ets" - ], - "ignore": [ - "**/src/ohosTest/**/*", - "**/src/test/**/*", - "**/src/mock/**/*", - "**/node_modules/**/*", - "**/oh_modules/**/*", - "**/build/**/*", - "**/.preview/**/*" - ], - "ruleSet": [ - "plugin:@performance/recommended", - "plugin:@typescript-eslint/recommended" - ], - "rules": { - "@security/no-unsafe-aes": "error", - "@security/no-unsafe-hash": "error", - "@security/no-unsafe-mac": "warn", - "@security/no-unsafe-dh": "error", - "@security/no-unsafe-dsa": "error", - "@security/no-unsafe-ecdsa": "error", - "@security/no-unsafe-rsa-encrypt": "error", - "@security/no-unsafe-rsa-sign": "error", - "@security/no-unsafe-rsa-key": "error", - "@security/no-unsafe-dsa-key": "error", - "@security/no-unsafe-dh-key": "error", - "@security/no-unsafe-3des": "error" - } -} \ No newline at end of file diff --git a/curios/harmony-connector/companion/entry/.gitignore b/curios/harmony-connector/companion/entry/.gitignore deleted file mode 100644 index e2713a2..0000000 --- a/curios/harmony-connector/companion/entry/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -/node_modules -/oh_modules -/.preview -/build -/.cxx -/.test \ No newline at end of file diff --git a/curios/harmony-connector/companion/entry/build-profile.json5 b/curios/harmony-connector/companion/entry/build-profile.json5 deleted file mode 100644 index 65845b1..0000000 --- a/curios/harmony-connector/companion/entry/build-profile.json5 +++ /dev/null @@ -1,38 +0,0 @@ -{ - "apiType": "stageMode", - "buildOption": { - "resOptions": { - "copyCodeResource": { - "enable": false - } - }, - "externalNativeOptions": { - "path": "./src/main/cpp/CMakeLists.txt", - "arguments": "", - "cppFlags": "", - "abiFilters": [ - "arm64-v8a" - ] - } - }, - "buildOptionSet": [ - { - "name": "release", - "arkOptions": { - "obfuscation": { - "ruleOptions": { - "enable": false, - "files": [ - "./obfuscation-rules.txt" - ] - } - } - } - } - ], - "targets": [ - { - "name": "default" - } - ] -} diff --git a/curios/harmony-connector/companion/entry/hvigorfile.ts b/curios/harmony-connector/companion/entry/hvigorfile.ts deleted file mode 100644 index 52c2861..0000000 --- a/curios/harmony-connector/companion/entry/hvigorfile.ts +++ /dev/null @@ -1,7 +0,0 @@ -// @ts-nocheck – Template file, only used when copied into a project directory -import { hapTasks } from '@ohos/hvigor-ohos-plugin'; - -export default { - system: hapTasks /* Built-in plugin of Hvigor. It cannot be modified. */, - plugins: [] /* Custom plugin to extend the functionality of Hvigor. */, -}; diff --git a/curios/harmony-connector/companion/entry/obfuscation-rules.txt b/curios/harmony-connector/companion/entry/obfuscation-rules.txt deleted file mode 100644 index ded9d40..0000000 --- a/curios/harmony-connector/companion/entry/obfuscation-rules.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Define project specific obfuscation rules here. -# You can include the obfuscation configuration files in the current module's build-profile.json5. - -# Obfuscation options: -# -disable-obfuscation: disable all obfuscations -# -enable-property-obfuscation: obfuscate the property names -# -enable-toplevel-obfuscation: obfuscate the names in the global scope -# -compact: remove unnecessary blank spaces and all line feeds -# -remove-log: remove all console.* statements -# -print-namecache: print the name cache that contains the mapping from the old names to new names -# -apply-namecache: reuse the given cache file - -# Keep options: -# -keep-property-name: specifies property names that you want to keep -# -keep-global-name: specifies names that you want to keep in the global scope - --enable-property-obfuscation --enable-toplevel-obfuscation --enable-filename-obfuscation --enable-export-obfuscation \ No newline at end of file diff --git a/curios/harmony-connector/companion/entry/oh-package-lock.json5 b/curios/harmony-connector/companion/entry/oh-package-lock.json5 deleted file mode 100644 index efe2f1c..0000000 --- a/curios/harmony-connector/companion/entry/oh-package-lock.json5 +++ /dev/null @@ -1,19 +0,0 @@ -{ - "meta": { - "stableOrder": true, - "enableUnifiedLockfile": false - }, - "lockfileVersion": 3, - "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.", - "specifiers": { - "libharmony_stream.so@src/main/cpp/types/libharmony_stream": "libharmony_stream.so@src/main/cpp/types/libharmony_stream" - }, - "packages": { - "libharmony_stream.so@src/main/cpp/types/libharmony_stream": { - "name": "libharmony_stream.so", - "version": "0.1.0", - "resolved": "src/main/cpp/types/libharmony_stream", - "registryType": "local" - } - } -} \ No newline at end of file diff --git a/curios/harmony-connector/companion/entry/oh-package.json5 b/curios/harmony-connector/companion/entry/oh-package.json5 deleted file mode 100644 index dbf40ee..0000000 --- a/curios/harmony-connector/companion/entry/oh-package.json5 +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "entry", - "version": "1.0.0", - "description": "System-authorized H.264 screen sharing", - "main": "", - "author": "Trove", - "dependencies": { - "libharmony_stream.so": "file:./src/main/cpp/types/libharmony_stream" - } -} diff --git a/curios/harmony-connector/companion/entry/src/main/cpp/CMakeLists.txt b/curios/harmony-connector/companion/entry/src/main/cpp/CMakeLists.txt deleted file mode 100644 index e0beb1e..0000000 --- a/curios/harmony-connector/companion/entry/src/main/cpp/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -cmake_minimum_required(VERSION 3.5.0) -project(harmony_stream) -set(CMAKE_CXX_STANDARD 17) -add_library(harmony_stream SHARED stream.cpp) -target_compile_options(harmony_stream PRIVATE -Wall -Wextra) -target_link_libraries(harmony_stream PUBLIC libace_napi.z.so - libnative_avscreen_capture.so libnative_media_venc.so - libnative_media_codecbase.so libnative_media_core.so libnative_window.so) diff --git a/curios/harmony-connector/companion/entry/src/main/cpp/stream.cpp b/curios/harmony-connector/companion/entry/src/main/cpp/stream.cpp deleted file mode 100644 index 5804d6b..0000000 --- a/curios/harmony-connector/companion/entry/src/main/cpp/stream.cpp +++ /dev/null @@ -1,354 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { -using Clock = std::chrono::steady_clock; -constexpr size_t MAX_PACKET = 2 * 1024 * 1024; -constexpr size_t MAX_QUEUE = 4 * 1024 * 1024; -constexpr int FPS = 30; - -struct Packet { - std::vector data; - uint32_t flags; - int64_t pts; -}; - -void put32(std::vector &out, uint32_t value) -{ - for (int shift = 24; shift >= 0; shift -= 8) out.push_back(value >> shift); -} - -// One worker owns all media resources and sockets. Callbacks only copy into a -// bounded queue; slow consumers terminate the session instead of accumulating delay. -class Stream { -public: - ~Stream() { stop(); } - void start(int width, int height, int port, std::string token) - { - stop(); - cancelled = false; - activated = false; - frames = 0; - bytes = 0; - { - std::lock_guard lock(mutex); - error.clear(); - state = "waiting"; - packets.clear(); - queuedBytes = 0; - } - worker = std::thread([=] { run(width, height, port, token); }); - } - void stop() - { - cancelled = true; - if (worker.joinable()) worker.join(); - } - napi_value status(napi_env env) - { - std::lock_guard lock(mutex); - napi_value result, value; - napi_create_object(env, &result); - for (auto item : {std::make_pair("state", state), std::make_pair("error", error)}) { - napi_create_string_utf8(env, item.second.c_str(), NAPI_AUTO_LENGTH, &value); - napi_set_named_property(env, result, item.first, value); - } - napi_create_double(env, static_cast(frames.load()), &value); - napi_set_named_property(env, result, "frames", value); - napi_create_double(env, static_cast(bytes.load()), &value); - napi_set_named_property(env, result, "bytes", value); - return result; - } - -private: - std::atomic cancelled{true}; - std::atomic activated{false}; - std::atomic frames{0}, bytes{0}; - std::mutex mutex; - std::thread worker; - std::string state = "stopped", error; - std::deque packets; - size_t queuedBytes = 0; - - void fail(const std::string &message) - { - std::lock_guard lock(mutex); - if (error.empty()) error = message; - cancelled = true; - } - bool check(int code, const char *operation) - { - if (!code) return true; - fail(std::string(operation) + " (" + std::to_string(code) + ")"); - return false; - } - static void captureState(OH_AVScreenCapture *, OH_AVScreenCaptureStateCode code, void *context) - { - auto &self = *static_cast(context); - if (code == OH_SCREEN_CAPTURE_STATE_STARTED) { - self.activated = true; - std::lock_guard lock(self.mutex); - self.state = "sharing"; - } else if (code == OH_SCREEN_CAPTURE_STATE_CANCELED) { - self.fail("手机未同意共享屏幕。"); - } else if (code == OH_SCREEN_CAPTURE_STATE_STOPPED_BY_USER || - code == OH_SCREEN_CAPTURE_STATE_INTERRUPTED_BY_OTHER || - code == OH_SCREEN_CAPTURE_STATE_STOPPED_BY_CALL || - code == OH_SCREEN_CAPTURE_STATE_STOPPED_BY_USER_SWITCHES) { - self.cancelled = true; - } - // Private scenes remain masked by the system; never substitute an old frame. - } - static void captureError(OH_AVScreenCapture *, int32_t code, void *context) - { - static_cast(context)->fail("Screen capture error (" + std::to_string(code) + ")"); - } - static void encoderError(OH_AVCodec *, int32_t code, void *context) - { - static_cast(context)->fail("Video encoder error (" + std::to_string(code) + ")"); - } - static void output(OH_AVCodec *encoder, uint32_t index, OH_AVBuffer *buffer, void *context) - { - auto &self = *static_cast(context); - OH_AVCodecBufferAttr attr{}; - const auto capacity = OH_AVBuffer_GetCapacity(buffer); - auto *data = OH_AVBuffer_GetAddr(buffer); - if (OH_AVBuffer_GetBufferAttr(buffer, &attr) != AV_ERR_OK || attr.offset < 0 || - attr.size < 0 || attr.size > static_cast(MAX_PACKET) || - static_cast(attr.offset) + attr.size > capacity || (!data && attr.size)) { - self.fail("编码器返回无效数据。"); - } else if (attr.size && !self.cancelled) { - Packet packet{{data + attr.offset, data + attr.offset + attr.size}, attr.flags, attr.pts}; - std::lock_guard lock(self.mutex); - if (self.packets.size() >= 6 || self.queuedBytes + packet.data.size() > MAX_QUEUE) { - self.error = "Mac 接收速度不足，共享已停止，请重新连接。"; - self.cancelled = true; - } else { - self.queuedBytes += packet.data.size(); - self.packets.push_back(std::move(packet)); - } - } - self.check(OH_VideoEncoder_FreeOutputBuffer(encoder, index), "release encoded buffer"); - } - bool io(int socket, uint8_t *data, size_t size, bool sendData, int timeoutMs) - { - auto deadline = Clock::now() + std::chrono::milliseconds(timeoutMs); - while (size && !cancelled && Clock::now() < deadline) { - pollfd p{socket, static_cast(sendData ? POLLOUT : POLLIN), 0}; - int ready = poll(&p, 1, 50); - if (ready < 0 && errno == EINTR) continue; - if (ready < 0 || (ready && (p.revents & (POLLERR | POLLHUP | POLLNVAL)))) return false; - if (!ready) continue; - auto count = sendData ? send(socket, data, size, MSG_NOSIGNAL | MSG_DONTWAIT) - : recv(socket, data, size, MSG_DONTWAIT); - if (count < 0 && (errno == EINTR || errno == EAGAIN)) continue; - if (count <= 0) return false; - size -= count; - data += count; - } - return size == 0; - } - bool setup(OH_AVCodec *encoder, OH_AVScreenCapture *capture, OHNativeWindow **surface, int width, int height) - { - OH_AVCodecCallback callbacks{encoderError, [](OH_AVCodec *, OH_AVFormat *, void *) {}, - [](OH_AVCodec *, uint32_t, OH_AVBuffer *, void *) {}, output}; - if (!check(OH_VideoEncoder_RegisterCallback(encoder, callbacks, this), "encoder callbacks")) return false; - auto *format = OH_AVFormat_Create(); - if (!format) { fail("无法创建视频配置。"); return false; } - OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, width); - OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, height); - OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, AV_PIXEL_FORMAT_SURFACE_FORMAT); - OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, FPS); - OH_AVFormat_SetLongValue(format, OH_MD_KEY_BITRATE, 4000000); - OH_AVFormat_SetIntValue(format, OH_MD_KEY_VIDEO_ENCODE_BITRATE_MODE, BITRATE_MODE_CBR); - OH_AVFormat_SetIntValue(format, OH_MD_KEY_PROFILE, AVC_PROFILE_BASELINE); - OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, 1000); - const int configured = OH_VideoEncoder_Configure(encoder, format); - OH_AVFormat_Destroy(format); - if (!check(configured, "configure encoder") || - !check(OH_VideoEncoder_GetSurface(encoder, surface), "encoder surface") || - !check(OH_VideoEncoder_Prepare(encoder), "prepare encoder") || - !check(OH_VideoEncoder_Start(encoder), "start encoder")) return false; - OH_AVScreenCaptureConfig config{}; - config.captureMode = OH_CAPTURE_HOME_SCREEN; - config.dataType = OH_ORIGINAL_STREAM; - config.videoInfo.videoCapInfo.videoSource = OH_VIDEO_SOURCE_SURFACE_RGBA; - config.videoInfo.videoCapInfo.videoFrameWidth = width; - config.videoInfo.videoCapInfo.videoFrameHeight = height; - return check(OH_AVScreenCapture_SetStateCallback(capture, captureState, this), "capture state") && - check(OH_AVScreenCapture_SetErrorCallback(capture, captureError, this), "capture errors") && - check(OH_AVScreenCapture_SetMicrophoneEnabled(capture, false), "disable microphone") && - check(OH_AVScreenCapture_Init(capture, config), "initialize capture") && - check(OH_AVScreenCapture_StartScreenCaptureWithSurface(capture, *surface), "request screen sharing"); - } - void run(int displayWidth, int displayHeight, int port, const std::string &token) - { - int listener = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0), client = -1; - OH_AVCodec *encoder = nullptr; - OH_AVScreenCapture *capture = nullptr; - OHNativeWindow *surface = nullptr; - bool mediaStarted = false; - auto perform = [&] { - sockaddr_in address{}; - address.sin_family = AF_INET; - address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - address.sin_port = htons(port); - if (listener < 0 || bind(listener, reinterpret_cast(&address), sizeof(address)) || - listen(listener, 1)) { fail("无法建立共享连接，请在 Mac 重新开始。"); return; } - auto deadline = Clock::now() + std::chrono::seconds(45); - while (!cancelled && Clock::now() < deadline) { - pollfd p{listener, POLLIN, 0}; - if (poll(&p, 1, 100) <= 0) continue; - client = accept(listener, nullptr, nullptr); - if (client < 0) continue; - std::vector auth(64); - if (io(client, auth.data(), auth.size(), false, 2000) && - std::equal(auth.begin(), auth.end(), token.begin())) break; - close(client); - client = -1; - } - if (client < 0 || cancelled) { if (!cancelled) fail("Mac 连接超时，请重新开始。"); return; } - close(listener); - listener = -1; - const int noDelay = 1; - setsockopt(client, IPPROTO_TCP, TCP_NODELAY, &noDelay, sizeof(noDelay)); - auto *capability = OH_AVCodec_GetCapabilityByCategory(OH_AVCODEC_MIMETYPE_VIDEO_AVC, true, HARDWARE); - if (!capability) { fail("设备没有可用的 H.264 硬件编码器。"); return; } - int32_t widthAlignment = 0, heightAlignment = 0; - if (!check(OH_AVCapability_GetVideoWidthAlignment(capability, &widthAlignment), "width alignment") || - !check(OH_AVCapability_GetVideoHeightAlignment(capability, &heightAlignment), "height alignment")) return; - if (widthAlignment < 1 || heightAlignment < 1 || widthAlignment > 1280 || heightAlignment > 1280) { - fail("编码器返回不支持的画面尺寸约束。"); return; - } - widthAlignment = std::lcm(widthAlignment, 2); - heightAlignment = std::lcm(heightAlignment, 2); - const double scale = std::min(1.0, 1280.0 / std::max(displayWidth, displayHeight)); - const int width = static_cast(displayWidth * scale) / widthAlignment * widthAlignment; - const int height = static_cast(displayHeight * scale) / heightAlignment * heightAlignment; - if (width < 2 || height < 2 || !OH_AVCapability_IsVideoSizeSupported(capability, width, height)) { - fail("硬件编码器不支持当前屏幕尺寸。"); return; - } - std::vector hello{'H', 'C', 'S', '1'}; - for (auto value : {width, height, displayWidth, displayHeight, FPS}) put32(hello, value); - if (!io(client, hello.data(), hello.size(), true, 500)) return; - const char *encoderName = OH_AVCapability_GetName(capability); - if (!encoderName) { fail("设备未返回硬件编码器名称。"); return; } - encoder = OH_VideoEncoder_CreateByName(encoderName); - capture = OH_AVScreenCapture_Create(); - if (!encoder || !capture) { fail("设备无法创建录屏编码器。"); return; } - { std::lock_guard lock(mutex); state = "authorizing"; } - mediaStarted = setup(encoder, capture, &surface, width, height); - if (!mediaStarted) return; - auto lastFrame = Clock::now(); - while (!cancelled) { - if (activated.exchange(false) && - !check(OH_AVScreenCapture_SetMaxVideoFrameRate(capture, FPS), "frame rate")) break; - // Closing the host socket or sending 0 always stops capture. - uint8_t command; - const auto count = recv(client, &command, 1, MSG_DONTWAIT); - if (count == 0 || (count > 0 && command == 0)) break; - if (count < 0 && errno != EAGAIN && errno != EINTR) break; - Packet packet; - { - std::lock_guard lock(mutex); - if (!packets.empty()) { - packet = std::move(packets.front()); - packets.pop_front(); - queuedBytes -= packet.data.size(); - } - } - if (packet.data.empty()) { - if (Clock::now() - lastFrame > std::chrono::seconds(45)) { - fail("未收到视频画面，请确认手机上的共享授权。"); break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - continue; - } - std::vector header; - put32(header, packet.data.size()); - put32(header, packet.flags); - put32(header, static_cast(packet.pts) >> 32); - put32(header, static_cast(packet.pts)); - if (!io(client, header.data(), header.size(), true, 500) || - !io(client, packet.data.data(), packet.data.size(), true, 500)) break; - bytes += packet.data.size(); - if (!(packet.flags & AVCODEC_BUFFER_FLAGS_CODEC_DATA)) frames++; - lastFrame = Clock::now(); - } - }; - perform(); - cancelled = true; - if (capture) { - if (mediaStarted) OH_AVScreenCapture_StopScreenCapture(capture); - OH_AVScreenCapture_Release(capture); - } - if (encoder) { - if (mediaStarted) OH_VideoEncoder_NotifyEndOfStream(encoder); - OH_VideoEncoder_Stop(encoder); - OH_VideoEncoder_Destroy(encoder); - } - if (surface) OH_NativeWindow_DestroyNativeWindow(surface); - if (client >= 0) close(client); - if (listener >= 0) close(listener); - std::lock_guard lock(mutex); - packets.clear(); - queuedBytes = 0; - state = "stopped"; - } -}; - -Stream stream; -napi_value Start(napi_env env, napi_callback_info info) -{ - size_t argc = 4; - napi_value args[4]; - napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); - int32_t width = 0, height = 0, port = 0; - char token[65]{}; - size_t length = 0; - bool valid = argc == 4 && napi_get_value_int32(env, args[0], &width) == napi_ok && - napi_get_value_int32(env, args[1], &height) == napi_ok && - napi_get_value_int32(env, args[2], &port) == napi_ok && - napi_get_value_string_utf8(env, args[3], nullptr, 0, &length) == napi_ok && length == 64 && - napi_get_value_string_utf8(env, args[3], token, sizeof(token), &length) == napi_ok && - width >= 2 && height >= 2 && width <= 8192 && height <= 8192 && port >= 1024 && port <= 65535 && - std::all_of(token, token + 64, [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); }); - if (!valid) { napi_throw_error(env, "invalid_config", "请从 Mac 的 Harmony Connector 开始投屏。"); return nullptr; } - stream.start(width, height, port, std::string(token, 64)); - return nullptr; -} -napi_value Stop(napi_env, napi_callback_info) { stream.stop(); return nullptr; } -napi_value Status(napi_env env, napi_callback_info) { return stream.status(env); } -napi_value Init(napi_env env, napi_value exports) -{ - napi_property_descriptor methods[] = { - {"start", nullptr, Start, nullptr, nullptr, nullptr, napi_default, nullptr}, - {"stop", nullptr, Stop, nullptr, nullptr, nullptr, napi_default, nullptr}, - {"status", nullptr, Status, nullptr, nullptr, nullptr, napi_default, nullptr} - }; - napi_define_properties(env, exports, 3, methods); - napi_add_env_cleanup_hook(env, [](void *) { stream.stop(); }, nullptr); - return exports; -} -napi_module module = {1, 0, nullptr, Init, "harmony_stream", nullptr, {0}}; -__attribute__((constructor)) void Register() { napi_module_register(&module); } -} // namespace diff --git a/curios/harmony-connector/companion/entry/src/main/cpp/types/libharmony_stream/index.d.ts b/curios/harmony-connector/companion/entry/src/main/cpp/types/libharmony_stream/index.d.ts deleted file mode 100644 index 613c3df..0000000 --- a/curios/harmony-connector/companion/entry/src/main/cpp/types/libharmony_stream/index.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export interface StreamStatus { - state: string; - error: string; - frames: number; - bytes: number; -} -export const start: (width: number, height: number, port: number, token: string) => void; -export const stop: () => void; -export const status: () => StreamStatus; diff --git a/curios/harmony-connector/companion/entry/src/main/cpp/types/libharmony_stream/oh-package.json5 b/curios/harmony-connector/companion/entry/src/main/cpp/types/libharmony_stream/oh-package.json5 deleted file mode 100644 index 03734e4..0000000 --- a/curios/harmony-connector/companion/entry/src/main/cpp/types/libharmony_stream/oh-package.json5 +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "libharmony_stream.so", - "version": "0.1.0", - "types": "./index.d.ts" -} diff --git a/curios/harmony-connector/companion/entry/src/main/ets/entryability/EntryAbility.ets b/curios/harmony-connector/companion/entry/src/main/ets/entryability/EntryAbility.ets deleted file mode 100644 index 3948288..0000000 --- a/curios/harmony-connector/companion/entry/src/main/ets/entryability/EntryAbility.ets +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2026 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for details. - * Modified for Harmony Connector by Trove. - */ -import { UIAbility, Want } from '@kit.AbilityKit'; -import { window } from '@kit.ArkUI'; -import stream from 'libharmony_stream.so'; - -export default class EntryAbility extends UIAbility { - private configure(want: Want): void { - const token = want.parameters?.['token']; - const port = want.parameters?.['port']; - if (typeof token === 'string' && typeof port === 'number' && - /^[a-f0-9]{64}$/.test(token) && port >= 1024 && port <= 65535) { - stream.stop(); - AppStorage.setOrCreate('streamToken', token); - AppStorage.setOrCreate('streamPort', port); - } - } - onCreate(want: Want): void { this.configure(want); } - onNewWant(want: Want): void { this.configure(want); } - onWindowStageCreate(stage: window.WindowStage): void { - stage.loadContent('pages/Index').catch(() => { stream.stop(); }); - } - onDestroy(): void { stream.stop(); } -} diff --git a/curios/harmony-connector/companion/entry/src/main/ets/pages/Index.ets b/curios/harmony-connector/companion/entry/src/main/ets/pages/Index.ets deleted file mode 100644 index 8014c17..0000000 --- a/curios/harmony-connector/companion/entry/src/main/ets/pages/Index.ets +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2026 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed - * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for details. - * Modified for Harmony Connector by Trove. - */ -import { common, wantAgent } from '@kit.AbilityKit'; -import { display } from '@kit.ArkUI'; -import { backgroundTaskManager } from '@kit.BackgroundTasksKit'; -import stream from 'libharmony_stream.so'; - -@Entry -@Component -struct Index { - @StorageLink('streamToken') token: string = ''; - @StorageLink('streamPort') port: number = 0; - @State active: boolean = false; - @State message: string = '在 Mac 上连接手机并开始投屏。'; - @State stats: string = ''; - private timer: number = 0; - private backgroundRunning: boolean = false; - private keeping: boolean = false; - private generation: number = 0; - private displayWidth: number = 0; - private displayHeight: number = 0; - private previousFrames: number = 0; - private previousBytes: number = 0; - private previousTime: number = 0; - - private context(): common.UIAbilityContext { - return this.getUIContext().getHostContext() as common.UIAbilityContext; - } - private async keepRunning(): Promise { - if (this.backgroundRunning || this.keeping) return; - this.keeping = true; - const generation = this.generation; - try { - const agent = await wantAgent.getWantAgent({ - wants: [{ bundleName: 'ink.render.trove.harmonyconnector', abilityName: 'EntryAbility' }], - actionType: wantAgent.OperationType.START_ABILITY, - requestCode: 1, - actionFlags: [wantAgent.WantAgentFlags.UPDATE_PRESENT_FLAG] - }); - if (!this.active || generation !== this.generation) return; - await backgroundTaskManager.startBackgroundRunning(this.context(), - backgroundTaskManager.BackgroundMode.AUDIO_RECORDING, agent); - this.backgroundRunning = true; - if (!this.active || generation !== this.generation) await this.stop(); - } catch (error) { - if (this.active) { - await this.stop(); - this.message = `无法保持后台共享：${error.message}`; - } - } finally { this.keeping = false; } - } - private async stop(): Promise { - this.generation++; - stream.stop(); - this.active = false; - this.stats = ''; - if (this.backgroundRunning) { - this.backgroundRunning = false; - try { await backgroundTaskManager.stopBackgroundRunning(this.context()); } catch (_) {} - } - this.message = '共享已停止。可在 Mac 上重新开始。'; - } - private start(): void { - try { - const screen = display.getDefaultDisplaySync(); - this.displayWidth = screen.width; - this.displayHeight = screen.height; - this.previousTime = Date.now(); - this.previousFrames = 0; - this.previousBytes = 0; - this.generation++; - stream.start(screen.width, screen.height, this.port, this.token); - this.active = true; - this.message = '正在连接 Mac，请同意系统的屏幕共享请求。'; - } catch (error) { this.message = error.message; } - } - private update(): void { - if (!this.active) return; - const status = stream.status(); - if (status.state === 'stopped') { - this.stop().then(() => { if (status.error) this.message = status.error; }); - return; - } - if (status.state !== 'sharing') return; - let screen: display.Display; - try { screen = display.getDefaultDisplaySync(); } catch (_) { this.stop(); return; } - if (screen.width !== this.displayWidth || screen.height !== this.displayHeight) { - this.stop().then(() => { this.message = '屏幕方向已改变，请在 Mac 重新开始投屏。'; }); - return; - } - this.keepRunning(); - const now = Date.now(); - const seconds = (now - this.previousTime) / 1000; - this.message = '正在共享屏幕'; - if (seconds >= 1) { - this.stats = `${((status.frames - this.previousFrames) / seconds).toFixed(1)} 帧/秒 · ${((status.bytes - this.previousBytes) * 8 / seconds / 1000000).toFixed(2)} Mbps`; - this.previousTime = now; - this.previousFrames = status.frames; - this.previousBytes = status.bytes; - } - } - aboutToAppear(): void { this.timer = setInterval(() => this.update(), 250); } - aboutToDisappear(): void { clearInterval(this.timer); this.stop(); } - - build() { - Column({ space: 24 }) { - Text('Harmony Connector').fontSize(30).fontWeight(FontWeight.Bold) - Text(this.message).fontSize(18).textAlign(TextAlign.Center) - Text('通过 USB 将画面共享到 Mac。系统会遮挡受保护的内容，麦克风始终关闭。') - .fontSize(15).fontColor('#64748B').textAlign(TextAlign.Center) - if (this.stats) { Text(this.stats).fontSize(16) } - Button(this.active ? '停止共享' : '共享屏幕') - .enabled(this.active || (this.token.length === 64 && this.port > 0)) - .onClick(() => { if (this.active) this.stop(); else this.start(); }) - Text('共享期间可随时返回这里停止，也可在系统录屏通知中停止。') - .fontSize(13).fontColor('#64748B').textAlign(TextAlign.Center) - } - .width('100%').height('100%').padding(32).justifyContent(FlexAlign.Center) - } -} diff --git a/curios/harmony-connector/companion/entry/src/main/module.json5 b/curios/harmony-connector/companion/entry/src/main/module.json5 deleted file mode 100644 index c90a38e..0000000 --- a/curios/harmony-connector/companion/entry/src/main/module.json5 +++ /dev/null @@ -1,47 +0,0 @@ -{ - "module": { - "name": "entry", - "type": "entry", - "description": "$string:module_desc", - "mainElement": "EntryAbility", - "deviceTypes": [ - "phone" - ], - "deliveryWithInstall": true, - "installationFree": false, - "pages": "$profile:main_pages", - "abilities": [ - { - "name": "EntryAbility", - "srcEntry": "./ets/entryability/EntryAbility.ets", - "description": "$string:EntryAbility_desc", - "icon": "$media:layered_image", - "label": "$string:EntryAbility_label", - "startWindowIcon": "$media:startIcon", - "startWindowBackground": "$color:start_window_background", - "exported": true, - "skills": [ - { - "entities": [ - "entity.system.home" - ], - "actions": [ - "ohos.want.action.home" - ] - } - ], - "backgroundModes": [ - "audioRecording" - ] - } - ], - "requestPermissions": [ - { - "name": "ohos.permission.INTERNET" - }, - { - "name": "ohos.permission.KEEP_BACKGROUND_RUNNING" - } - ] - } -} diff --git a/curios/harmony-connector/companion/entry/src/main/resources/base/element/color.json b/curios/harmony-connector/companion/entry/src/main/resources/base/element/color.json deleted file mode 100644 index 3c71296..0000000 --- a/curios/harmony-connector/companion/entry/src/main/resources/base/element/color.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "color": [ - { - "name": "start_window_background", - "value": "#FFFFFF" - } - ] -} \ No newline at end of file diff --git a/curios/harmony-connector/companion/entry/src/main/resources/base/element/float.json b/curios/harmony-connector/companion/entry/src/main/resources/base/element/float.json deleted file mode 100644 index 33ea223..0000000 --- a/curios/harmony-connector/companion/entry/src/main/resources/base/element/float.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "float": [ - { - "name": "page_text_font_size", - "value": "50fp" - } - ] -} diff --git a/curios/harmony-connector/companion/entry/src/main/resources/base/element/string.json b/curios/harmony-connector/companion/entry/src/main/resources/base/element/string.json deleted file mode 100644 index 96f6f79..0000000 --- a/curios/harmony-connector/companion/entry/src/main/resources/base/element/string.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "string": [ - { - "name": "module_desc", - "value": "通过 USB 将手机画面共享到 Mac" - }, - { - "name": "EntryAbility_desc", - "value": "通过 USB 将手机画面共享到 Mac" - }, - { - "name": "EntryAbility_label", - "value": "Harmony Connector" - } - ] -} diff --git a/curios/harmony-connector/companion/entry/src/main/resources/base/media/background.png b/curios/harmony-connector/companion/entry/src/main/resources/base/media/background.png deleted file mode 100644 index e129e5a..0000000 Binary files a/curios/harmony-connector/companion/entry/src/main/resources/base/media/background.png and /dev/null differ diff --git a/curios/harmony-connector/companion/entry/src/main/resources/base/media/foreground.png b/curios/harmony-connector/companion/entry/src/main/resources/base/media/foreground.png deleted file mode 100644 index e129e5a..0000000 Binary files a/curios/harmony-connector/companion/entry/src/main/resources/base/media/foreground.png and /dev/null differ diff --git a/curios/harmony-connector/companion/entry/src/main/resources/base/media/layered_image.json b/curios/harmony-connector/companion/entry/src/main/resources/base/media/layered_image.json deleted file mode 100644 index fb49920..0000000 --- a/curios/harmony-connector/companion/entry/src/main/resources/base/media/layered_image.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "layered-image": - { - "background" : "$media:background", - "foreground" : "$media:foreground" - } -} \ No newline at end of file diff --git a/curios/harmony-connector/companion/entry/src/main/resources/base/media/startIcon.png b/curios/harmony-connector/companion/entry/src/main/resources/base/media/startIcon.png deleted file mode 100644 index e129e5a..0000000 Binary files a/curios/harmony-connector/companion/entry/src/main/resources/base/media/startIcon.png and /dev/null differ diff --git a/curios/harmony-connector/companion/entry/src/main/resources/base/profile/main_pages.json b/curios/harmony-connector/companion/entry/src/main/resources/base/profile/main_pages.json deleted file mode 100644 index 1898d94..0000000 --- a/curios/harmony-connector/companion/entry/src/main/resources/base/profile/main_pages.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "src": [ - "pages/Index" - ] -} diff --git a/curios/harmony-connector/companion/entry/src/main/resources/dark/element/color.json b/curios/harmony-connector/companion/entry/src/main/resources/dark/element/color.json deleted file mode 100644 index 79b11c2..0000000 --- a/curios/harmony-connector/companion/entry/src/main/resources/dark/element/color.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "color": [ - { - "name": "start_window_background", - "value": "#000000" - } - ] -} \ No newline at end of file diff --git a/curios/harmony-connector/companion/hvigor/hvigor-config.json5 b/curios/harmony-connector/companion/hvigor/hvigor-config.json5 deleted file mode 100644 index 2140b7a..0000000 --- a/curios/harmony-connector/companion/hvigor/hvigor-config.json5 +++ /dev/null @@ -1,23 +0,0 @@ -{ - "modelVersion": "26.0.0", - "dependencies": { - }, - "execution": { - // "analyze": "normal", /* Define the build analyze mode. Value: [ "normal" | "advanced" | "ultrafine" | false ]. Default: "normal" */ - // "daemon": true, /* Enable daemon compilation. Value: [ true | false ]. Default: true */ - // "incremental": true, /* Enable incremental compilation. Value: [ true | false ]. Default: true */ - // "parallel": true, /* Enable parallel compilation. Value: [ true | false ]. Default: true */ - // "typeCheck": false, /* Enable typeCheck. Value: [ true | false ]. Default: false */ - // "optimizationStrategy": "memory" /* Define the optimization strategy. Value: [ "memory" | "performance" ]. Default: "memory" */ - }, - "logging": { - // "level": "info" /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */ - }, - "debugging": { - // "stacktrace": false /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */ - }, - "nodeOptions": { - // "maxOldSpaceSize": 8192 /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/ - // "exposeGC": true /* Enable to trigger garbage collection explicitly. Default: true*/ - } -} diff --git a/curios/harmony-connector/companion/hvigorfile.ts b/curios/harmony-connector/companion/hvigorfile.ts deleted file mode 100644 index 8890d28..0000000 --- a/curios/harmony-connector/companion/hvigorfile.ts +++ /dev/null @@ -1,7 +0,0 @@ -// @ts-nocheck – Template file, only used when copied into a project directory -import { appTasks } from '@ohos/hvigor-ohos-plugin'; - -export default { - system: appTasks /* Built-in plugin of Hvigor. It cannot be modified. */, - plugins: [] /* Custom plugin to extend the functionality of Hvigor. */, -}; diff --git a/curios/harmony-connector/companion/legal/NOTICE.md b/curios/harmony-connector/companion/legal/NOTICE.md deleted file mode 100644 index 4fc1350..0000000 --- a/curios/harmony-connector/companion/legal/NOTICE.md +++ /dev/null @@ -1,17 +0,0 @@ -# Template attribution - -The initial HarmonyOS Empty Ability project was generated with Huawei's official -`@deveco/deveco-cli` 1.3.0-stable on 2026-09-10. Generated project configuration, -resource images, and the initial ArkTS entry/page came from that template. - -The generated ArkTS sources carry: - -Copyright (c) 2026 Huawei Device Co., Ltd. - -Licensed under the Apache License, Version 2.0. A copy is provided in -`Apache-2.0.txt`. - -Trove replaced the sample page and ability lifecycle with Harmony Connector's -screen-sharing interface and lifecycle. The native screen capture/transport -module and Mac measurement tools were written for this repository. No DevEco -private screen agent or phone system library is redistributed. diff --git a/curios/harmony-connector/companion/oh-package.json5 b/curios/harmony-connector/companion/oh-package.json5 deleted file mode 100644 index 7ae4fd7..0000000 --- a/curios/harmony-connector/companion/oh-package.json5 +++ /dev/null @@ -1,6 +0,0 @@ -{ - "modelVersion": "26.0.0", - "description": "Harmony Connector screen sharing companion", - "dependencies": {}, - "devDependencies": {} -} diff --git a/curios/harmony-connector/legal/THIRD-PARTY.md b/curios/harmony-connector/legal/THIRD-PARTY.md index 093ec6f..08d4fee 100644 --- a/curios/harmony-connector/legal/THIRD-PARTY.md +++ b/curios/harmony-connector/legal/THIRD-PARTY.md @@ -18,6 +18,14 @@ The native executable also uses Rust crates listed in `rust/`, with the license, copyright and notice files provided by each crate. Trove's UI SDK is distributed from the versioned SDK archive in the source repository. +The executable also embeds the unchanged `uitest_agent_v1.2.2.so` distributed in +`xdevice-devicetest` 6.1.0.210. Its package includes the Apache License 2.0; that +license and the original download URL, wheel member and SHA-256 checksums are in +`hypium/`. The binary is not modified or stripped. It is copied to a temporary +device file for an authorized HDC session and removed when that session closes. +The upstream wheel does not include this native agent's source; embedding this +binary does not make it independently rebuildable from this repository. + ## Building the distributed libraries The exact build commands and SHA-256 pins are in the accompanying `build.mjs`. diff --git a/curios/harmony-connector/native/src/diagnostics.rs b/curios/harmony-connector/native/src/diagnostics.rs index 9b188ca..6127d96 100644 --- a/curios/harmony-connector/native/src/diagnostics.rs +++ b/curios/harmony-connector/native/src/diagnostics.rs @@ -49,7 +49,7 @@ pub fn reconnect() -> anyhow::Result<()> { } // Uses the same management entry points as the UI. It never enables control or sends input. -pub fn screen(args: &[String]) -> anyhow::Result<()> { +pub fn screen(args: &[String], continuous: bool) -> anyhow::Result<()> { anyhow::ensure!( !args.is_empty() && args.len() <= 3, "用法：--check-screen /absolute/path/to/hdc [设备 ID] [保存目录]" @@ -78,16 +78,46 @@ pub fn screen(args: &[String]) -> anyhow::Result<()> { let id = json!({"session":opened["session"]}); println!( "{}", - json!({"stage":"opened","inputCapabilities":opened["inputCapabilities"],"control":false}) + json!({"stage":"opened","mode":opened["mode"],"inputCapabilities":opened["inputCapabilities"],"control":false}) ); let started = Instant::now(); let result = (|| -> anyhow::Result<()> { - for _ in 0..3 { - let frame = manager.call("screenFrame", id.clone())?; - println!( - "{}", - json!({"stage":"frame","sequence":frame["sequence"],"width":frame["width"],"height":frame["height"],"captureMs":frame["captureMs"]}) - ); + let mut after = 0u64; + let mut observations = 0; + let mut unique = 0; + let mut decoded = 0; + let mut report_at = Instant::now(); + while if continuous { + started.elapsed() < Duration::from_secs(20) + } else { + observations < 3 + } { + let frame = manager.call( + "screenFrame", + json!({"session":opened["session"],"after":after}), + )?; + observations += 1; + if frame["unchanged"] != true { + use base64::Engine; + let url = frame["dataUrl"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("Missing JPEG"))?; + let jpeg = base64::engine::general_purpose::STANDARD.decode( + url.strip_prefix("data:image/jpeg;base64,") + .ok_or_else(|| anyhow::anyhow!("Invalid JPEG URL"))?, + )?; + image::load_from_memory(&jpeg)?; + decoded += 1; + unique += 1; + } + after = frame["sequence"].as_u64().unwrap(); + if !continuous || report_at.elapsed() >= Duration::from_secs(1) { + println!( + "{}", + json!({"stage":"frame","sequence":after,"width":frame["width"],"height":frame["height"],"frameAgeMs":frame["frameAgeMs"],"receivedFps":frame["receivedFps"],"mbps":frame["mbps"],"uniqueFrames":unique}) + ); + report_at = Instant::now(); + } } if let Some(directory) = args.get(2) { let saved = manager.call( @@ -101,7 +131,7 @@ pub fn screen(args: &[String]) -> anyhow::Result<()> { } println!( "{}", - json!({"passed":true,"frames":3,"seconds":started.elapsed().as_secs_f64(),"inputSent":false}) + json!({"connectionPassed":decoded>0,"uniqueFrames":unique,"decodedFrames":decoded,"seconds":started.elapsed().as_secs_f64(),"consumerFps":if continuous {Some(unique as f64/started.elapsed().as_secs_f64())} else {None},"inputSent":false,"performanceRequiresMovingScreen":true}) ); Ok(()) })(); diff --git a/curios/harmony-connector/native/src/hdc.rs b/curios/harmony-connector/native/src/hdc.rs index 7347ce8..a0043ad 100644 --- a/curios/harmony-connector/native/src/hdc.rs +++ b/curios/harmony-connector/native/src/hdc.rs @@ -1,6 +1,8 @@ mod files; mod screen; +mod screen_agent; mod screen_input; +mod screen_protocol; use crate::{ command, hdc_data, model::{Fault, Result}, @@ -362,14 +364,18 @@ impl Manager { if method == "screenSave" { return self.screen.save(parse(value)?); } + if method == "screenFrame" { + return self.screen.frame(parse(value)?); + } + if method == "screenInput" { + return self.screen.input(parse(value)?); + } let guard = self.enter()?; match method { "screenStart" => { let r: Target = parse(value)?; self.screen.start(self.tool()?, r.target) } - "screenFrame" => self.screen.frame(parse(value)?), - "screenInput" => self.screen.input(parse(value)?), "tool" => { let path = { let mut path = self.path.lock().unwrap(); diff --git a/curios/harmony-connector/native/src/hdc/screen.rs b/curios/harmony-connector/native/src/hdc/screen.rs index 353b9d9..6d68df2 100644 --- a/curios/harmony-connector/native/src/hdc/screen.rs +++ b/curios/harmony-connector/native/src/hdc/screen.rs @@ -1,22 +1,21 @@ use super::screen_input::{self, Input}; use super::*; use base64::{engine::general_purpose::STANDARD, Engine}; -use image::{codecs::jpeg::JpegEncoder, ImageFormat, ImageReader}; use std::{ collections::HashSet, - io::{BufReader, Write}, + io::Write, time::{Instant, SystemTime, UNIX_EPOCH}, }; const LEASE: Duration = Duration::from_secs(15); struct Frame { - jpeg: Vec, + jpeg: Arc>, width: u32, height: u32, sequence: u64, geometry: u64, captured: Instant, - capture_ms: u128, + previous_sequence: u64, } struct Session { id: u64, @@ -27,11 +26,13 @@ struct Session { supported: HashSet, frame: Option, touched: Instant, + stream: Arc, } #[derive(Default)] pub(super) struct Screen { current: Mutex>, next: AtomicU64, + lifecycle: Mutex<()>, } #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -40,6 +41,13 @@ pub(super) struct Id { } #[derive(Deserialize)] #[serde(deny_unknown_fields)] +pub(super) struct FrameRequest { + pub session: u64, + #[serde(default)] + pub after: u64, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] pub(super) struct Control { pub session: u64, pub enabled: bool, @@ -64,49 +72,60 @@ fn missing() -> Fault { } impl Screen { pub fn close(&self, id: Option) -> Result { - let mut current = self.current.lock().unwrap(); - if let Some(id) = id { - if current.as_ref().is_some_and(|s| s.id != id) { - return Err(missing()); + let _lifecycle = self.lifecycle.lock().unwrap(); + let session = { + let mut current = self.current.lock().unwrap(); + if let Some(id) = id { + if current.as_ref().is_some_and(|s| s.id != id) { + return Err(missing()); + } } - } - if let Some(session) = current.take() { + current.take() + }; + if let Some(session) = session { session.cancel.store(true, Ordering::Release); + session.stream.stop(); } Ok(json!({"stopped":true})) } pub fn start(&self, tool: Tool, target: String) -> Result { + let _lifecycle = self.lifecycle.lock().unwrap(); hdc_data::target(&target)?; { - let current = self.current.lock().unwrap(); + let mut current = self.current.lock().unwrap(); if current .as_ref() .is_some_and(|s| s.touched.elapsed() < LEASE) { return Err(Fault::new("screen.active", "请先停止当前投屏。")); } + if let Some(session) = current.take() { + session.stream.stop(); + } } tool.connected(&target)?; - let help = tool.shell(&target, "uitest help")?; - if !help.contains("screenCap") { - return Err(Fault::new( - "screen.unavailable", - "设备没有开放屏幕采集，请检查调试工具与设备授权。", - )); - } - let input_help = tool + let help = tool .shell(&target, "uitest uiInput help") .unwrap_or_default(); - let supported = screen_input::capabilities(&input_help); + let mut supported = screen_input::capabilities(&help); + let stream = Arc::new(screen_agent::Stream::open(&tool, &target)?); + for input in [ + "click", + "doubleClick", + "longClick", + "swipe", + "drag", + "keyEvent", + ] { + supported.insert(input.into()); + } let id = self.next.fetch_add(1, Ordering::Relaxed) + 1; let mut capabilities: Vec<_> = supported.iter().cloned().collect(); capabilities.sort(); - self.close(None)?; - let mut current = self.current.lock().unwrap(); if tool.cancel.load(Ordering::Acquire) { return Err(missing()); } - *current = Some(Session { + *self.current.lock().unwrap() = Some(Session { id, target: target.clone(), path: tool.path, @@ -115,9 +134,10 @@ impl Screen { supported, frame: None, touched: Instant::now(), + stream, }); Ok( - json!({"session":id,"target":target,"inputCapabilities":capabilities,"control":false,"mode":"uitest-frames"}), + json!({"session":id,"target":target,"inputCapabilities":capabilities,"control":false,"mode":"hypium-stream"}), ) } fn access(&self, id: u64, f: impl FnOnce(&mut Session) -> Result) -> Result { @@ -137,17 +157,6 @@ impl Screen { s.touched = Instant::now(); f(s) } - fn context(&self, id: u64) -> Result<(Tool, String)> { - self.access(id, |s| { - Ok(( - Tool { - path: s.path.clone(), - cancel: s.cancel.clone(), - }, - s.target.clone(), - )) - }) - } pub fn control(&self, request: Control) -> Result { self.access(request.session, |s| { if request.enabled && request.geometry != s.frame.as_ref().map(|f| f.geometry) { @@ -166,21 +175,22 @@ impl Screen { Ok(json!({"control":s.control})) }) } - pub fn frame(&self, request: Id) -> Result { - let (tool, target) = self.context(request.session)?; - let started = Instant::now(); - let temporary = tempfile::NamedTempFile::new()?; + pub fn frame(&self, request: FrameRequest) -> Result { + let stream = self.access(request.session, |s| Ok(s.stream.clone()))?; let result = (|| { - capture(&tool, &target, temporary.path())?; - let (jpeg, width, height) = encode_frame(temporary.path())?; + let (frame, fps, mbps) = stream.frame(request.after)?; self.access(request.session, |s| { - let changed = s.frame.as_ref().is_some_and(|f| (f.width,f.height) != (width,height)); - let sequence = s.frame.as_ref().map_or(1, |f| f.sequence + 1); - let geometry = s.frame.as_ref().map_or(1, |f| f.geometry + u64::from(changed)); + let changed = s.frame.as_ref().is_some_and(|f| f.geometry != frame.geometry); + let previous = s.frame.as_ref().map_or(0, |f| if f.sequence != frame.sequence {f.sequence} else {f.previous_sequence}); if changed { s.control = false; } - let capture_ms = started.elapsed().as_millis(); - let value = json!({"session":s.id,"sequence":sequence,"geometry":geometry,"width":width,"height":height,"dataUrl":format!("data:image/jpeg;base64,{}",STANDARD.encode(&jpeg)),"captureMs":capture_ms,"control":s.control,"rotated":changed}); - s.frame = Some(Frame { jpeg,width,height,sequence,geometry,captured:Instant::now(),capture_ms }); + let mut value = json!({"session":s.id,"sequence":frame.sequence,"geometry":frame.geometry, + "width":frame.width,"height":frame.height,"frameAgeMs":frame.received.elapsed().as_millis(), + "receivedFps":fps,"mbps":mbps,"control":s.control,"rotated":changed,"unchanged":frame.sequence==request.after}); + if frame.sequence != request.after { + value["dataUrl"] = format!("data:image/jpeg;base64,{}",STANDARD.encode(frame.jpeg.as_slice())).into(); + } + s.frame = Some(Frame {jpeg:frame.jpeg,width:frame.width,height:frame.height, + sequence:frame.sequence,geometry:frame.geometry,captured:Instant::now(),previous_sequence:previous}); Ok(value) }) })(); @@ -190,21 +200,20 @@ impl Screen { result } pub fn input(&self, request: Event) -> Result { - let (tool, target, cmd) = self.access(request.session, |s| { + let (tool, target, stream, cmd) = self.access(request.session, |s| { if !s.control { return Err(Fault::new("screen.read_only", "请先启用控制。")); } let frame = s.frame.as_ref().ok_or_else(missing)?; if frame.geometry != request.geometry + || s.stream.current()?.geometry != frame.geometry || request.sequence == 0 - || request.sequence > frame.sequence - || frame.sequence - request.sequence > 1 + || ![frame.sequence, frame.previous_sequence].contains(&request.sequence) || frame.captured.elapsed() > Duration::from_secs(3) - || frame.capture_ms > 5000 { return Err(Fault::new( "screen.stale_frame", - "画面已变化或响应过慢，请等待新画面后再操作。", + "画面已变化，请等待新画面后再操作。", )); } let cmd = request @@ -216,10 +225,14 @@ impl Screen { cancel: s.cancel.clone(), }, s.target.clone(), + s.stream.clone(), cmd, )) })?; - let result = tool.shell(&target, &cmd); + let result = match request.input.rpc() { + Some((api, args)) => stream.driver(api, args).map(|_| ()), + None => tool.shell(&target, &cmd).map(|_| ()), + }; if result.is_err() { self.access(request.session, |s| { s.control = false; @@ -258,77 +271,3 @@ impl Screen { Ok(json!({"path":destination,"bytes":jpeg.len()})) } } -fn capture(tool: &Tool, target: &str, destination: &Path) -> Result<()> { - let nonce = tempfile::Builder::new() - .prefix("harmony-screen-") - .tempdir()?; - let remote = format!( - "/data/local/tmp/{}.png", - nonce.path().file_name().unwrap().to_string_lossy() - ); - let result = (|| { - tool.run( - &[ - "-t", - target, - "shell", - &format!("uitest screenCap -p {remote}"), - ], - 6, - )?; - tool.run( - &[ - "-t", - target, - "file", - "recv", - &remote, - destination.to_str().unwrap(), - ], - 6, - )?; - if destination.metadata()?.len() > 32 * 1024 * 1024 { - return Err(Fault::new( - "screen.frame_too_large", - "设备画面超过支持范围。", - )); - } - Ok(()) - })(); - let cleanup = Tool { - path: tool.path.clone(), - cancel: Arc::new(AtomicBool::new(false)), - }; - let _ = cleanup.run(&["-t", target, "shell", &format!("rm -f {remote}")], 2); - result -} -fn encode_frame(path: &Path) -> Result<(Vec, u32, u32)> { - let mut reader = - ImageReader::with_format(BufReader::new(fs::File::open(path)?), ImageFormat::Png); - let mut limits = image::Limits::default(); - limits.max_image_width = Some(8192); - limits.max_image_height = Some(8192); - limits.max_alloc = Some(64 * 1024 * 1024); - reader.limits(limits); - let image = reader - .decode() - .map_err(|_| Fault::new("screen.invalid_frame", "手机未返回有效画面，投屏已停止。"))?; - let (width, height) = (image.width(), image.height()); - if width == 0 || height == 0 { - return Err(Fault::new("screen.invalid_frame", "画面尺寸无效。")); - } - let scaled = if width > 1600 || height > 1600 { - image.thumbnail(1600, 1600) - } else { - image - } - .to_rgb8(); - let mut jpeg = Vec::new(); - JpegEncoder::new_with_quality(&mut jpeg, 82) - .encode_image(&scaled) - .map_err(|_| Fault::new("screen.encode_failed", "无法处理手机画面。"))?; - if jpeg.len() > 2 * 1024 * 1024 { - return Err(Fault::new("screen.frame_too_large", "画面数据过大。")); - } - Ok((jpeg, width, height)) -} diff --git a/curios/harmony-connector/native/src/hdc/screen_agent.rs b/curios/harmony-connector/native/src/hdc/screen_agent.rs new file mode 100644 index 0000000..e24c39d --- /dev/null +++ b/curios/harmony-connector/native/src/hdc/screen_agent.rs @@ -0,0 +1,493 @@ +//! One HDC-owned UITest daemon and two sockets: image push and control RPC. +//! Only the latest image is retained. Closing a session releases its own daemon, +//! extension file and forwarding rule; other debugging sessions are untouched. +use super::screen_protocol::{self, Decoder, Packet}; +use super::*; +use image::{ImageFormat, ImageReader}; +use std::{ + collections::VecDeque, + io::{Cursor, Read, Write}, + net::{Shutdown, TcpListener, TcpStream}, + sync::Condvar, + time::Instant, +}; + +const AGENT: &[u8] = + include_bytes!("../../../../../vendor/harmony-connector/hypium/uitest_agent_v1.2.2.so"); +const SOCKET: &str = "localabstract:uitest_socket"; +const LEASE: Duration = Duration::from_secs(15); + +fn fault(message: impl Into) -> Fault { + Fault::new("screen.connection_failed", message) +} +fn io(error: std::io::Error) -> Fault { + fault(format!("投屏连接已中断：{error}")) +} +fn response(value: Value) -> Result { + if value.get("exception").is_some_and(|v| !v.is_null()) { + return Err(Fault::new( + "screen.remote_error", + format!( + "设备未能执行操作：{}", + value["exception"]["message"].as_str().unwrap_or("未知错误") + ), + )); + } + value + .get("result") + .cloned() + .ok_or_else(|| fault("设备没有返回有效操作结果。")) +} +struct Rpc { + socket: TcpStream, + decoder: Decoder, + sequence: u32, + driver: String, +} +impl Rpc { + fn connect(port: u16) -> Result { + let socket = + TcpStream::connect_timeout(&([127, 0, 0, 1], port).into(), Duration::from_secs(2)) + .map_err(io)?; + socket.set_nodelay(true).map_err(io)?; + socket + .set_read_timeout(Some(Duration::from_secs(2))) + .map_err(io)?; + socket + .set_write_timeout(Some(Duration::from_secs(1))) + .map_err(io)?; + Ok(Self { + socket, + decoder: Decoder::default(), + sequence: 0, + driver: String::new(), + }) + } + fn send(&mut self, method: &str, params: Value) -> Result<()> { + self.sequence = self.sequence.wrapping_add(1); + let value = json!({"module":"com.ohos.devicetest.hypiumApiHelper","method":method,"params":params,"request_id":self.sequence.to_string()}); + self.socket + .write_all(&screen_protocol::request(&value, self.sequence)) + .map_err(io) + } + fn call(&mut self, method: &str, params: Value) -> Result { + let result = self.exchange(method, params); + if result + .as_ref() + .is_err_and(|error| error.code != "screen.remote_error") + { + // A timed-out response has no request ID on this protocol version. + // Never let a later command consume that ambiguous response. + let _ = self.socket.shutdown(Shutdown::Both); + } + result + } + fn exchange(&mut self, method: &str, params: Value) -> Result { + self.send(method, params)?; + let deadline = Instant::now() + Duration::from_secs(3); + let mut bytes = [0; 8192]; + loop { + if let Some(packet) = self.decoder.next()? { + return match packet { + Packet::Json(v) => response(v), + Packet::Jpeg(_) => Err(fault("控制通道返回了意外画面。")), + }; + } + if Instant::now() > deadline { + return Err(fault("手机控制响应超时。")); + } + let size = self.socket.read(&mut bytes).map_err(io)?; + if size == 0 { + return Err(fault("手机已关闭投屏连接。")); + } + self.decoder.push(&bytes[..size])?; + } + } + fn driver(&mut self, api: &str, args: Value) -> Result { + self.call( + "callHypiumApi", + json!({"api":api,"this":self.driver,"args":args,"message_type":"hypium"}), + ) + } + fn dimensions(&mut self) -> Result<(u32, u32)> { + let value = self.driver("Driver.getDisplaySize", json!([]))?; + let dimension = |key: &str| { + value[key] + .as_u64() + .filter(|n| (1..=8192).contains(n)) + .map(|n| n as u32) + .ok_or_else(|| fault("设备屏幕尺寸无效。")) + }; + Ok((dimension("x")?, dimension("y")?)) + } +} +struct DeviceSession { + tool: Tool, + target: String, + name: String, + forward: Option, +} +impl Drop for DeviceSession { + fn drop(&mut self) { + let cleanup = Tool { + path: self.tool.path.clone(), + cancel: Arc::new(AtomicBool::new(false)), + }; + if let Some(local) = &self.forward { + let _ = cleanup.run(&["-t", &self.target, "fport", "rm", local, SOCKET], 2); + } + // Never kill all UITest processes or remove a wildcard in /data/local/tmp. + if let Ok(pids) = cleanup.run_output(&["-t", &self.target, "shell", "pidof uitest"], 2) { + for pid in pids + .split_whitespace() + .filter(|p| p.bytes().all(|b| b.is_ascii_digit())) + { + if let Ok(cmdline) = cleanup.run_output( + &[ + "-t", + &self.target, + "shell", + &format!("cat /proc/{pid}/cmdline"), + ], + 2, + ) { + if cmdline.split('\0').any(|part| part == self.name) { + let _ = + cleanup.run(&["-t", &self.target, "shell", &format!("kill {pid}")], 2); + } + } + } + } + let _ = cleanup.run( + &[ + "-t", + &self.target, + "shell", + &format!("rm -f /data/local/tmp/{}", self.name), + ], + 2, + ); + } +} +#[derive(Clone)] +pub(super) struct Frame { + pub jpeg: Arc>, + pub width: u32, + pub height: u32, + pub sequence: u64, + pub geometry: u64, + pub received: Instant, +} +struct State { + latest: Option, + error: Option, + touched: Instant, + samples: VecDeque, + bytes: VecDeque<(Instant, usize)>, +} +struct Shared { + state: Mutex, + changed: Condvar, + cancel: Arc, +} +pub(super) struct Stream { + shared: Arc, + rpc: Arc>, + socket: TcpStream, + worker: Mutex>>, +} +impl Stream { + pub fn open(tool: &Tool, target: &str) -> Result { + if !tool + .run_output(&["-t", target, "shell", "pidof uitest"], 2)? + .trim() + .is_empty() + { + return Err(Fault::new( + "screen.debugger_busy", + "设备正在被其他 UI 调试工具使用，请结束该调试会话后再开始投屏。", + )); + } + let mut file = tempfile::Builder::new() + .prefix("trove-screen-") + .suffix(".so") + .tempfile()?; + file.write_all(AGENT)?; + let name = file + .path() + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + let mut resources = DeviceSession { + tool: Tool { + path: tool.path.clone(), + cancel: tool.cancel.clone(), + }, + target: target.into(), + name: name.clone(), + forward: None, + }; + tool.run( + &[ + "-t", + target, + "file", + "send", + file.path().to_str().unwrap(), + &format!("/data/local/tmp/{name}"), + ], + 10, + )?; + tool.run( + &[ + "-t", + target, + "shell", + &format!("uitest start-daemon singleness --extension-name {name}"), + ], + 5, + )?; + let reservation = TcpListener::bind(("127.0.0.1", 0)).map_err(io)?; + let port = reservation.local_addr().map_err(io)?.port(); + drop(reservation); + let local = format!("tcp:{port}"); + tool.run(&["-t", target, "fport", &local, SOCKET], 3)?; + resources.forward = Some(local); + let deadline = Instant::now() + Duration::from_secs(5); + let (mut control, driver) = loop { + if tool.cancel.load(Ordering::Acquire) { + return Err(fault("投屏连接已取消。")); + } + let attempt = (|| { + let mut control = Rpc::connect(port)?; + control.driver = "Driver#0".into(); + let driver = control.driver("Driver.create", json!([]))?; + Ok((control, driver)) + })(); + match attempt { + Ok(result) => break result, + Err(error) if Instant::now() >= deadline => return Err(error), + Err(_) => thread::sleep(Duration::from_millis(100)), + } + }; + control.driver = driver + .as_str() + .filter(|s| s.starts_with("Driver#") && s.len() < 64) + .ok_or_else(|| fault("设备未能启动投屏控制组件。"))? + .into(); + let dimensions = control.dimensions()?; + let mut video = Rpc::connect(port)?; + video.send( + "Captures", + json!({"api":"startCaptureScreen","args":{"options":{"displayId":0,"scale":0.5}}}), + )?; + video + .socket + .set_read_timeout(Some(Duration::from_millis(100))) + .map_err(io)?; + let socket = video.socket.try_clone().map_err(io)?; + let rpc = Arc::new(Mutex::new(control)); + let shared = Arc::new(Shared { + state: Mutex::new(State { + latest: None, + error: None, + touched: Instant::now(), + samples: VecDeque::new(), + bytes: VecDeque::new(), + }), + changed: Condvar::new(), + cancel: Arc::new(AtomicBool::new(false)), + }); + let worker_shared = shared.clone(); + let worker_rpc = rpc.clone(); + let parent_cancel = tool.cancel.clone(); + let worker = thread::spawn(move || { + let result = receive( + &mut video, + &worker_rpc, + &worker_shared, + dimensions, + &parent_cancel, + ); + if let Err(error) = result { + worker_shared.state.lock().unwrap().error = Some(error); + } + worker_shared.cancel.store(true, Ordering::Release); + worker_shared.changed.notify_all(); + let _ = video.send("Captures", json!({"api":"stopCaptureScreen","args":{}})); + let _ = video.socket.shutdown(Shutdown::Both); + let _ = worker_rpc.lock().unwrap().socket.shutdown(Shutdown::Both); + drop(resources); + }); + Ok(Self { + shared, + rpc, + socket, + worker: Mutex::new(Some(worker)), + }) + } + pub fn frame(&self, after: u64) -> Result<(Frame, f64, f64)> { + let mut state = self.shared.state.lock().unwrap(); + state.touched = Instant::now(); + let timeout = if state.latest.is_none() { + Duration::from_secs(5) + } else { + Duration::from_millis(100) + }; + let (state, _) = self + .shared + .changed + .wait_timeout_while(state, timeout, |s| { + s.error.is_none() + && !self.shared.cancel.load(Ordering::Acquire) + && s.latest.as_ref().is_none_or(|f| f.sequence <= after) + }) + .unwrap(); + if let Some(error) = &state.error { + return Err(error.clone()); + } + if self.shared.cancel.load(Ordering::Acquire) { + return Err(fault("投屏连接已结束，请重新开始。")); + } + let frame = state + .latest + .clone() + .ok_or_else(|| fault("手机没有返回画面，请解锁手机并检查 USB 调试授权。"))?; + let recent = |time: &&Instant| time.elapsed() < Duration::from_secs(1); + let fps = state.samples.iter().filter(recent).count() as f64; + let mbps = state + .bytes + .iter() + .filter(|(t, _)| t.elapsed() < Duration::from_secs(1)) + .map(|(_, n)| *n as f64) + .sum::() + * 8.0 + / 1_000_000.0; + Ok((frame, fps, mbps)) + } + pub fn current(&self) -> Result { + let state = self.shared.state.lock().unwrap(); + if let Some(error) = &state.error { + return Err(error.clone()); + } + if self.shared.cancel.load(Ordering::Acquire) { + return Err(fault("投屏连接已结束。")); + } + state + .latest + .clone() + .ok_or_else(|| fault("请等待手机画面。")) + } + pub fn driver(&self, api: &str, args: Value) -> Result { + self.current()?; + self.rpc.lock().unwrap().driver(api, args) + } + pub fn stop(&self) { + self.shared.cancel.store(true, Ordering::Release); + self.shared.changed.notify_all(); + let _ = self.socket.shutdown(Shutdown::Read); + if let Some(worker) = self.worker.lock().unwrap().take() { + let _ = worker.join(); + } + } +} +impl Drop for Stream { + fn drop(&mut self) { + self.stop(); + } +} +fn receive( + video: &mut Rpc, + control: &Mutex, + shared: &Shared, + mut dimensions: (u32, u32), + parent_cancel: &AtomicBool, +) -> Result<()> { + let mut buffer = [0; 65536]; + let mut heartbeat = Instant::now(); + let mut encoded_size = None; + loop { + if shared.cancel.load(Ordering::Acquire) || parent_cancel.load(Ordering::Acquire) { + return Ok(()); + } + if shared.state.lock().unwrap().touched.elapsed() > LEASE { + return Err(fault("投屏窗口已停止响应，连接已关闭。")); + } + if heartbeat.elapsed() >= Duration::from_secs(1) { + dimensions = control.lock().unwrap().dimensions()?; + heartbeat = Instant::now(); + } + match video.socket.read(&mut buffer) { + Ok(0) => return Err(fault("手机已断开投屏连接。")), + Ok(n) => video.decoder.push(&buffer[..n])?, + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + continue + } + Err(e) => return Err(io(e)), + } + while let Some(packet) = video.decoder.next()? { + let jpeg = match packet { + Packet::Json(v) => { + response(v)?; + continue; + } + Packet::Jpeg(bytes) => bytes, + }; + let size = ImageReader::with_format(Cursor::new(&jpeg), ImageFormat::Jpeg) + .into_dimensions() + .map_err(|_| fault("设备返回了无效 JPEG 画面。"))?; + if size.0 == 0 || size.1 == 0 || size.0 > 8192 || size.1 > 8192 { + return Err(fault("画面尺寸超过支持范围。")); + } + if encoded_size.is_some_and(|previous| previous != size) { + dimensions = control.lock().unwrap().dimensions()?; + } + encoded_size = Some(size); + // A display change must be observed before coordinates can be used. + if dimensions.0 / 2 != size.0 || dimensions.1 / 2 != size.1 { + dimensions = control.lock().unwrap().dimensions()?; + } + if dimensions.0 / 2 != size.0 || dimensions.1 / 2 != size.1 { + return Err(fault("手机画面与屏幕尺寸不一致，请重新开始投屏。")); + } + let now = Instant::now(); + let mut state = shared.state.lock().unwrap(); + let sequence = state.latest.as_ref().map_or(1, |f| f.sequence + 1); + let geometry = state.latest.as_ref().map_or(1, |f| { + f.geometry + u64::from((f.width, f.height) != dimensions) + }); + state.samples.push_back(now); + state.bytes.push_back((now, jpeg.len())); + while state + .samples + .front() + .is_some_and(|t| t.elapsed() > Duration::from_secs(1)) + { + state.samples.pop_front(); + } + while state + .bytes + .front() + .is_some_and(|(t, _)| t.elapsed() > Duration::from_secs(1)) + { + state.bytes.pop_front(); + } + state.latest = Some(Frame { + jpeg: Arc::new(jpeg), + width: dimensions.0, + height: dimensions.1, + sequence, + geometry, + received: now, + }); + shared.changed.notify_all(); + } + } +} diff --git a/curios/harmony-connector/native/src/hdc/screen_input.rs b/curios/harmony-connector/native/src/hdc/screen_input.rs index 36aa7f9..807c255 100644 --- a/curios/harmony-connector/native/src/hdc/screen_input.rs +++ b/curios/harmony-connector/native/src/hdc/screen_input.rs @@ -1,5 +1,6 @@ use crate::model::{Fault, Result}; use serde::Deserialize; +use serde_json::{json, Value}; use std::collections::HashSet; #[derive(Deserialize)] @@ -96,6 +97,34 @@ pub(super) fn capabilities(help: &str) -> HashSet { .collect() } impl Input { + pub fn rpc(&self) -> Option<(&'static str, Value)> { + Some(match self { + Self::Click { x, y } => ("Driver.click", json!([x, y])), + Self::DoubleClick { x, y } => ("Driver.doubleClick", json!([x, y])), + Self::LongClick { x, y } => ("Driver.longClick", json!([x, y])), + Self::Swipe { + x, + y, + to_x, + to_y, + speed, + } => ("Driver.swipe", json!([x, y, to_x, to_y, speed])), + Self::Drag { + x, + y, + to_x, + to_y, + speed, + } => ("Driver.drag", json!([x, y, to_x, to_y, speed])), + Self::Key { key: Key::Home } => ("Driver.pressHome", json!([])), + Self::Key { key: Key::Back } => ("Driver.pressBack", json!([])), + Self::Key { key } => ( + "Driver.triggerKey", + json!([key.code().parse::().ok()?]), + ), + Self::Text { .. } => return None, + }) + } pub fn command(&self, width: u32, height: u32, supported: &HashSet) -> Result { let point = |x, y| { if x < width && y < height { diff --git a/curios/harmony-connector/native/src/hdc/screen_protocol.rs b/curios/harmony-connector/native/src/hdc/screen_protocol.rs new file mode 100644 index 0000000..21aa9ed --- /dev/null +++ b/curios/harmony-connector/native/src/hdc/screen_protocol.rs @@ -0,0 +1,243 @@ +//! Hypium agent wire format. JPEG segments are parsed by length, so EXIF/ICC +//! bytes containing an EOI marker cannot truncate a frame. +use crate::model::{Fault, Result}; +use serde_json::Value; + +const HEAD: &[u8] = b"_uitestkit_rpc_message_head_"; +const TAIL: &[u8] = b"_uitestkit_rpc_message_tail_"; +pub const MAX_FRAME: usize = 2 * 1024 * 1024; +const MAX_JSON: usize = 64 * 1024; + +fn invalid() -> Fault { + Fault::new( + "screen.invalid_stream", + "设备返回了无效的投屏数据，连接已停止。", + ) +} +pub fn request(body: &Value, sequence: u32) -> Vec { + let body = serde_json::to_vec(body).unwrap(); + let mut bytes = Vec::with_capacity(HEAD.len() + body.len() + TAIL.len() + 8); + bytes.extend_from_slice(HEAD); + bytes.extend_from_slice(&sequence.to_be_bytes()); + bytes.extend_from_slice(&(body.len() as u32).to_be_bytes()); + bytes.extend_from_slice(&body); + bytes.extend_from_slice(TAIL); + bytes +} +#[derive(Debug)] +pub enum Packet { + Json(Value), + Jpeg(Vec), +} +#[derive(Default)] +pub struct Decoder { + bytes: Vec, +} +impl Decoder { + pub fn push(&mut self, bytes: &[u8]) -> Result<()> { + if self.bytes.len() + bytes.len() > MAX_FRAME + 65536 { + return Err(invalid()); + } + self.bytes.extend_from_slice(bytes); + Ok(()) + } + pub fn next(&mut self) -> Result> { + let whitespace = self + .bytes + .iter() + .take_while(|b| matches!(b, b'\n' | b'\r')) + .count(); + self.bytes.drain(..whitespace); + if self.bytes.is_empty() { + return Ok(None); + } + let (start, length, consumed) = if self.bytes.starts_with(HEAD) { + let start = HEAD.len() + 8; + if self.bytes.len() < start { + return Ok(None); + } + let length = + u32::from_be_bytes(self.bytes[start - 4..start].try_into().unwrap()) as usize; + if length == 0 || length > MAX_FRAME { + return Err(invalid()); + } + let consumed = start + length + TAIL.len(); + if self.bytes.len() < consumed { + return Ok(None); + } + if &self.bytes[start + length..consumed] != TAIL { + return Err(invalid()); + } + (start, length, consumed) + } else if self.bytes.starts_with(b"{") { + let Some(end) = self.bytes.iter().position(|b| *b == b'\n') else { + if self.bytes.len() > MAX_JSON { + return Err(invalid()); + } + return Ok(None); + }; + (0, end, end + 1) + } else if self.bytes.starts_with(b"\xff\xd8") { + let Some(length) = jpeg_length(&self.bytes)? else { + if self.bytes.len() > MAX_FRAME { + return Err(invalid()); + } + return Ok(None); + }; + (0, length, length) + } else if HEAD.starts_with(&self.bytes) || b"\xff\xd8".starts_with(&self.bytes) { + return Ok(None); + } else { + return Err(invalid()); + }; + if length > MAX_FRAME { + return Err(invalid()); + } + let body = &self.bytes[start..start + length]; + let packet = if body.starts_with(b"\xff\xd8") { + if jpeg_length(body)? != Some(body.len()) { + return Err(invalid()); + } + Packet::Jpeg(body.to_vec()) + } else { + if body.len() > MAX_JSON { + return Err(invalid()); + } + Packet::Json(serde_json::from_slice(body).map_err(|_| invalid())?) + }; + self.bytes.drain(..consumed); + Ok(Some(packet)) + } +} +fn jpeg_length(bytes: &[u8]) -> Result> { + let mut position = 2; + let mut entropy = false; + loop { + if entropy { + while position < bytes.len() && bytes[position] != 0xff { + position += 1; + } + } + if position >= bytes.len() { + return Ok(None); + } + if bytes[position] != 0xff { + return Err(invalid()); + } + while position < bytes.len() && bytes[position] == 0xff { + position += 1; + } + if position >= bytes.len() { + return Ok(None); + } + let marker = bytes[position]; + position += 1; + if entropy && (marker == 0 || (0xd0..=0xd7).contains(&marker)) { + continue; + } + if marker == 0xd9 { + return Ok(Some(position)); + } + if marker == 0 || marker == 0xd8 { + return Err(invalid()); + } + if marker == 1 { + continue; + } + if bytes.len() < position + 2 { + return Ok(None); + } + let length = u16::from_be_bytes(bytes[position..position + 2].try_into().unwrap()) as usize; + if length < 2 { + return Err(invalid()); + } + if bytes.len() < position + length { + return Ok(None); + } + position += length; + entropy = marker == 0xda; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use image::codecs::jpeg::JpegEncoder; + use serde_json::json; + fn jpeg() -> Vec { + let mut bytes = Vec::new(); + JpegEncoder::new(&mut bytes) + .encode_image(&image::RgbImage::new(40, 80)) + .unwrap(); + bytes.splice(2..2, [0xff, 0xe1, 0, 6, 0xff, 0xd9, 0, 0]); + bytes + } + #[test] + fn decodes_fragmented_mixed_responses_and_real_jpegs_without_truncating_exif() { + let jpeg = jpeg(); + let bytes = [ + b"{\"result\":true}\n".as_slice(), + &jpeg, + b"\n", + &jpeg, + b"\n", + ] + .concat(); + for chunk_size in [1, 7, 512, 65536] { + let mut decoder = Decoder::default(); + let mut images = 0; + let mut responses = 0; + for chunk in bytes.chunks(chunk_size) { + decoder.push(chunk).unwrap(); + while let Some(packet) = decoder.next().unwrap() { + match packet { + Packet::Json(v) => { + assert_eq!(v["result"], true); + responses += 1; + } + Packet::Jpeg(bytes) => { + assert_eq!(bytes, jpeg); + assert_eq!(image::load_from_memory(&bytes).unwrap().height(), 80); + images += 1; + } + } + } + } + assert_eq!((responses, images), (1, 2)); + } + } + #[test] + fn rejects_oversized_malformed_and_corrupt_framed_data() { + let mut decoder = Decoder::default(); + assert!(decoder.push(&vec![0; MAX_FRAME + 65537]).is_err()); + for bytes in [ + b"not a frame".to_vec(), + b"{invalid}\n".to_vec(), + b"\xff\xd8\xff\xe1\0\0".to_vec(), + [ + HEAD, + &0u32.to_be_bytes(), + &((MAX_FRAME + 1) as u32).to_be_bytes(), + ] + .concat(), + ] { + let mut decoder = Decoder::default(); + decoder.push(&bytes).unwrap(); + assert!(decoder.next().is_err()); + } + let mut bytes = request(&json!({"result":true}), 3); + *bytes.last_mut().unwrap() = b'!'; + let mut decoder = Decoder::default(); + decoder.push(&bytes).unwrap(); + assert!(decoder.next().is_err()); + } + #[test] + fn accepts_framed_json_and_waits_for_partial_data() { + let bytes = request(&json!({"result":{"x":1260,"y":2844}}), 7); + let mut decoder = Decoder::default(); + decoder.push(&bytes[..bytes.len() - 1]).unwrap(); + assert!(decoder.next().unwrap().is_none()); + decoder.push(&bytes[bytes.len() - 1..]).unwrap(); + assert!(matches!(decoder.next().unwrap(), Some(Packet::Json(_)))); + } +} diff --git a/curios/harmony-connector/native/src/hdc/screen_tests.rs b/curios/harmony-connector/native/src/hdc/screen_tests.rs index fa0006a..ed85df7 100644 --- a/curios/harmony-connector/native/src/hdc/screen_tests.rs +++ b/curios/harmony-connector/native/src/hdc/screen_tests.rs @@ -3,29 +3,25 @@ use base64::{engine::general_purpose::STANDARD, Engine}; fn phone() -> (tempfile::TempDir, Arc) { let (directory, manager) = fixture(false); - fs::write(directory.path().join("hdc"), r#"#!/bin/sh -printf '%s\n' "$*" >> "$0.calls" -case "$*" in - 'list targets -v') printf 'phone USB Connected device\n' ;; - '-t phone shell uitest help') printf 'screenCap uiInput\n' ;; - '-t phone shell uitest uiInput help') printf 'click doubleClick longClick swipe drag text keyEvent\n' ;; - '-t phone shell uitest screenCap -p '*) - if [ -f "$0.slow" ]; then touch "$0.capturing"; sleep 20; fi - printf 'Screen capture saved\n' ;; - '-t phone file recv '*) cp "$0.png" "$6" ;; - '-t phone shell rm -f /data/local/tmp/harmony-screen-'*) printf 'cleaned\n' ;; - '-t phone shell uitest uiInput '*) - if [ -f "$0.fail-input" ]; then printf '[Fail] input not permitted\n'; else printf 'completed\n'; fi ;; - *) printf '[Fail] unexpected invocation\n' ;; -esac -"#).unwrap(); + fs::write( + directory.path().join("hdc"), + include_str!("../../tests/hypium_device.py"), + ) + .unwrap(); frame_file(&directory, 600, 1200, [30, 80, 120]); (directory, manager) } fn frame_file(directory: &tempfile::TempDir, width: u32, height: u32, rgb: [u8; 3]) { - image::RgbImage::from_pixel(width, height, image::Rgb(rgb)) - .save(directory.path().join("hdc.png")) + fs::write( + directory.path().join("hdc.dimensions"), + serde_json::to_vec(&json!({"x":width,"y":height})).unwrap(), + ) + .unwrap(); + let temporary = directory.path().join("frame.jpg"); + image::RgbImage::from_pixel(width / 2, height / 2, image::Rgb(rgb)) + .save(&temporary) .unwrap(); + fs::rename(temporary, directory.path().join("hdc.jpg")).unwrap(); } fn start(manager: &Arc) -> Value { let opened = manager @@ -67,9 +63,14 @@ fn shows_real_decodable_frames_and_saves_latest_without_overwriting() { .unwrap() }; let jpeg = decode(&first); - assert_eq!(image::load_from_memory(&jpeg).unwrap().height(), 1200); + assert_eq!(image::load_from_memory(&jpeg).unwrap().height(), 600); frame_file(&directory, 600, 1200, [220, 60, 30]); - let next = manager.call("screenFrame", id.clone()).unwrap(); + let next = manager + .call( + "screenFrame", + json!({"session":id["session"],"after":first["sequence"]}), + ) + .unwrap(); assert!(next["sequence"].as_u64() > first["sequence"].as_u64()); assert_ne!(decode(&next), jpeg); let destination = tempfile::tempdir().unwrap(); @@ -110,7 +111,12 @@ fn only_explicit_control_of_current_geometry_can_send_input() { true ); frame_file(&directory, 1200, 600, [30, 80, 120]); - let rotated = manager.call("screenFrame", id.clone()).unwrap(); + let rotated = manager + .call( + "screenFrame", + json!({"session":id["session"],"after":frame["sequence"]}), + ) + .unwrap(); assert_eq!(rotated["control"], false); enable(&manager, &id, &rotated); assert_eq!( @@ -121,7 +127,7 @@ fn only_explicit_control_of_current_geometry_can_send_input() { assert_eq!( calls .lines() - .filter(|line| line.contains("uiInput click")) + .filter(|line| line.contains("Driver.click")) .count(), 1 ); @@ -143,8 +149,16 @@ fn rejects_out_of_bounds_unsupported_and_old_events_without_sending_them() { .call("screenInput", input(&id, &first, event)) .is_err()); } - for _ in 0..2 { - manager.call("screenFrame", id.clone()).unwrap(); + let mut after = first["sequence"].clone(); + for red in [110, 120] { + frame_file(&directory, 600, 1200, [red, 80, 120]); + after = manager + .call( + "screenFrame", + json!({"session":id["session"],"after":after}), + ) + .unwrap()["sequence"] + .clone(); } assert_eq!( manager @@ -236,10 +250,16 @@ fn invalid_capture_closes_the_session_and_remote_input_failure_disables_control( manager.call("screenFrame", id.clone()).unwrap()["control"], false ); - fs::write(directory.path().join("hdc.png"), b"success but no image").unwrap(); + fs::write(directory.path().join("hdc.jpg"), b"success but no image").unwrap(); assert_eq!( - manager.call("screenFrame", id.clone()).unwrap_err().code, - "screen.invalid_frame" + manager + .call( + "screenFrame", + json!({"session":id["session"],"after":frame["sequence"]}) + ) + .unwrap_err() + .code, + "screen.invalid_stream" ); assert_eq!( manager @@ -252,3 +272,52 @@ fn invalid_capture_closes_the_session_and_remote_input_failure_disables_control( "screen.closed" ); } + +#[test] +fn static_stream_remains_connected_without_inventing_frames() { + let (_directory, manager) = phone(); + let id = start(&manager); + let first = manager.call("screenFrame", id.clone()).unwrap(); + enable(&manager, &id, &first); + for _ in 0..12 { + let next = manager + .call( + "screenFrame", + json!({"session":id["session"],"after":first["sequence"]}), + ) + .unwrap(); + assert_eq!(next["unchanged"], true); + assert_eq!(next["sequence"], first["sequence"]); + assert!(next.get("dataUrl").is_none()); + assert_eq!(next["control"], true); + } + manager + .call( + "screenInput", + input(&id, &first, json!({"kind":"key","key":"home"})), + ) + .unwrap(); + manager.call("screenStop", id).unwrap(); +} + +#[test] +fn refuses_to_take_over_another_debugging_session() { + let (directory, manager) = phone(); + fs::write( + directory.path().join("hdc.daemon"), + "another-tools-agent.so", + ) + .unwrap(); + assert_eq!( + manager + .call("screenStart", json!({"target":"phone"})) + .unwrap_err() + .code, + "screen.debugger_busy" + ); + let calls = fs::read_to_string(directory.path().join("hdc.calls")).unwrap(); + assert!( + !calls.contains("file send") && !calls.contains("shell kill") && !calls.contains("fport") + ); + assert!(directory.path().join("hdc.daemon").exists()); +} diff --git a/curios/harmony-connector/native/src/main.rs b/curios/harmony-connector/native/src/main.rs index be14190..f0d43ca 100644 --- a/curios/harmony-connector/native/src/main.rs +++ b/curios/harmony-connector/native/src/main.rs @@ -21,7 +21,10 @@ fn main() -> anyhow::Result<()> { } let args: Vec<_> = std::env::args().collect(); if args.get(1).is_some_and(|s| s == "--check-screen") { - return diagnostics::screen(&args[2..]); + return diagnostics::screen(&args[2..], false); + } + if args.get(1).is_some_and(|s| s == "--check-screen-stream") { + return diagnostics::screen(&args[2..], true); } let _ = mtp::version(); wire::run(connector::Connector::new()) diff --git a/curios/harmony-connector/native/tests/hypium_device.py b/curios/harmony-connector/native/tests/hypium_device.py new file mode 100644 index 0000000..68f39e4 --- /dev/null +++ b/curios/harmony-connector/native/tests/hypium_device.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Local test peer for the public HDC and Hypium protocol. Never contacts USB.""" +import json +import os +from pathlib import Path +import signal +import socket +import struct +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).parent +HEAD = b'_uitestkit_rpc_message_head_' +TAIL = b'_uitestkit_rpc_message_tail_' +def path(name): return ROOT / ('hdc.' + name) +def log(value): + with path('calls').open('a') as f: f.write(value + '\n') +def exact(sock, length): + data = bytearray() + while len(data) < length: + chunk = sock.recv(length-len(data)) + if not chunk: raise EOFError() + data.extend(chunk) + return bytes(data) +def client(sock): + done = threading.Event() + write_lock = threading.Lock() + def send(data): + with write_lock: sock.sendall(data) + def video(): + previous = None + try: + while not done.wait(0.01) and ROOT.exists(): + if path('slow').exists(): + path('capturing').touch() + continue + jpeg = path('jpg').read_bytes() + if jpeg != previous: + send(jpeg+b'\n') + previous=jpeg + except (OSError, EOFError): pass + try: + while True: + assert exact(sock,len(HEAD)) == HEAD + _,length=struct.unpack('>II',exact(sock,8)) + assert 0 < length <= 65536 + message=json.loads(exact(sock,length)) + assert exact(sock,len(TAIL)) == TAIL + params=message['params']; api=params['api'] + log(json.dumps({'api':api,'args':params['args']})) + result=None + if api=='Driver.create': result='Driver#fixture' + elif api=='Driver.getDisplaySize': result=json.loads(path('dimensions').read_text()) + elif api=='startCaptureScreen': result=True + elif api=='stopCaptureScreen': result=True;done.set() + elif path('fail-input').exists(): + send(b'{"exception":{"code":17000005,"message":"input not permitted"}}\n') + continue + send(json.dumps({'result':result}).encode()+b'\n') + if api=='startCaptureScreen': threading.Thread(target=video,daemon=True).start() + except (OSError,EOFError): pass + finally: done.set();sock.close() +def serve(port): + with socket.socket() as listener: + listener.bind(('127.0.0.1',port));listener.listen() + path('ready').touch() + while True: + sock,_=listener.accept() + threading.Thread(target=client,args=(sock,),daemon=True).start() +if sys.argv[1:2]==['server']: + serve(int(sys.argv[2]));sys.exit() +args=sys.argv[1:] +log(' '.join(args)) +if args[:2]==['-t','phone']: args=args[2:] +if args==['list','targets','-v']: print('phone USB Connected device') +elif args==['shell','uitest uiInput help']: print('click doubleClick longClick swipe drag text keyEvent') +elif args==['shell','pidof uitest']: + if path('daemon').exists(): print('4242') +elif args[:2]==['file','send']: print('FileTransfer finish') +elif args[0]=='shell' and args[1].startswith('uitest start-daemon'): + path('daemon').write_text(args[1].split()[-1]) +elif args[0]=='shell' and args[1]=='cat /proc/4242/cmdline': + sys.stdout.write('uitest\0start-daemon\0'+path('daemon').read_text()+'\0') +elif args[:2]==['shell','kill 4242']: path('daemon').unlink(missing_ok=True) +elif args[0]=='shell' and args[1].startswith('rm -f /data/local/tmp/'): pass +elif args[0]=='shell' and args[1].startswith('uitest uiInput text '): print('completed') +elif args[:2]==['fport','rm']: + pid=int(path('pid').read_text());os.kill(pid,signal.SIGTERM) + path('pid').unlink();path('ready').unlink(missing_ok=True) + print('Remove forward ruler success') +elif args[0]=='fport': + process=subprocess.Popen([sys.executable,__file__,'server',args[1].split(':')[1]], + stdin=subprocess.DEVNULL,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True) + path('pid').write_text(str(process.pid)) + until=time.monotonic()+3 + while not path('ready').exists(): + if time.monotonic()>until: raise RuntimeError('Fixture listener not ready') + time.sleep(0.01) + print('Forwardport result:OK') +else: print('[Fail] unexpected invocation') diff --git a/curios/harmony-connector/ui/screen.js b/curios/harmony-connector/ui/screen.js index 9925d40..3abcbd3 100644 --- a/curios/harmony-connector/ui/screen.js +++ b/curios/harmony-connector/ui/screen.js @@ -24,6 +24,7 @@ export function screen({ call, host, notice, onBusy }) { pending: null, timer: null, flight: null, + inputFlight: null, pointer: null, wheel: null, wheelTimer: null, @@ -88,7 +89,8 @@ export function screen({ call, host, notice, onBusy }) { $("screen-placeholder").hidden = false; $("screen-stats").textContent = ""; $("screen-text").value = ""; - $("screen-input-status").textContent = "默认仅观看，启用控制后才会发送操作。"; + $("screen-input-status").textContent = + "默认仅观看，启用控制后才会发送操作。"; } async function stop() { const session = state.session; @@ -104,14 +106,14 @@ export function screen({ call, host, notice, onBusy }) { } catch (error) { if (!state.disposed) notice(error.message || String(error)); } finally { - await state.flight?.catch(() => {}); + await Promise.allSettled([state.flight, state.inputFlight]); state.stopping = false; if (state.disposed) return; if (!state.starting) onBusy(false); controls(); } } - function schedule(delay = 100) { + function schedule(delay = 0) { clearTimeout(state.timer); if (state.active && !state.disposed) state.timer = setTimeout(pump, delay); } @@ -119,31 +121,23 @@ export function screen({ call, host, notice, onBusy }) { if (!state.active || state.flight || state.disposed) return; const revision = state.revision; state.flight = (async () => { - if (state.pending) { - const pending = state.pending; - state.pending = null; - if (Date.now() - pending.created <= 1500 && state.control) { - const before = performance.now(); - try { - await call("screenInput", pending.request); - if (revision === state.revision && !state.disposed) - $("screen-input-status").textContent = - `操作已发送 · ${Math.round(performance.now() - before)} ms`; - } catch (error) { - if (revision !== state.revision || state.disposed) return; - state.control = false; - clearPending(); - controls(); - notice(error.message || String(error)); - } - } else - $("screen-input-status").textContent = - "画面响应较慢，已丢弃等待过久的操作。"; - } if (!state.active || revision !== state.revision) return; const controlRevision = state.controlRevision; - const frame = await call("screenFrame", { session: state.session }); + const frame = await call("screenFrame", { + session: state.session, + after: state.frame?.sequence || 0, + }); if (!state.active || revision !== state.revision) return; + if (frame.unchanged) { + if (!state.frame || frame.geometry !== state.frame.geometry) + throw new Error("手机画面状态无效。"); + state.displayedAt = Date.now(); + if (controlRevision === state.controlRevision) + state.control = !!frame.control; + updateStats(frame); + controls(); + return; + } if ( !/^data:image\/jpeg;base64,/.test(frame.dataUrl) || !Number.isInteger(frame.width) || @@ -155,6 +149,8 @@ export function screen({ call, host, notice, onBusy }) { const image = $("screen-image"); image.src = frame.dataUrl; if (image.decode) await image.decode(); + if (window.requestAnimationFrame) + await new Promise((resolve) => window.requestAnimationFrame(resolve)); if (!state.active || revision !== state.revision) { image.removeAttribute("src"); return; @@ -172,14 +168,12 @@ export function screen({ call, host, notice, onBusy }) { state.control = !!frame.control; state.displayedAt = Date.now(); state.samples.push(performance.now()); - if (state.samples.length > 8) state.samples.shift(); - const elapsed = state.samples.at(-1) - state.samples[0]; - const fps = - elapsed > 0 - ? (((state.samples.length - 1) * 1000) / elapsed).toFixed(1) - : "—"; - $("screen-stats").textContent = - `${frame.width} × ${frame.height} · ${fps} 帧/秒 · 采集 ${Math.round(frame.captureMs)} ms`; + while ( + state.samples.length && + performance.now() - state.samples[0] > 1000 + ) + state.samples.shift(); + updateStats(frame); image.hidden = false; $("screen-placeholder").hidden = true; controls(); @@ -196,7 +190,46 @@ export function screen({ call, host, notice, onBusy }) { if (failure) { notice(failure.message || String(failure)); await stop(); - } else schedule(state.pending ? 0 : 100); + } else schedule(0); + } + function updateStats(frame) { + const received = Number.isFinite(frame.receivedFps) + ? frame.receivedFps.toFixed(0) + : "—"; + const displayed = state.samples.filter( + (t) => performance.now() - t <= 1000, + ).length; + const bitrate = Number.isFinite(frame.mbps) + ? ` · ${frame.mbps.toFixed(1)} Mbps` + : ""; + $("screen-stats").textContent = + `${frame.width} × ${frame.height} · 接收 ${received} 帧/秒 · 显示 ${displayed} 帧/秒${bitrate}`; + } + async function flushInput() { + if (state.inputFlight || !state.pending || !canControl()) return; + const pending = state.pending; + state.pending = null; + if (Date.now() - pending.created > 1500) return; + const revision = state.revision; + const before = performance.now(); + state.inputFlight = call("screenInput", pending.request); + try { + await state.inputFlight; + if (revision === state.revision && !state.disposed) + $("screen-input-status").textContent = + `操作已发送 · ${Math.round(performance.now() - before)} ms`; + } catch (error) { + if (revision === state.revision && !state.disposed) { + state.controlRevision++; + state.control = false; + clearPending(); + controls(); + notice(error.message || String(error)); + } + } finally { + state.inputFlight = null; + if (revision === state.revision && !state.disposed) void flushInput(); + } } async function start() { if ( @@ -272,7 +305,7 @@ export function screen({ call, host, notice, onBusy }) { }, }; $("screen-input-status").textContent = "等待发送操作…"; - if (!state.flight) schedule(0); + void flushInput(); } function point(event) { return state.frame diff --git a/docs/harmony-connector.md b/docs/harmony-connector.md index 2f073a9..4ff0603 100644 --- a/docs/harmony-connector.md +++ b/docs/harmony-connector.md @@ -67,15 +67,15 @@ macOS 上的鸿蒙设备连接工具。通过 USB 收发文件与完整文件夹 在「设备与应用」选择已授权的 HDC 目标，点击「开始投屏」。默认仅观看；画面出现后勾选「启用鼠标与键盘控制」。功能依设备的 `uitest` 帮助和实际操作结果开放。 -- 连续显示手机画面，显示真实分辨率、实测刷新帧率和采集耗时。可放大面板，将当前画面保存为 JPEG。 +- 连续显示手机画面，显示真实分辨率、接收/显示帧率和码率。可放大面板，将当前画面保存为 JPEG。 - 点击、双击、长按、滑动、拖拽与滚轮滚动；坐标按实际手机尺寸映射，忽略画面周围留白。 - 返回、主页、音量、方向键、回车、退格等按键。快捷键只在手机画面获得焦点时生效。 - 用专用文字框发送文本，支持输入法；不读取系统剪贴板。文字能否进入目标输入框，以手机焦点和输入权限为准。 - 一个屏幕会话只保留一项待发送操作。停止、切换工作区、页面隐藏、断线时清空队列和旧画面，不在重连后重放输入。横竖屏变化后需重新启用控制。 -当前实现通过 HDC + UITest 连续采集截图，完成一帧再开始下一帧；不是 H.264/H.265 视频流，不承诺 scrcpy 的 30/60 FPS、音频或低延迟多点触控。拖动在松开时发送完整手势。采集临时文件使用每次唯一名称并清理；远端清理被断线阻断时，不扫描删除其他文件。 +当前源代码通过 HDC 部署官方 Hypium 代理，持续接收 JPEG 画面，并通过独立连接发送输入。用户无需开发者账号、编译、签名或安装手机应用。拖动在松开时发送完整手势。停止会删除本会话的代理和转发；其他调试工具占用设备时会提示。已发布 0.3.2 仍为逐帧截图，新路径的动态真机验收状态见[开发验证](harmony-realtime-development.md)。 -图像解码限制内存与尺寸，传给 UI 的 JPEG 最长边不超过 1600 像素；坐标仍按手机原始尺寸计算。画面过旧、方向变化、会话过期或输入失败时会阻止控制。受保护画面和锁屏遵循系统限制，不自动解锁。自动化测试和真机结果分别记录，详见[投屏能力调研](research/harmony-screen-control.md)。 +JPEG 采用半尺寸采集，坐标按手机原始尺寸计算。画面不变化时保留显示，通过独立控制连接检查存活；方向变化、会话过期或输入失败时会阻止控制。受保护画面和锁屏遵循系统限制，不自动解锁。自动化测试和真机结果分别记录，详见[HDC 持续投屏调研](research/harmony-hdc-only-screen.md)。 ## 重连问题与 debug diff --git a/docs/harmony-realtime-development.md b/docs/harmony-realtime-development.md index b9a6405..ff0868d 100644 --- a/docs/harmony-realtime-development.md +++ b/docs/harmony-realtime-development.md @@ -1,78 +1,40 @@ -# Harmony Connector 连续视频开发验证 +# Harmony Connector 持续投屏开发验证 -当前为待真机验证的开发组件，尚未接入插件播放器，也未发布新插件版本。已发布的 0.3.2 仍使用逐帧截图。不要将本篇的编码配置或本机生成视频测试结果描述成手机实测性能。 +运行路径为：已授权的 HDC USB 连接 → 官方 Hypium 设备代理 → 持续 JPEG 推送 → 插件显示。用户无需开发者账号、AppGallery Connect、HAP 安装、编译或签名。代理由插件自动部署到本次会话的临时文件，停止时删除。 -## 已完成与待验证 +## 当前状态 -2026-09-10 的原链路真机采集耗时为 696、803、802 ms，约 1.3 fps。截图命令、PNG 文件回传和图像转换是逐帧执行的；调整界面定时器不足以实现实时视频。 +2026-09-10：已发布的 0.3.2 仍使用逐张截图，真机原链路为 696、803、802 ms，约 1.3 fps。当前源代码已替换该路径，尚未发布新版本。 -新增源码： +官方 `xdevice-devicetest` 6.1.0.210 的 `uitest_agent_v1.2.2.so` 已在 HUAWEI Pura 70 Ultra 的商业系统上成功加载。独立探针收到真实 JPEG、屏幕尺寸 1260 × 2844，并成功调用尺寸、旋转和唤醒接口。设备端文件为 600245 字节，未改动签名原件。没有安装 HAP，也没有账号或系统共享确认步骤。 -- `curios/harmony-connector/companion`：官方 Empty Ability 模板上的普通手机应用。用户点击共享后，经系统授权，将 AVScreenCapture Surface 接到 H.264 编码器。初始配置最长边 1280、30 fps、4 Mbps CBR、Baseline、1 秒关键帧间隔，不采集音频。 -- `tools/harmony-stream.py`：独立 Mac 开发验证程序，建立本会话 HDC 转发、启动组件、接收与解码视频，输出接收帧率、解码帧数和码率。默认不保存画面文件、不发送输入。`--view` 用于人工观看。 -- `tools/harmony-stream.test.py`：协议尺寸与长度限制、断流、超时、非 H.264 数据，以及真正经过 FFmpeg 编码与解码的 20 帧生成视频测试。 +探测期间手机主要处于锁屏或静止状态。黑色首帧、短暂变化及收包时间不能证明动态帧率或端到端延迟达标。接入生产后端时手机变为 Offline，后端的完整真机连通、持续动态画面、输入及重连仍待验收。自动化替身测试不替代这项验收。完整自动化检查通过 85 项：47 项原生、32 项 UI/格式/坐标、6 项 USB 桥接与底座协议检查。 -已使用官方 Command Line Tools 26.0.0.821 编译出 arm64-v8a 未签名 HAP，包含 ArkTS 和本地 C++ 动态库；编译兼容级别 API 17，使用的采集接口最低 API 14。编译成功不代表商业设备运行授权与编码能力已经验证。 +之前的 HAP 实验源码和签名流程已移除。本机签名材料不属于插件或发行包。原方案的研究过程保留在[历史记录](research/harmony-realtime-stream.md)，当前方案及来源见[HDC 持续投屏调研](research/harmony-hdc-only-screen.md)。 -签名工具已登录，但调用 `signature generate` 返回账号尚未完成开发者实名认证。尚未安装伴随应用、取得真机 H.264 视频、测量呈现帧率或端到端延迟。 - -## 构建与签名 - -需要完整官方 Command Line Tools、Node.js 22+、官方 `@deveco/deveco-cli@1.3.0-stable`。`DEVECO_CLI_CLT_PATH` 指向包含 `sdk`、`tool`、`hvigor` 的 `command-line-tools` 根目录，不是单独的 HDC 所在目录。 +## 构建和验证 ```sh -cd curios/harmony-connector/companion -# 首次准备；已有配置时保留本机签名。 -test -f build-profile.json5 || cp build-profile.template.json5 build-profile.json5 +node curios/harmony-connector/build.mjs +node tools/test-harmony.mjs -# 预先按本机实际路径设置 DEVECO_CLI_CLT_PATH。 -npx --yes @deveco/deveco-cli@1.3.0-stable build --modules entry --build-mode debug +# 手机开启 USB 调试并授权 Mac；多设备时在末尾加目标 ID。 +curios/harmony-connector/backend/harmony-connector --check-screen /absolute/path/to/hdc + +# 连续 20 秒，通过与 UI 相同的后端接口接收并实际解码 JPEG。 +curios/harmony-connector/backend/harmony-connector --check-screen-stream /absolute/path/to/hdc ``` -得到 `entry/build/default/outputs/default/entry-default-unsigned.hap`。真实手机安装必须有合法签名；需要已实名认证的华为开发者账号，按官方流程配置可调试设备。认证入口是 [AppGallery Connect](https://developer.huawei.com/consumer/cn/service/josp/agc/index.html)，见[官方开发准备](https://developer.huawei.com/consumer/cn/appgallery/devstart/)。 +动态测量时保持手机解锁，并连续滑动桌面或滚动页面。`receivedFps` 与 `mbps` 是最近一秒收到的帧数和码率；`consumerFps` 是此验证程序在完整观察期解码的不同帧数除以时长，不是 WebView 呈现帧率。`frameAgeMs` 是收到画面后经过的时间，不是端到端延迟。静止画面不重复计帧。 -```sh -npx --yes @deveco/deveco-cli@1.3.0-stable auth login -npx --yes @deveco/deveco-cli@1.3.0-stable signature generate --product default -npx --yes @deveco/deveco-cli@1.3.0-stable build --modules entry --build-mode debug -``` +上述命令默认不保存画面、不启用控制，也不发送输入。基础连通命令的第三个参数可以是用户指定的保存目录。真实交互仍需在插件中检验：启用控制、点击、拖动、返回/主页、文字输入、停止后重新开始，以及拔线后旧画面清空。 -`build-profile.json5`、签名证书、profile 和密码材料均属于本机文件，不提交到仓库。模板 `build-profile.template.json5` 不包含签名配置。不要将绑定调试设备的 HAP 当作所有用户均可安装的发行包；正式手机组件需要发布者签名和正式分发流程。 +## 会话与协议 -## 真机验证 +- 插件只保留最新一张 JPEG，不积压视频队列。UI 用上一帧序号等待变化；静止时返回状态，不重复发送图片。 +- 图像与控制使用两个独立 socket，共用本会话的 HDC 转发。按键和手势通过持久 RPC 发送，文字使用已验证的 `uitest uiInput text`。 +- 代理回包包括 JSON 行与 JPEG；解析 JPEG 段长度，避免把 EXIF 数据中的字节误认作图像结尾。单帧上限 2 MiB，JSON 上限 64 KiB。 +- 静止画面不视为断线；独立控制连接查询尺寸检查存活。UI 停止请求超过 15 秒会结束会话。屏幕几何变化会关闭输入控制。 +- 每次部署使用唯一文件名，只结束加载该文件的进程，只删除本次转发和文件。发现其他 UITest 会话时提示占用，不接管或结束其他调试工具。 -先确认 `hdc list targets -v` 显示目标为 `Connected`。安装刚构建的 **signed** HAP，包名为 `ink.render.trove.harmonyconnector`。安装应选择明确目标，不自动安装到其他设备。 - -在仓库根目录执行以下命令，将 HDC 路径替换为本机工具路径；连接多台设备时另加 `--target`： - -```sh -python3 tools/harmony-stream.py --hdc /absolute/path/to/hdc --seconds 20 -# 人工观看连续画面；需 ffplay。 -python3 tools/harmony-stream.py --hdc /absolute/path/to/hdc --seconds 20 --view -``` - -运行时在手机点击「共享屏幕」，同意系统请求，滚动页面或移动窗口产生持续变化。默认验证需要 FFmpeg；它只用于开发测量，未引入生产插件依赖。 - -接收帧率目标至少 20 fps，默认解码模式同时要求解码帧数达到收到视频帧数的 95%。这些只是开发验收门槛，不能代替端到端延迟测量。`--view` 的 FFplay 不提供解码帧计数，需另外运行默认模式；人工确认的流畅程度应单独记录。静态画面可能由系统抑制重复帧，性能测试必须保持画面变化。 - -停止共享、系统拒绝、拔线、关闭验证程序都必须终止手机采集并清理仅本会话的转发。当前原型在屏幕尺寸变化时停止共享，提示重新开始；自动旋转重建、插件播放器集成及流式会话中的输入验证仍待完成。 - -## 传输与资源边界 - -手机仅监听 IPv4 loopback，由启动端提供随机远端端口和 32 字节会话随机值。连接后首先发送 64 个小写十六进制字符进行会话校验，不通过则断开。正常用户操作仍需手机点击共享以及系统录屏授权。 - -验证成功后，手机发送固定 24 字节头：`HCS1` + 5 个大端 u32，依次为编码宽、高、屏幕宽、高、配置帧率。随后每包为大端 `u32 payloadLength + u32 flags + i64 ptsUs + Annex B H.264 payload`，沿用公共编码器的 buffer flags。SPS/PPS 配置数据与编码帧均按输出顺序保留。 - -单包上限 2 MiB；采集回调复制数据后归还编码器 buffer。发送队列上限 6 包和 4 MiB，超过就结束会话，不积压旧帧，也不任意丢弃参考帧。Socket 发送超时 500 ms。主机发送一个零字节或断开连接可停止采集。系统停止与错误统一由资源所属线程收尾，释放 capture、encoder、surface、socket。 - -这仍是开发协议，尚未替代正式产品的配对、版本协商和播放器生命周期设计。普通录屏授权也不会赋予任意输入注入权限；后续控制需复用已授权的 UITest 能力并验证响应。 - -## 自动化验证 - -```sh -python3 tools/harmony-stream.test.py -v -``` - -测试不连接手机；使用本地 socket pair 验证接收行为，使用生成图案验证实际 H.264 解码。当前 8 项通过，20 帧均成功解码。它们不证明手机端授权、编码吞吐、后台保活或 USB 实时性。 - -API、权限和被拒绝的无安装路线见[连续视频调研](research/harmony-realtime-stream.md)。 +运行 HDC 不需要完整 SDK：本机独立验证了 `hdc` 与同目录 `libusb_shared.dylib` 的版本命令，总计约 6.4 MiB。完整工具包用于开发构建，不应成为普通投屏用户的依赖。本次尚未新增精简 HDC 发行包；插件继续使用已配置或自动发现的官方工具。 diff --git a/docs/research/harmony-hdc-only-screen.md b/docs/research/harmony-hdc-only-screen.md new file mode 100644 index 0000000..d59d155 --- /dev/null +++ b/docs/research/harmony-hdc-only-screen.md @@ -0,0 +1,149 @@ +# Harmony Connector：仅通过 HDC 的连续投屏 + +调研日期：2026-09-10。产品约束：最终用户只启用 USB 调试并授权 Mac，不注册开发者账号、不安装 HAP、不修改手机安全策略。目标是可交互的连续画面，不限定 H.264；帧率和延迟必须在真机上测量。本篇研究未操作设备。 + +## 当前结论 + +找到了一条此前漏查的公开分发路线：**使用华为 Hypium 的公开 PyPI 依赖包中原样分发的 UITest agent，由设备内置 `uitest` 加载，通过 HDC 转发持续 JPEG 与触控 RPC。** 该路线无需用户编译、签名或安装手机应用；它仍会向设备临时推送一个 agent 文件，作用类似 scrcpy 的临时设备端服务。不能把自编 unsigned ELF 或 addon 被 XPM 拒绝，推广为所有官方签名 agent 均无法加载。 + +Hypium 的公开说明支持 USB/Wi-Fi 调试连接，并明确 Driver 模式可作为 SDK 集成到其他工具。其 `xdevice-devicetest` 包公开包含原生 agent；对应的 `xdevice-ohos` 包提供选择、推送、启动和转发实现。[Hypium 官方发布页](https://pypi.org/project/hypium/6.1.0.210/)、[设备测试资源包](https://pypi.org/project/xdevice-devicetest/6.1.0.210/)、[设备连接包](https://pypi.org/project/xdevice-ohos/6.1.0.210/) + +**主任务真机进展：**官方 `v1.2.2.so` 使用唯一 `--extension-name` 已成功加载，Unix socket 能返回 JPEG，`Driver.create`、屏幕尺寸和旋转查询成功；不需要安装 HAP、开发者账号或录屏确认弹窗。初次收到的是 630 × 1422 黑帧，首帧约 195–235 ms，15 秒仅收到 3 帧；当时可能处于锁屏状态，**这些数字只证明链路连通，不能作为动态画面流畅度或端到端延迟的结论**。本文作者未操作手机，真机结果来自主任务当轮报告。 + +## 精确来源与许可证据 + +本轮从 PyPI 元数据取得 wheel 下载地址，在内存读取并校验 SHA256，没有安装或执行其中代码。 + +| 资源 | 版本、路径及 SHA256 | +| --- | --- | +| xdevice-devicetest wheel | `6.1.0.210`；`1c80f0da8d497d4e034f9873e966e8b2d8ef23ecc376c64c839c1493a813980e` | +| 新版 ARM64 agent | `devicetest/res/prototype/native/uitest_agent_v1.2.2.so`；`e1b8e75fad983aa29640784ee4b457fe7ac1c916a15f72c8b899d5a7716da651` | +| 旧版 ARM64 agent | `devicetest/res/prototype/native/uitest_agent_v1.1.10.so`；`1c9286456fb003ee15d86fef04e8c93f004027349e6dd2ef972c792e0a6d4bf8` | +| xdevice-ohos wheel | `6.1.0.210`；`548a09bb6fc9fcfd92a7dbb5b69180de0159ce8959057e133dec34774054bb12` | + +资源包的 `METADATA` 明确列出 `License-File: LICENSE`，`xdevice_devicetest-6.1.0.210.dist-info/LICENSE` 是完整 Apache License 2.0；本次清单未发现额外 EULA、NOTICE 或对 agent 的另行许可文件。Apache 2.0 提供商业使用及 object form 再分发条件；打包时保留该许可证、来源版本和校验值，保留现有版权声明，修改自有封装代码应说明。不要以第三方仓库的许可证替代原始发布包的许可。[原始 wheel](https://files.pythonhosted.org/packages/a2/b5/55bdfe4d669558331499848dbc68269506abb9b4a771e3524a65f9845127/xdevice_devicetest-6.1.0.210-py3-none-any.whl)、[Apache 2.0 条款](https://www.apache.org/licenses/LICENSE-2.0) + +原始 `v1.2.2.so` 尾部包含 ELF 签名块标记，并含 Huawei CBG Software Signing Service CA 证书链字符串。这里只确认原件附带签名结构，**没有进行独立密码学验签，也没有据此声称目标设备必然接受**。部署必须保持字节不变，不 strip、不重链接、不截掉 ELF 尾部。该 wheel 没有提供 agent 的 C++ 源码，因此应称“公开分发的官方 agent”，不能称为设备端也能完全自行重建的开源实现。 + +## 真实选择与启动路径 + +当前手机 UITest 版本由主任务报告为 `6.0.2.3`。官方 `xdevice-ohos 6.1.0.210` 内： + +- `ohos/environment/device.py::check_if_unix_socket_mode`：UITest 版本高于 `6.0.2.1` 且设备不是 x86_64 时使用 Unix socket。 +- `ohos/proxy/proxy_base.py::_init_so_resource`：此分支选择 `uitest_agent_v1.2.2.so`；旧分支才使用 v1.1.10/v1.1.5/v1.1.3。 +- `ohos/proxy/ui_proxy.py`：默认推送到 `/data/local/tmp/agent.so`，运行 `/system/bin/uitest start-daemon singleness`，连接 `localabstract:uitest_socket`。旧协议为设备 TCP 8012。 + +因此不能把“设备 6.x”硬编码为旧 TCP 8012，也不能只依据文件名相似选择第三方 agent。[对应公开 wheel](https://files.pythonhosted.org/packages/78/a6/8ac621bb958813cde648d515069c30188939b492ffcdd29e8d1e234550cf/xdevice_ohos-6.1.0.210-py3-none-any.whl) + +以下最小命令形态已由主任务用本轮官方原件在当前手机验证；参数仍必须来自所选设备与本机会话： + +```text +hdc -t file send /data/local/tmp/ +hdc -t shell uitest start-daemon singleness --extension-name +hdc -t fport tcp: localabstract:uitest_socket +``` + +独有 `--extension-name` 来自 OpenHarmony UITest 的扩展加载器：它将名称解析到 `/data/local/tmp/` 并加载扩展。官方 Python 默认使用 `agent.so`，独有名已在本轮商业手机上验证，其他固件仍需兼容验证。不要覆盖其他工具已有的 `agent.so`；官方框架内部的通配清理、强杀其他进程不应照搬到插件。[扩展加载器](https://github.com/openharmony/testfwk_arkxtest/blob/b04e30c40cf266c9abfbd52933e43a0456eb99a2/uitest/addon/extension_executor.cpp) + +## 持续 JPEG 与控制协议 + +公开客户端已有实际代码实现，不依赖教程中的口头描述。本节以 `devhelmkit` 锁定提交 `05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00` 为协议参考，设备 agent 原件仍从官方 wheel 获取。该项目的 `device/hdc.py`、`rpc/proxy_v2.py`、`screenshot_stream.py` 分别实现传输、RPC 和连续画面。[传输实现](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/device/hdc.py)、[RPC](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/rpc/proxy_v2.py)、[连续帧](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/screenshot_stream.py) + +客户端发送请求的 TCP 封包顺序： + +```text +ASCII "_uitestkit_rpc_message_head_" +u32 big-endian session ID +u32 big-endian UTF-8 JSON byte length +JSON body +ASCII "_uitestkit_rpc_message_tail_" +``` + +初始化和控制使用独立连接。调用 `Driver.create` 后使用返回的实际引用值，不能把 `Driver#0` 当成所有连接的固定保证。外层 `request_id` 为本次唯一字符串，transport session ID 是另一字段。 + +```json +{ + "module": "com.ohos.devicetest.hypiumApiHelper", + "method": "callHypiumApi", + "params": { + "api": "Driver.create", + "this": null, + "args": [], + "message_type": "hypium" + }, + "request_id": "" +} +``` + +后续只替换 `params` 中的 API、引用与参数，外层保持相同。以下是主任务确认的**解析后结果值**，不代表复制完整回包结构： + +| API | `this` / `args` | 当前手机结果 | +| --- | --- | --- | +| `Driver.create` | `null` / `[]` | `"Driver#0"` | +| `Driver.getDisplaySize` | 返回的 Driver 引用 / `[]` | `{"displayId":0,"x":1260,"y":2844}` | +| `Driver.getDisplayRotation` | 返回的 Driver 引用 / `[]` | `0` | +| `Driver.wakeUpDisplay` | 返回的 Driver 引用 / `[]` | `null` | +| `Driver.getDeviceInfo` | 返回的 Driver 引用 / `[]` | `No handler found`，当前手机不可用，不用于探活。 | + +物理输入坐标应依据屏幕尺寸与旋转转换，不能直接使用 JPEG 缩小后的像素坐标。`getDisplaySize` 可作为独立控制连接的低频探活；无新图不等于连接已断。 + +在独立采集 socket 发送： + +```json +{ + "module": "com.ohos.devicetest.hypiumApiHelper", + "method": "Captures", + "params": { + "api": "startCaptureScreen", + "args": { "options": { "displayId": 0, "scale": 0.5 } } + }, + "request_id": "" +} +``` + +注意 `Captures.args` 是对象，与 `callHypiumApi.args` 数组不同。JPEG 被推送到发送采集请求的 socket。**当前官方 v1.2.2 真机回包实测为 JSON 行 `{"result":true,"pts":...}\n` 与裸 JPEG `FF D8 … FF D9` 加换行，不使用上述 HEAD/TAIL 请求封包包住 JPEG。**不能只实现带 HEAD/TAIL 的接收器。参考客户端兼容协议封装与裸 JPEG；实现应依据实际格式使用有界长度及完整帧解析，不能假设一次 `recv` 对应一张图，也不能忽略 ACK/错误后无限等待。[采集请求与分帧](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/screenshot_stream.py#L677) + +实时触控使用相同外层与 `method: "Gestures"`，内层为 `api: "touchDown" | "touchMove" | "touchUp"`、`args: {"x": ..., "y": ...}`。客户端实现按下、移动、抬起接口，适合替换拖动后才发送整段 swipe 的延迟；必须限制 move 排队，并保证取消、失焦时结束本会话已按下触点。键盘、文字可继续使用现有已验证的 UITest 输入路径。[触控实现](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/driver.py#L687) + +未找到 `Gestures.keyDown`、`Gestures.keyUp` 或 `Gestures.text` 的公开调用依据。普通按键使用 `callHypiumApi`：`Driver.triggerKey` 的 `args` 为 `[keyCode]`；组合键 `Driver.triggerCombineKeys` 为 `[key1,key2]` 或三个键；Home/Back 分别是 `Driver.pressHome` / `Driver.pressBack`、`args:[]`。不要使用不存在的 `Driver.pressKey`。官方 `hypium/uidriver/arkuidriver.py` 声明 `triggerKey`、`triggerCombineKeys`、`pressBack`；其 Home 包装存在 `UiDriver.pressHome` 命名差异，因此仍应按当前设备能力验证客户端使用的 `Driver.pressHome`。[官方 Hypium wheel](https://files.pythonhosted.org/packages/b4/dc/f3b0abf2c518d11d57d8e9487026c133cc7eb0e3479af4b90d13c6db6169/hypium-6.1.0.210-py3-none-any.whl)、[客户端按键调用](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/driver.py#L712) + +官方 Python 的 `Driver.inputText` 参数为 `[{"x":x,"y":y},"text"]`，有定位输入的语义；它并不是已确认的 `Gestures` 当前光标文字接口。公开客户端对当前焦点使用 `uitest uiInput text `。当前阶段保留已有验证的文字通道，可避免引入未经验证的光标、追加与清空语义。[当前焦点输入](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/driver.py#L1487) + +停止时在采集连接发送 `method: "Captures"`、`params: {"api":"stopCaptureScreen","args":{}}`，随后关闭 socket、停止本机接收线程、移除本会话转发。若启用设备守护进程清理，仅结束自己创建并持有身份的进程，不使用 `killall uitest`。主机解码与渲染只保留最新完整帧；静止画面没有新帧不应误判断线，可用独立控制通道探活。[停止实现](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/screenshot_stream.py#L565) + +`startCaptureScreen` 是 agent 提供的 RPC 名称，**不是**设备 `uitest help` 中能直接运行的 CLI 子命令。设备帮助没有它，不证明公开 agent 不能提供它。 + +## 画面变化、帧率与保活 + +公开 UITest `screen_copy.cpp` 的采集线程调用 `DisplayManager::GetScreenshot`，通过 `memcmp` 比较前后像素，只有变化才通知 JPEG 编码线程。屏幕关闭时提供黑色画面并暂停常规采集，50 ms 的 `usleep` 仅用于检查关闭的屏幕是否重新亮起，**不能据此推算正常采集被限制在 20 fps**。常规采集循环没有显式 fps 计时上限，编码线程取最新帧。商业 agent 未提供其 C++ 源码，此处解释公开 UITest 实现，不能当作官方 v1.2.2 每一项内部行为均相同的证明；本机静止画面不连续发帧与这套行为一致。[变化检测与屏幕关闭](https://github.com/openharmony/testfwk_arkxtest/blob/b04e30c40cf266c9abfbd52933e43a0456eb99a2/uitest/addon/screen_copy.cpp#L153)、[消费最新帧](https://github.com/openharmony/testfwk_arkxtest/blob/b04e30c40cf266c9abfbd52933e43a0456eb99a2/uitest/addon/screen_copy.cpp#L245) + +核查的公开扩展与客户端只暴露 `displayId`、`scale`，没有可据以实现的 fps 或 JPEG quality 参数。`scale` 在获取原屏截图后缩放像素，能降低输出尺寸与编码、传输成本，但不消除原屏截图成本。Devhelmkit 的 `MAX_RECORD_FPS=30` 只约束事后 MP4 合成，与实时采集上限无关。不要把虚构的 `fps:60` 参数加入产品后声称已提速。[缩放位置](https://github.com/openharmony/testfwk_arkxtest/blob/b04e30c40cf266c9abfbd52933e43a0456eb99a2/uitest/addon/screen_copy.cpp#L221)、[实际采集参数](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/screenshot_stream.py#L670) + +参考客户端的采集 socket 连接后设为阻塞读取，没有周期发送采集心跳；尚无“必须在采集 socket 定期发送心跳”的源码证据。可保留本机读取超时用来检查取消状态，但超时本身不应标为失败；用独立控制连接的 `Driver.getDisplaySize` 区分静态画面与设备失联。若控制连接也无响应、TCP 关闭或 HDC 确认失联，再终止会话。[socket 建立与持续读取](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/src/devhelmkit/harmony/screenshot_stream.py#L651) + +## 两个开源客户端的来源核对 + +| 项目 | 本轮源码与文件核对 | 结论 | +| --- | --- | --- | +| `yabi-zzh/devhelmkit` | Apache 2.0；有连续 JPEG、独立控制 socket、down/move/up 和采集帧率统计；仓库 `agent_v2.so` SHA256 与本次官方 wheel 的 agent 不一致。 | 可参考其客户端协议，不把其未核清来源的 blob 作为发行原件。 | +| `yiyifred/Cloud-Phone` | Apache 2.0；包含 `uitest_agent_v1.1.10.so`，其 SHA256 **完全匹配**官方 wheel v1.1.10；`Gestures` 代码注释中的 ECHO/hdckit 指同类 RPC 操作，并不是设备原生内置 H.264 服务。 | 证明实际客户端采用 Hypium agent；其旧 agent/TCP 假设不能直接用于当前设备。 | + +[devhelmkit 许可](https://github.com/yabi-zzh/devhelmkit/blob/05c9bc0f7f9ea763d61c8b4b5d1cfbb076626b00/LICENSE)、[Cloud-Phone agent](https://github.com/yiyifred/Cloud-Phone/blob/98d130b20474cf8b18d51161d3bb0b65472c4778/backend/assets/harmony/uitest_agent_v1.1.10.so)、[Cloud-Phone RPC](https://github.com/yiyifred/Cloud-Phone/blob/98d130b20474cf8b18d51161d3bb0b65472c4778/backend/node/src/services/harmony-cast/uitest-rpc.js)、[Cloud-Phone 许可](https://github.com/yiyifred/Cloud-Phone/blob/98d130b20474cf8b18d51161d3bb0b65472c4778/LICENSE) + +## H.264 与其他路径的边界 + +同一官方资源 wheel 包含 `devicetest/res/recorder/libscrcpy_server1.z.so` 到 `libscrcpy_server4.z.so`。其 `controllers/tools/recorder/record_agent.py` 推送为 `/data/local/tmp/libscreen_recorder.z.so`，通过 UITest extension 启动，参数为 `-p -m 1 -screenId `，默认 gRPC 端口 5001；对应 `rpc_manager.py` 和生成的 protobuf 定义含 `ScrcpyService/onStart`、`onEnd`、`onRequestIDRFrame`。当前 Python 调用方主要用于步骤视频，停止后拉取 `mytest.mp4`。**仅凭资源名不能认定它是可直接使用的实时 H.264 投屏服务**；格式、stream payload 与当前固件兼容性仍需专门验证。它不是本轮 JPEG 路线的前置条件。[官方资源 wheel](https://pypi.org/project/xdevice-devicetest/6.1.0.210/) + +设备内置 UITest 公开 CLI 的 screenCap 是单张 PNG，uiRecord 是操作记录，start-daemon 是测试服务入口；不提供已确认的直接视频 stdout 命令。官方系统录屏组件的公开流程是录制后导出文件。对已经确认不存在的 `screenrecord`，不应反复尝试 Android 参数。[UITest 命令](https://github.com/openharmony/docs/blob/f41b9345badd47c7ab0c263344cd7f4b5a549afb/zh-cn/application-dev/application-test/uitest-guidelines.md)、[官方系统录屏](https://developer.huawei.com/consumer/cn/doc/harmonyos-guides/ide-screen-recording) + +自编 ELF 的执行权限错误与自编 addon 的 XPM 加载拒绝已由主任务复现；它们不应作为继续尝试关闭 XPM、SELinux、root 或借用私人签名的理由。公开 Hypium 原件是另一条正常工具分发与加载链路，需按原件、版本、协议重新验证。 + +## 产品验收与明确未知 + +- 当前证据确认公开来源、许可文件、官方选择和加载方式，以及开源客户端连续帧/触控协议；主任务已验证当前手机原件加载、Unix socket、Driver 初始化、JPEG 解码与屏幕尺寸查询。 +- 初测黑帧只证明连通，仍需设备解锁后的真实内容与交互画面验证；不得把收到 JPEG 等同于已达到 scrcpy 的流畅体验。 +- 用手机实际动画/滑动测量收到与呈现的帧率，独立记录排队及输入往返；静止画面频率不能当作最大帧率。没有实际结果前不承诺 30/60 fps。 +- 验证拖动跟手、旋转后的物理坐标、拒绝/断开、停止后不再采集与控制，以及重复启动清理；不影响其他使用 UITest 的工具。 +- 未确认 `Captures/Gestures` 跨版本稳定性；应固定已验收的 agent，报告能力与错误，避免无提示降为旧截图模式。 + +最终用户流程可以保持“连接 USB → 允许调试 → 开始投屏”。官方 agent 的获取、校验、部署、转发和清理应由插件完成，不能再把开发者账号或 HAP 签名作为该流程的要求。 diff --git a/docs/research/harmony-realtime-stream.md b/docs/research/harmony-realtime-stream.md index 21778e9..6ebb676 100644 --- a/docs/research/harmony-realtime-stream.md +++ b/docs/research/harmony-realtime-stream.md @@ -1,5 +1,7 @@ # Harmony Connector 实时视频流调研 +> 历史方案记录：本文的 H.264/HAP 实验已停止，实验源码和账号签名流程已移除。当前采用无需用户开发者账号的[官方 Hypium/HDC 路线](harmony-hdc-only-screen.md)，状态及验证命令见[持续投屏开发验证](../harmony-realtime-development.md)。下文的早期“无安装”失败仅涉及自编未签名原件，不能推导为必须安装 HAP。 + 调研日期：2026-09-10。承接 [投屏与控制调研](harmony-screen-control.md)，本篇只核查替换逐张截图链路所需的公开 API、进程授权和构建条件。文献研究与本机集成验证分别记录；尚未验证真机 H.264 视频性能。 ## 结论与实现顺序 @@ -140,7 +142,7 @@ devecocli build --modules entry --build-mode debug UITest addon 未进入 `OnInit`。随后读取设备日志确认：MUSL-LDSO 报 `Xpm check failed`、`errno for mmap is:13`，UiTestKit_Addon 的 `dlopen` 返回 `Permission denied`。这将本次失败定位到临时未签名模块的加载校验，不能解释为 ABI 不兼容；没有尝试关闭或绕过 XPM。临时程序、模块和手机上的本会话文件均已清理。 -华为公开 LLDB 调试矩阵将 user 镜像、SELinux 开启场景的支持范围列为签名 debug HAP；并区分其与 root 镜像上的普通可执行文件调试。它与上述限制相符，但不能仅凭该文档判定具体拒绝来自 SELinux、代码签名还是其他策略。继续走正常应用签名路径，不以关闭安全策略推进。[华为 Native 调试条件](https://developer.huawei.com/consumer/cn/doc/doccenter-capabilities/debug-lldb) +华为公开 LLDB 调试矩阵将 user 镜像、SELinux 开启场景的支持范围列为签名 debug HAP；并区分其与 root 镜像上的普通可执行文件调试。它与上述限制相符，但不能仅凭该文档判定具体拒绝来自 SELinux、代码签名还是其他策略。当时选择了普通应用签名实验；该路线随后已停止，当前改用官方分发且无需用户签名的 Hypium 代理。[华为 Native 调试条件](https://developer.huawei.com/consumer/cn/doc/doccenter-capabilities/debug-lldb) ## 最小验收 @@ -156,4 +158,4 @@ UITest addon 未进入 `OnInit`。随后读取设备日志确认：MUSL-LDSO 报 集成方已创建普通手机应用 `ink.render.trove.harmonyconnector`，实现系统授权的 Surface 采集、H.264 编码、带会话校验的 loopback TCP 和停止收尾。官方 CLI 已成功构建 ArkTS 与 arm64-v8a native 模块并生成未签名 HAP。独立 Mac 接收器的 8 项测试通过，其中生成的 20 帧视频经过真实 H.264 解码全部返回。 -官方 CLI 登录成功，但签名预检返回账号尚未完成开发者实名认证，未生成真机可安装的调试签名。因此未安装组件、未取得手机视频流，插件播放器集成、旋转重建和端到端时延验收仍未完成。源码和可复现命令见[开发验证说明](../harmony-realtime-development.md)。 +最初签名预检被账号认证状态阻塞；用户认证并刷新官方授权后，曾成功生成正常调试签名、构建并安装 HAP，收到首个 H.264 帧。接收器在第一帧统计时发生除零错误，故没有取得持续帧率或端到端延迟结果。随后按用户要求停止 HAP 路线，实验源码已从主分支移除；历史实现保留于提交 `ca382e6`。当前无需账号的实现和验证状态见[开发验证说明](../harmony-realtime-development.md)。 diff --git a/tools/harmony-stream.py b/tools/harmony-stream.py deleted file mode 100644 index 6adbab4..0000000 --- a/tools/harmony-stream.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 -"""Measure the companion's HCS1 video stream; no screenshots/files are retained. - -Development tool, not the plugin's production player. Requires official HDC, -the signed companion on the phone, and ffmpeg (or ffplay with --view). -""" -import argparse -import json -import os -from pathlib import Path -import re -import secrets -import select -import shutil -import socket -import struct -import subprocess -import threading -import time - -BUNDLE = "ink.render.trove.harmonyconnector" -MAX_PACKET = 2 * 1024 * 1024 - - -def read_exact(sock, size, deadline): - result = bytearray() - while len(result) < size: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError("Video stream timed out") - sock.settimeout(min(remaining, 0.2)) - try: - part = sock.recv(size - len(result)) - except socket.timeout: - continue - if not part: - raise EOFError("Screen sharing stopped or USB disconnected") - result.extend(part) - return bytes(result) - - -def read_hello(sock, deadline): - magic, width, height, display_width, display_height, fps = struct.unpack( - ">4s5I", read_exact(sock, 24, deadline)) - if (magic != b"HCS1" or not 1 <= fps <= 60 or - not all(2 <= d <= 8192 for d in (width, height, display_width, display_height)) or - width % 2 or height % 2 or max(width, height) > 1280): - raise ValueError("Unsupported stream header") - return {"width": width, "height": height, "displayWidth": display_width, - "displayHeight": display_height, "configuredFps": fps} - - -def read_packet(sock, deadline): - size, flags, pts = struct.unpack( - ">IIq", read_exact(sock, 16, deadline)) - if not 0 < size <= MAX_PACKET or flags & ~0x7f or pts < 0: - raise ValueError("Invalid video packet") - data = read_exact(sock, size, deadline) - if not data.startswith((b"\0\0\1", b"\0\0\0\1")): - raise ValueError("Encoder did not return Annex B H.264") - return flags, pts, data - - -class Decoder: - def __init__(self, view): - name = "ffplay" if view else "ffmpeg" - executable = shutil.which(name) - if not executable: - raise RuntimeError(f"Development measurement requires {name} on PATH") - args = [executable, "-hide_banner", "-loglevel", "error", - "-flags", "low_delay", "-probesize", "32", "-analyzeduration", "0", - "-f", "h264", "-i", "pipe:0"] - if view: - args += ["-window_title", "Harmony Connector · H.264 preview", "-autoexit"] - else: - args += ["-progress", "pipe:1", "-stats_period", "0.25", "-f", "null", "-"] - self.frames = 0 - self.error = bytearray() - self.process = subprocess.Popen(args, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - os.set_blocking(self.process.stdin.fileno(), False) - self.threads = [threading.Thread(target=self.progress, daemon=True), - threading.Thread(target=self.errors, daemon=True)] - for thread in self.threads: - thread.start() - - def progress(self): - for line in self.process.stdout: - if line.startswith(b"frame="): - self.frames = int(line.split(b"=", 1)[1]) - - def errors(self): - for line in self.process.stderr: - if len(self.error) < 8192: - self.error.extend(line[:8192 - len(self.error)]) - - def write(self, data): - deadline = time.monotonic() + 1 - view = memoryview(data) - while view: - if self.process.poll() is not None: - raise RuntimeError("Video decoder exited: " + self.error.decode(errors="replace")) - if time.monotonic() >= deadline: - raise TimeoutError("Decoder is falling behind; stopped instead of queuing video") - if not select.select([], [self.process.stdin], [], 0.05)[1]: - continue - try: - count = os.write(self.process.stdin.fileno(), view) - view = view[count:] - except BlockingIOError: - continue - - def close(self): - self.process.stdin.close() - try: - self.process.wait(timeout=3) - except subprocess.TimeoutExpired: - self.process.terminate() - try: - self.process.wait(timeout=1) - except subprocess.TimeoutExpired: - self.process.kill() - self.process.wait() - for thread in self.threads: - thread.join(timeout=1) - self.process.stdout.close() - self.process.stderr.close() - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--hdc", required=True, type=Path) - parser.add_argument("--target") - parser.add_argument("--seconds", type=int, default=20) - parser.add_argument("--view", action="store_true") - args = parser.parse_args() - if not 5 <= args.seconds <= 120 or not args.hdc.is_file(): - parser.error("Use an existing HDC executable and 5–120 measurement seconds") - - def hdc(*command): - try: - result = subprocess.run([str(args.hdc.resolve()), *command], capture_output=True, - text=True, timeout=10, check=False) - except subprocess.TimeoutExpired: - raise RuntimeError("HDC command timed out") from None - if result.returncode: - raise RuntimeError("HDC command did not complete successfully") - if re.search(r"\b(fail(?:ed)?|error)\b", result.stdout + result.stderr, re.I): - raise RuntimeError("HDC command failed: " + result.stdout + result.stderr) - return result.stdout - - targets = [line.split()[0] for line in hdc("list", "targets", "-v").splitlines() - if "Connected" in line.split()] - target = args.target or (targets[0] if len(targets) == 1 else None) - if target not in targets: - raise RuntimeError("Select one connected, authorized phone with --target") - installed = hdc("-t", target, "shell", f"bm dump -n {BUNDLE}") - if BUNDLE not in installed: - raise RuntimeError("Install the signed Harmony Connector companion before measuring") - decoder = Decoder(args.view) - sock = None - forward = False - with socket.socket() as reservation: - reservation.bind(("127.0.0.1", 0)) - local_port = reservation.getsockname()[1] - remote_port = 20000 + secrets.randbelow(40000) - local, remote = f"tcp:{local_port}", f"tcp:{remote_port}" - token = secrets.token_hex(32) - frames = total_bytes = 0 - started = first_frame = None - try: - hdc("-t", target, "fport", local, remote) - forward = True - hdc("-t", target, "shell", f"aa start -b {BUNDLE} -a EntryAbility -m entry " - f"--ps token {token} --pi port {remote_port}") - print("请在手机点击「共享屏幕」，并同意系统授权。", flush=True) - deadline = time.monotonic() + 90 - while time.monotonic() < deadline: - try: - sock = socket.create_connection(("127.0.0.1", local_port), timeout=1) - sock.sendall(token.encode("ascii")) - hello = read_hello(sock, min(deadline, time.monotonic() + 2)) - break - except (OSError, EOFError, TimeoutError): - if sock: - sock.close() - sock = None - time.sleep(0.2) - if not sock: - raise TimeoutError("Phone did not start screen sharing") - print(json.dumps({"stage": "connected", **hello}), flush=True) - started = time.monotonic() - last_report = started - while first_frame is None or time.monotonic() - first_frame < args.seconds: - flags, _, data = read_packet(sock, time.monotonic() + (45 if first_frame is None else 3)) - decoder.write(data) - total_bytes += len(data) - now = time.monotonic() - if not flags & 8: - frames += 1 - if first_frame is None: - first_frame = now - print(json.dumps({"stage": "first-frame", "waitMs": round((now - started) * 1000)}), flush=True) - if first_frame and now - last_report >= 1: - elapsed = now - first_frame - print(json.dumps({"stage": "receiving", "frames": frames, - "receivedFps": round((frames - 1) / elapsed, 1), - "decodedFrames": decoder.frames, - "mbps": round(total_bytes * 8 / elapsed / 1000000, 2)}), flush=True) - last_report = now - elapsed = time.monotonic() - first_frame - finally: - if sock: - try: - sock.sendall(b"\0") - except OSError: - pass - sock.close() - try: - if forward: - hdc("-t", target, "fport", "rm", local, remote) - finally: - decoder.close() - passed = (frames - 1) / elapsed >= 20 and (args.view or decoder.frames >= frames * 0.95) - print(json.dumps({"passed": passed, "frames": frames, "decodedFrames": decoder.frames, - "receivedFps": round((frames - 1) / elapsed, 1), "durationSeconds": round(elapsed, 2), - "endToEndLatencyMeasured": False, "inputSent": False, "screenFilesSaved": False}), flush=True) - if not passed: - raise SystemExit(1) - - -if __name__ == "__main__": - main() diff --git a/tools/harmony-stream.test.py b/tools/harmony-stream.test.py deleted file mode 100644 index 5ad7fac..0000000 --- a/tools/harmony-stream.test.py +++ /dev/null @@ -1,87 +0,0 @@ -import importlib.util -from pathlib import Path -import re -import shutil -import socket -import struct -import subprocess -import time -import unittest - -spec = importlib.util.spec_from_file_location("harmony_stream", Path(__file__).with_name("harmony-stream.py")) -stream = importlib.util.module_from_spec(spec) -spec.loader.exec_module(stream) - - -class StreamProtocolTests(unittest.TestCase): - def setUp(self): - self.sender, self.receiver = socket.socketpair() - self.deadline = time.monotonic() + 1 - - def tearDown(self): - self.sender.close() - self.receiver.close() - - def test_negotiated_dimensions(self): - self.sender.sendall(struct.pack(">4s5I", b"HCS1", 566, 1280, 1260, 2844, 30)) - hello = stream.read_hello(self.receiver, self.deadline) - self.assertEqual((hello["width"], hello["displayHeight"], hello["configuredFps"]), (566, 2844, 30)) - - def test_unknown_version_rejected(self): - self.sender.sendall(struct.pack(">4s5I", b"HCS2", 640, 480, 1280, 960, 30)) - with self.assertRaises(ValueError): - stream.read_hello(self.receiver, self.deadline) - - def test_dimensions_bounded_before_video_allocations(self): - self.sender.sendall(struct.pack(">4s5I", b"HCS1", 65536, 480, 1280, 960, 30)) - with self.assertRaises(ValueError): - stream.read_hello(self.receiver, self.deadline) - - def test_oversized_packet_rejected_without_reading_payload(self): - self.sender.sendall(struct.pack(">IIq", stream.MAX_PACKET + 1, 2, 0)) - with self.assertRaises(ValueError): - stream.read_packet(self.receiver, self.deadline) - - def test_disconnect_in_packet_is_not_a_frame(self): - self.sender.sendall(struct.pack(">IIq", 32, 2, 42) + b"\0\0\1\x65") - self.sender.shutdown(socket.SHUT_WR) - with self.assertRaises(EOFError): - stream.read_packet(self.receiver, self.deadline) - - def test_silent_connection_has_bounded_wait(self): - with self.assertRaises(TimeoutError): - stream.read_packet(self.receiver, time.monotonic() + 0.02) - - def test_non_h264_payload_rejected(self): - self.sender.sendall(struct.pack(">IIq", 8, 2, 42) + b"JPEGdata") - with self.assertRaises(ValueError): - stream.read_packet(self.receiver, self.deadline) - - @unittest.skipUnless(shutil.which("ffmpeg"), "ffmpeg required for actual decoder validation") - def test_twenty_generated_frames_survive_transport_and_decode(self): - encoded = subprocess.run([ - "ffmpeg", "-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", - "testsrc2=size=160x96:rate=30", "-frames:v", "20", "-c:v", "libx264", - "-preset", "ultrafast", "-tune", "zerolatency", "-profile:v", "baseline", - "-g", "10", "-bf", "0", "-x264-params", "aud=1:repeat-headers=1", - "-f", "h264", "pipe:1"], capture_output=True, check=True, timeout=10).stdout - # Access unit delimiters separate independently timestamped encoder callbacks. - boundaries = [match.start() for match in re.finditer(b"\x00\x00\x00\x01\x09", encoded)] - self.assertEqual(len(boundaries), 20) - decoder = stream.Decoder(False) - try: - for index, (start, end) in enumerate(zip(boundaries, boundaries[1:] + [len(encoded)])): - data = encoded[start:end] - flags = 2 if index % 10 == 0 else 0 - self.sender.sendall(struct.pack(">IIq", len(data), flags, index * 33333) + data) - actual_flags, pts, actual_data = stream.read_packet(self.receiver, time.monotonic() + 1) - self.assertEqual((actual_flags, pts), (flags, index * 33333)) - decoder.write(actual_data) - finally: - decoder.close() - self.assertEqual(decoder.process.returncode, 0, decoder.error.decode()) - self.assertEqual(decoder.frames, 20, decoder.error.decode()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/harmony-ui.test.mjs b/tools/harmony-ui.test.mjs index 723d672..bcf7710 100644 --- a/tools/harmony-ui.test.mjs +++ b/tools/harmony-ui.test.mjs @@ -59,6 +59,7 @@ async function app(t) { screenControl: false, screenFrameWait: null, screenFrameError: null, + screenUnchanged: false, openedUrls: [], selectedDirectory: "/fixture/folder", configureError: null, @@ -105,11 +106,23 @@ async function app(t) { handler("hdc.screenFrame", async () => { if (model.screenFrameWait) await model.screenFrameWait; if (model.screenFrameError) throw model.screenFrameError; + if (model.screenUnchanged && model.screenSequence) + return { + unchanged: true, + width: 1200, + height: 2400, + sequence: model.screenSequence, + geometry: model.screenGeometry, + control: model.screenControl, + receivedFps: 0, + mbps: 0, + }; return { dataUrl: "data:image/jpeg;base64,/9j/2Q==", width: 1200, height: 2400, - captureMs: 300, + receivedFps: 27, + mbps: 3.2, sequence: ++model.screenSequence, geometry: model.screenGeometry, control: model.screenControl, @@ -473,7 +486,7 @@ test("device disconnect clears app details and prevents stale target operations" assert.equal($("app-launch").disabled, true); }); -test("screen viewing starts read-only, keeps refreshing and reports measured capture time", async (t) => { +test("screen viewing starts read-only, keeps refreshing and reports stream throughput", async (t) => { const { $, click, tick, model } = await app(t); model.hdcAvailable = true; await click("tab-management"); @@ -482,7 +495,10 @@ test("screen viewing starts read-only, keeps refreshing and reports measured cap await tick(); assert.equal($("screen-image").hidden, false); assert.match($("screen-status").textContent, /仅观看/); - assert.match($("screen-stats").textContent, /1200 × 2400.*300 ms/); + assert.match( + $("screen-stats").textContent, + /1200 × 2400.*接收 27 帧\/秒.*3.2 Mbps/, + ); assert.equal($("screen-control").checked, false); assert.equal($("app-install").disabled, true); const before = model.screenSequence; @@ -516,6 +532,52 @@ test("screen text and navigation require enabled control and use the displayed s assert.equal($("app-install").disabled, false); }); +test("an unchanged live screen preserves its image and does not count repeated frames", async (t) => { + const { $, click, tick, model } = await app(t); + model.hdcAvailable = true; + await click("tab-management"); + await click("screen-start"); + await tick(); + await click("screen-control"); + const source = $("screen-image").src; + const sequence = model.screenSequence; + model.screenUnchanged = true; + await tick(); + await tick(); + assert.equal($("screen-image").src, source); + assert.equal($("screen-image").hidden, false); + assert.equal(model.screenSequence, sequence); + assert.equal($("screen-control").checked, true); + assert.match($("screen-stats").textContent, /接收 0 帧\/秒/); + assert.ok(!model.calls.includes("hdc.screenStop")); +}); + +test("control does not wait behind a pending video frame", async (t) => { + const { $, click, tick, model } = await app(t); + model.hdcAvailable = true; + await click("tab-management"); + await click("screen-start"); + await tick(); + await click("screen-control"); + let release; + model.screenFrameWait = new Promise((resolve) => { + release = resolve; + }); + await tick(); + $("screen-inputs").querySelector('[data-screen-key="home"]').click(); + await settle(); + assert.ok( + model.parameters.some( + ([name, params]) => + name === "hdc.screenInput" && params.input.key === "home", + ), + ); + release(); + model.screenFrameWait = null; + await settle(); + await click("screen-stop"); +}); + test("stopping during a pending frame discards that response and permits a fresh screen session", async (t) => { const { $, click, tick, model } = await app(t); model.hdcAvailable = true; diff --git a/curios/harmony-connector/companion/legal/Apache-2.0.txt b/vendor/harmony-connector/hypium/LICENSE similarity index 89% rename from curios/harmony-connector/companion/legal/Apache-2.0.txt rename to vendor/harmony-connector/hypium/LICENSE index d645695..f433b1a 100644 --- a/curios/harmony-connector/companion/legal/Apache-2.0.txt +++ b/vendor/harmony-connector/hypium/LICENSE @@ -175,28 +175,3 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/harmony-connector/hypium/provenance.json b/vendor/harmony-connector/hypium/provenance.json new file mode 100644 index 0000000..18a7d8e --- /dev/null +++ b/vendor/harmony-connector/hypium/provenance.json @@ -0,0 +1,10 @@ +{ + "package": "xdevice-devicetest", + "version": "6.1.0.210", + "url": "https://files.pythonhosted.org/packages/a2/b5/55bdfe4d669558331499848dbc68269506abb9b4a771e3524a65f9845127/xdevice_devicetest-6.1.0.210-py3-none-any.whl", + "wheelSha256": "1c80f0da8d497d4e034f9873e966e8b2d8ef23ecc376c64c839c1493a813980e", + "member": "devicetest/res/prototype/native/uitest_agent_v1.2.2.so", + "sha256": "e1b8e75fad983aa29640784ee4b457fe7ac1c916a15f72c8b899d5a7716da651", + "license": "Apache-2.0", + "modified": false +} diff --git a/vendor/harmony-connector/hypium/uitest_agent_v1.2.2.so b/vendor/harmony-connector/hypium/uitest_agent_v1.2.2.so new file mode 100644 index 0000000..d54e0a9 Binary files /dev/null and b/vendor/harmony-connector/hypium/uitest_agent_v1.2.2.so differ