curios/curios/harmony-connector/viewer/ScreenView.swift

import AppKit
import AVFoundation
import QuartzCore

final class ScreenView: NSView {
    private let picture = AVSampleBufferDisplayLayer()
    private var frameInfo: ScreenFrame?
    private var displaySize = CGSize(width: 1, height: 2)
    private var pressed: CGPoint?
    private var wheelEnd: DispatchWorkItem?
    var readOnly = false { willSet { if newValue { releaseTouch() } } }

    override var acceptsFirstResponder: Bool { true }
    override var isFlipped: Bool { true }

    override init(frame: NSRect) {
        super.init(frame: frame)
        wantsLayer = true
        layer?.backgroundColor = NSColor.black.cgColor
        picture.videoGravity = .resizeAspect
        layer?.addSublayer(picture)
    }

    required init?(coder: NSCoder) { fatalError("init(coder:) is unavailable") }

    func display(_ image: CVPixelBuffer, frame: ScreenFrame) {
        if frameInfo?.geometry != frame.geometry { releaseTouch() }
        displaySize = CGSize(width: CVPixelBufferGetWidth(image), height: CVPixelBufferGetHeight(image))
        let aspectMatches = abs(log((displaySize.width / displaySize.height) / (CGFloat(frame.width) / CGFloat(frame.height)))) < 0.025
        if !aspectMatches { releaseTouch() }
        frameInfo = aspectMatches ? frame : nil
        var format: CMVideoFormatDescription?
        guard CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: image, formatDescriptionOut: &format) == noErr, let format else { return }
        var timing = CMSampleTimingInfo(duration: .invalid, presentationTimeStamp: .zero, decodeTimeStamp: .invalid)
        var sample: CMSampleBuffer?
        guard CMSampleBufferCreateReadyWithImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: image, formatDescription: format, sampleTiming: &timing, sampleBufferOut: &sample) == noErr, let sample else { return }
        if let attachments = CMSampleBufferGetSampleAttachmentsArray(sample, createIfNecessary: true) as? [NSMutableDictionary] {
            attachments.first?[kCMSampleAttachmentKey_DisplayImmediately] = true
        }
        CATransaction.begin()
        CATransaction.setDisableActions(true)
        picture.frame = imageRect()
        CATransaction.commit()
        if picture.status == .failed { picture.flush() }
        picture.enqueue(sample)
    }

    private func imageRect() -> CGRect {
        let scale = min(bounds.width / displaySize.width, bounds.height / displaySize.height)
        let size = CGSize(width: displaySize.width * scale, height: displaySize.height * scale)
        return CGRect(x: (bounds.width - size.width) / 2, y: (bounds.height - size.height) / 2, width: size.width, height: size.height)
    }

    override func layout() {
        super.layout()
        CATransaction.begin()
        CATransaction.setDisableActions(true)
        picture.frame = imageRect()
        CATransaction.commit()
    }

    private func point(_ event: NSEvent) -> CGPoint? {
        guard !readOnly, let frame = frameInfo else { return nil }
        let position = convert(event.locationInWindow, from: nil), rect = imageRect()
        guard rect.contains(position), rect.width > 0, rect.height > 0 else { return nil }
        return CGPoint(x: floor((position.x - rect.minX) * CGFloat(frame.width) / rect.width),
                       y: floor((position.y - rect.minY) * CGFloat(frame.height) / rect.height))
    }

    private func input(_ value: [String: Any], frame: ScreenFrame? = nil) {
        guard !readOnly, let frame = frame ?? frameInfo else { return }
        ViewerWire.send(["event": "input", "sequence": frame.sequence, "geometry": frame.geometry, "input": value])
    }

    override func mouseDown(with event: NSEvent) {
        window?.makeFirstResponder(self)
        releaseTouch()
        guard let point = point(event) else { return }
        pressed = point
        touch("touchDown", point)
    }

    override func mouseDragged(with event: NSEvent) {
        guard pressed != nil, let point = point(event) else { return }
        pressed = point
        touch("touchMove", point)
    }

    override func mouseUp(with event: NSEvent) {
        if pressed != nil, let point = point(event) { pressed = point }
        releaseTouch()
    }

    private func touch(_ kind: String, _ point: CGPoint) {
        input(["kind": kind, "x": Int(point.x), "y": Int(point.y)])
    }

    func releaseTouch() {
        wheelEnd?.cancel(); wheelEnd = nil
        if let point = pressed { touch("touchUp", point) }
        pressed = nil
    }

    override func rightMouseUp(with event: NSEvent) { input(["kind": "key", "key": "back"]) }
    override func otherMouseUp(with event: NSEvent) { if event.buttonNumber == 2 { input(["kind": "key", "key": "home"]) } }

    override func scrollWheel(with event: NSEvent) {
        guard let location = point(event), let frame = frameInfo else { return }
        if pressed == nil { pressed = location; touch("touchDown", location) }
        guard let point = pressed else { return }
        let factor: CGFloat = event.hasPreciseScrollingDeltas ? 3 : 30
        let next = CGPoint(x: max(0, min(CGFloat(frame.width - 1), point.x + event.scrollingDeltaX * factor)),
                           y: max(0, min(CGFloat(frame.height - 1), point.y + event.scrollingDeltaY * factor)))
        pressed = next
        touch("touchMove", next)
        wheelEnd?.cancel()
        let end = DispatchWorkItem { [weak self] in self?.releaseTouch() }
        wheelEnd = end
        DispatchQueue.main.asyncAfter(deadline: .now() + 0.12, execute: end)
    }

    override func keyDown(with event: NSEvent) {
        guard !readOnly else { return }
        if event.modifierFlags.contains(.command) {
            if event.charactersIgnoringModifiers == "v", let text = NSPasteboard.general.string(forType: .string) {
                input(["kind": "text", "text": text])
            } else { super.keyDown(with: event) }
            return
        }
        let keys: [UInt16: String] = [53: "back", 36: "enter", 51: "backspace", 117: "delete", 48: "tab", 123: "left", 124: "right", 125: "down", 126: "up", 115: "home"]
        if let key = keys[event.keyCode] { input(["kind": "key", "key": key]) }
        else if let text = event.characters, !text.isEmpty { input(["kind": "text", "text": text]) }
    }

    override func resignFirstResponder() -> Bool { releaseTouch(); return true }
}