iOSAI/Module/DeviceInfo.py

# -*- coding: utf-8 -*-
import os
import signal
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
from pathlib import Path
from typing import Dict, Optional, List, Any
import random
import socket
import http.client
import psutil
import hashlib  # 保留扩展
import platform

import tidevice
import wda
from tidevice import Usbmux, ConnectionType
from tidevice._device import BaseDevice

from Entity.DeviceModel import DeviceModel
from Entity.Variables import WdaAppBundleId, wdaFunctionPort, wdaScreenPort
from Module.FlaskSubprocessManager import FlaskSubprocessManager
from Module.IOSActivator import IOSActivator
from Utils.LogManager import LogManager


def _monotonic() -> float:
    """统一用 monotonic 计时，避免系统时钟跳变影响定时/退避。"""
    return time.monotonic()


class DeviceInfo:

    # --- 时序参数（支持环境变量覆盖） ---
    REMOVE_GRACE_SEC = float(os.getenv("REMOVE_GRACE_SEC", "8.0"))     # 设备离线宽限期
    ADD_STABLE_SEC   = float(os.getenv("ADD_STABLE_SEC", "2.5"))       # 设备上线稳定期
    ORPHAN_COOLDOWN  = float(os.getenv("ORPHAN_COOLDOWN", "8.0"))      # 拓扑变更后暂停孤儿清理
    HEAL_INTERVAL    = float(os.getenv("HEAL_INTERVAL", "5.0"))        # 健康巡检间隔

    # 端口策略（支持环境变量覆盖）
    PORT_RAND_LOW_1  = int(os.getenv("PORT_RAND_LOW_1", "9111"))
    PORT_RAND_HIGH_1 = int(os.getenv("PORT_RAND_HIGH_1", "9499"))
    PORT_RAND_LOW_2  = int(os.getenv("PORT_RAND_LOW_2", "20000"))
    PORT_RAND_HIGH_2 = int(os.getenv("PORT_RAND_HIGH_2", "48000"))
    PORT_SCAN_START  = int(os.getenv("PORT_SCAN_START", "49152"))
    PORT_SCAN_LIMIT  = int(os.getenv("PORT_SCAN_LIMIT", "10000"))

    # 自愈退避
    BACKOFF_MAX_SEC  = float(os.getenv("BACKOFF_MAX_SEC", "15.0"))
    BACKOFF_MIN_SEC  = float(os.getenv("BACKOFF_MIN_SEC", "1.5"))
    BACKOFF_GROWTH   = float(os.getenv("BACKOFF_GROWTH", "1.7"))

    # WDA Ready 等待（HTTP 轮询方式，不触发 xctest）
    WDA_READY_TIMEOUT = float(os.getenv("WDA_READY_TIMEOUT", "35.0"))

    # WDA 轻量复位策略
    MJPEG_BAD_THRESHOLD = int(os.getenv("MJPEG_BAD_THRESHOLD", "3"))   # 连续几次 mjpeg 健康失败才重置 WDA
    WDA_RESET_COOLDOWN  = float(os.getenv("WDA_RESET_COOLDOWN", "10")) # WDA 复位冷却，避免风暴

    # 防连坐参数（支持环境变量）
    GLITCH_SUPPRESS_SEC = float(os.getenv("GLITCH_SUPPRESS_SEC", "6.0"))   # 扫描异常后抑制移除的秒数
    MASS_DROP_RATIO     = float(os.getenv("MASS_DROP_RATIO", "0.6"))       # 一次性丢失占比阈值
    ABSENT_TICKS_BEFORE_REMOVE = int(os.getenv("ABSENT_TICKS_BEFORE_REMOVE", "3"))  # 连续缺席轮数

    def __init__(self):
        # 自增端口游标仅作兜底扫描使用
        self._port         = 9110
        self._models: Dict[str, DeviceModel] = {}
        self._procs: Dict[str, subprocess.Popen] = {}
        self._manager      = FlaskSubprocessManager.get_instance()
        self._iproxy_path  = self._find_iproxy()
        self._pool         = ThreadPoolExecutor(max_workers=6)

        self._last_heal_check_ts = 0.0
        self._heal_backoff: Dict[str, float] = {}  # udid -> next_allowed_ts

        # 并发保护 & 状态表
        self._lock = threading.RLock()
        self._port_by_udid: Dict[str, int] = {}   # UDID -> 当前使用的本地端口（映射 wdaScreenPort）
        self._pid_by_udid: Dict[str, int] = {}    # UDID -> iproxy PID

        # 抗抖
        self._last_seen: Dict[str, float]   = {}  # udid -> ts
        self._first_seen: Dict[str, float]  = {}  # udid -> ts(首次在线)
        self._last_topology_change_ts = 0.0

        # 短缓存：设备信任、WDA运行态（仅作节流）
        self._trusted_cache: Dict[str, float] = {}    # udid -> expire_ts
        self._wda_ok_cache: Dict[str, float] = {}     # udid -> expire_ts

        # 新增：MJPEG 连续坏计数 + 最近一次 WDA 复位时间
        self._mjpeg_bad_count: Dict[str, int] = {}
        self._last_wda_reset: Dict[str, float] = {}

        # 新增：按 UDID 的 /status 探测单飞锁，避免临时 iproxy 并发
        self._probe_locks: Dict[str, threading.Lock] = {}

        # 防连坐
        self._scan_glitch_until = 0.0               # 截止到该时间前，认为扫描不可靠，跳过移除
        self._absent_ticks: Dict[str, int] = {}     # udid -> 连续缺席次数

        LogManager.info("DeviceInfo init 完成；日志已启用", udid="system")

    # ---------------- 主循环 ----------------
    def listen(self):
        method = "listen"
        LogManager.method_info("进入主循环", method, udid="system")
        orphan_gc_tick = 0
        while True:
            now = _monotonic()
            try:
                usb = Usbmux().device_list()
                online_now = {d.udid for d in usb if d.conn_type == ConnectionType.USB}
            except Exception as e:
                LogManager.warning(f"[device_list] 异常：{e}", udid="system")
                time.sleep(1)
                continue

            # 记录“看到”的时间戳
            for u in online_now:
                if u not in self._first_seen:
                    self._first_seen[u] = now
                    LogManager.method_info("first seen", method, udid=u)
                self._last_seen[u] = now

            with self._lock:
                known = set(self._models.keys())

            # -------- 全局扫描异常检测（防连坐）--------
            missing = [u for u in known if u not in online_now]
            mass_drop = (len(known) > 0) and (
                (len(online_now) == 0) or
                (len(missing) / max(1, len(known)) >= self.MASS_DROP_RATIO)
            )
            if mass_drop:
                self._scan_glitch_until = now + self.GLITCH_SUPPRESS_SEC
                LogManager.method_warning(
                    f"检测到扫描异常：known={len(known)}, online={len(online_now)}, "
                    f"missing={len(missing)}，进入抑制窗口 {self.GLITCH_SUPPRESS_SEC}s",
                    method, udid="system"
                )

            # 真正移除（仅在非抑制窗口内 + 连续缺席达到阈值 才移除）
            for udid in list(known):
                if udid in online_now:
                    # 在线：清空缺席计数
                    self._absent_ticks.pop(udid, None)
                    continue

                # 离线：记录一次缺席
                miss = self._absent_ticks.get(udid, 0) + 1
                self._absent_ticks[udid] = miss

                last = self._last_seen.get(udid, 0.0)
                exceed_grace = (now - last) >= self.REMOVE_GRACE_SEC
                exceed_ticks = miss >= self.ABSENT_TICKS_BEFORE_REMOVE

                # 抑制窗口内：跳过任何移除
                if now < self._scan_glitch_until:
                    continue

                if exceed_grace and exceed_ticks:
                    # --- 移除前的“可达性”反校验 ---
                    try:
                        with self._lock:
                            model = self._models.get(udid)
                            port  = model.screenPort if model else -1
                        reachable = False
                        # 1) ip:port 的 MJPEG 是否还在
                        if port and port > 0 and self._health_check_mjpeg(port, timeout=0.8):
                            reachable = True
                        # 2) WDA /status 是否仍然正常
                        if not reachable and self._health_check_wda(udid):
                            reachable = True

                        if reachable:
                            # 误报：续命
                            self._last_seen[udid] = now
                            self._absent_ticks[udid] = 0
                            LogManager.method_info("离线误报：反校验可达，取消移除并续命", method, udid=udid)
                            continue
                    except Exception as e:
                        LogManager.method_warning(f"离线反校验异常：{e}", method, udid=udid)

                    LogManager.info(
                        f"设备判定离线（超过宽限期 {self.REMOVE_GRACE_SEC}s 且 连续缺席 {self._absent_ticks[udid]} 次）",
                        udid=udid
                    )
                    self._remove_device(udid)
                    self._last_topology_change_ts = now
                    # 清理计数
                    self._absent_ticks.pop(udid, None)

            # 真正新增（连续在线超过稳定期）
            new_candidates = [u for u in online_now if u not in known]
            to_add = [u for u in new_candidates if (now - self._first_seen.get(u, now)) >= self.ADD_STABLE_SEC]
            if to_add:
                LogManager.info(f"新增设备稳定上线：{to_add}", udid="system")
                futures = {self._pool.submit(self._add_device, u): u for u in to_add}
                try:
                    for f in as_completed(futures, timeout=45):
                        try:
                            f.result()
                            self._last_topology_change_ts = _monotonic()
                        except Exception as e:
                            LogManager.error(f"异步连接失败：{e}", udid="system")
                except TimeoutError:
                    for fut, u in futures.items():
                        if not fut.done():
                            fut.cancel()
                            LogManager.method_warning("新增设备任务超时已取消", method, udid=u)

            # 定期健康检查 + 自愈
            self._check_and_heal_tunnels(interval=self.HEAL_INTERVAL)

            # 周期性孤儿清理（拓扑变更冷却之后）
            orphan_gc_tick += 1
            if orphan_gc_tick >= 10:
                orphan_gc_tick = 0
                if (_monotonic() - self._last_topology_change_ts) >= self.ORPHAN_COOLDOWN:
                    self._cleanup_orphan_iproxy()

            time.sleep(1)

    # ---------------- 新增设备 ----------------
    def _add_device(self, udid: str):
        method = "_add_device"
        LogManager.method_info("开始新增设备", method, udid=udid)

        if not self._trusted(udid):
            LogManager.method_warning("未信任设备，跳过", method, udid=udid)
            return

        # 获取系统主版本
        try:
            dev = tidevice.Device(udid)
            system_version_major = int(dev.product_version.split(".")[0])
        except Exception as e:
            LogManager.method_warning(f"读取系统版本失败：{e}", method, udid=udid)
            system_version_major = 0  # 保底

        # === iOS>17：被动探测 WDA；未运行则交给 IOSActivator，并通过 HTTP 轮询等待 ===
        if system_version_major > 17:
            if self._wda_is_running(udid):
                LogManager.method_info("检测到 WDA 已运行，直接映射", method, udid=udid)
            else:
                LogManager.method_info("WDA 未运行，调用 IOSActivator（pymobiledevice3 自动挂载）", method, udid=udid)
                try:
                    ios = IOSActivator()
                    threading.Thread(target=ios.activate, args=(udid,), daemon=True).start()
                except Exception as e:
                    LogManager.method_error(f"IOSActivator 启动异常：{e}", method, udid=udid)
                    return
                # 关键：HTTP 轮询等待 WDA Ready（默认最多 35s），不会触发 xctest
                if not self._wait_wda_ready_http(udid, total_timeout_sec=self.WDA_READY_TIMEOUT):
                    LogManager.method_error("WDA 未在超时内就绪（iOS>17 分支）", method, udid=udid)
                    return
        else:
            # iOS <= 17：保持原逻辑（app_start + 简单等待）
            try:
                dev = tidevice.Device(udid)
                LogManager.method_info(f"app_start WDA: {WdaAppBundleId}", method, udid=udid)
                dev.app_start(WdaAppBundleId)
                time.sleep(3)
            except Exception as e:
                LogManager.method_error(f"WDA 启动异常：{e}", method, udid=udid)
                return

        # 获取屏幕信息
        w, h, s = self._screen_info(udid)
        if w == 0 or h == 0 or s == 0:
            LogManager.method_warning("未获取到屏幕信息，放弃新增", method, udid=udid)
            return

        # 启动 iproxy（不复用端口：直接新端口）
        proc = self._start_iproxy(udid, port=None)
        if not proc:
            LogManager.method_error("启动 iproxy 失败，放弃新增", method, udid=udid)
            return

        with self._lock:
            port = self._port_by_udid[udid]
            model = DeviceModel(deviceId=udid, screenPort=port, width=w, height=h, scale=s, type=1)
            model.ready = True
            self._models[udid] = model
            self._procs[udid]  = proc
            # 初始化计数
            self._mjpeg_bad_count[udid] = 0

        LogManager.method_info(f"设备添加完成，port={port}, {w}x{h}@{s}", method, udid=udid)
        self._manager_send(model)

    # ---------------- 移除设备（修复：总是发送离线通知） ----------------
    def _remove_device(self, udid: str):
        method = "_remove_device"
        LogManager.method_info("开始移除设备", method, udid=udid)
        with self._lock:
            model = self._models.pop(udid, None)
            proc  = self._procs.pop(udid, None)
            pid   = self._pid_by_udid.pop(udid, None)
            self._port_by_udid.pop(udid, None)
            # 清缓存，防止误判
            self._trusted_cache.pop(udid, None)
            self._wda_ok_cache.pop(udid, None)
            self._last_seen.pop(udid, None)
            self._first_seen.pop(udid, None)
            self._mjpeg_bad_count.pop(udid, None)
            self._last_wda_reset.pop(udid, None)
            self._absent_ticks.pop(udid, None)

        self._kill(proc)
        if pid:
            self._kill_pid_gracefully(pid)

        if model is None:
            # 构造一个最小的“离线模型”通知前端
            try:
                offline = DeviceModel(deviceId=udid, screenPort=-1, width=0, height=0, scale=0.0, type=2)
                offline.ready = False
                self._manager_send(offline)
                LogManager.method_info("设备移除完毕（无原模型，已发送离线通知）", method, udid=udid)
            except Exception as e:
                LogManager.method_warning(f"离线通知（构造模型）异常：{e}", method, udid=udid)
            return

        model.type = 2
        model.ready = False
        model.screenPort = -1
        try:
            self._manager_send(model)
        finally:
            LogManager.method_info("设备移除完毕（已发送离线通知）", method, udid=udid)

    # ---------------- 工具函数 ----------------
    def _trusted(self, udid: str) -> bool:
        # 30s 短缓存，减少 IO
        now = _monotonic()
        exp = self._trusted_cache.get(udid, 0.0)
        if exp > now:
            return True
        try:
            BaseDevice(udid).get_value("DeviceName")
            self._trusted_cache[udid] = now + 30.0
            return True
        except Exception:
            return False

    # ======= WDA 探测/等待（仅走 iproxy+HTTP，不触发 xctest） =======
    def _get_probe_lock(self, udid: str) -> threading.Lock:
        with self._lock:
            lk = self._probe_locks.get(udid)
            if lk is None:
                lk = threading.Lock()
                self._probe_locks[udid] = lk
            return lk

    def _wda_http_status_ok(self, udid: str, timeout_sec: float = 1.2) -> bool:
        """起临时 iproxy 到 wdaFunctionPort，探测 /status。增加单飞锁与严格清理。"""
        method = "_wda_http_status_ok"
        lock = self._get_probe_lock(udid)
        if not lock.acquire(timeout=3.0):
            # 有并发探测在进行，避免同时起多个 iproxy；直接返回“未知→False”
            LogManager.method_info("状态探测被并发锁抑制", method, udid=udid)
            return False
        try:
            tmp_port = self._pick_new_port()
            proc = None
            try:
                cmd = [self._iproxy_path, "-u", udid, str(tmp_port), str(wdaFunctionPort)]

                # --- Windows 下隐藏 iproxy 控制台 ---
                creationflags = 0
                startupinfo = None
                if os.name == "nt":
                    creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000) | \
                                    getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200)
                    si = subprocess.STARTUPINFO()
                    si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                    si.wShowWindow = 0
                    startupinfo = si

                proc = subprocess.Popen(
                    cmd,
                    stdout=subprocess.DEVNULL,
                    stderr=subprocess.DEVNULL,
                    creationflags=creationflags,
                    startupinfo=startupinfo
                )

                if not self._wait_until_listening(tmp_port, initial_timeout=1.0):
                    LogManager.method_info(f"WDA探测：临时端口未监听（{tmp_port}）", method, udid=udid)
                    return False

                # /status 双重尝试，减少瞬态抖动
                for _ in (1, 2):
                    try:
                        conn = http.client.HTTPConnection("127.0.0.1", tmp_port, timeout=timeout_sec)
                        conn.request("GET", "/status")
                        resp = conn.getresponse()
                        _ = resp.read(256)
                        code = getattr(resp, "status", 0)
                        ok = 200 <= code < 400
                        LogManager.method_info(f"WDA探测：/status code={code}, ok={ok}", method, udid=udid)
                        try:
                            conn.close()
                        except Exception:
                            pass
                        if ok:
                            return True
                        time.sleep(0.2)
                    except Exception as e:
                        LogManager.method_info(f"WDA探测异常：{e}", method, udid=udid)
                        time.sleep(0.2)
                return False
            finally:
                if proc:
                    try:
                        p = psutil.Process(proc.pid)
                        p.terminate()
                        try:
                            p.wait(timeout=1.2)
                        except psutil.TimeoutExpired:
                            p.kill()
                            p.wait(timeout=1.2)
                    except Exception:
                        # 兜底强杀
                        try:
                            os.kill(proc.pid, signal.SIGTERM)
                        except Exception:
                            pass
        finally:
            try:
                lock.release()
            except Exception:
                pass

    def _wait_wda_ready_http(self, udid: str, total_timeout_sec: float = None, interval_sec: float = 0.6) -> bool:
        """通过 _wda_http_status_ok 轮询等待 WDA Ready。"""
        method = "_wait_wda_ready_http"
        if total_timeout_sec is None:
            total_timeout_sec = self.WDA_READY_TIMEOUT
        deadline = _monotonic() + total_timeout_sec
        while _monotonic() < deadline:
            if self._wda_http_status_ok(udid, timeout_sec=1.2):
                LogManager.method_info("WDA 就绪（HTTP轮询）", method, udid=udid)
                return True
            time.sleep(interval_sec)
        LogManager.method_warning(f"WDA 等待超时（HTTP轮询，{total_timeout_sec}s）", method, udid=udid)
        return False

    def _wda_is_running(self, udid: str, cache_sec: float = 2.0) -> bool:
        """轻量速查，走 HTTP /status（短缓存节流），避免触发 xctest。"""
        now = _monotonic()
        exp = self._wda_ok_cache.get(udid, 0.0)
        if exp > now:
            return True
        ok = self._wda_http_status_ok(udid, timeout_sec=1.2)
        if ok:
            self._wda_ok_cache[udid] = now + cache_sec
        return ok

    def _screen_info(self, udid: str):
        method = "_screen_info"
        try:
            c = wda.USBClient(udid, wdaFunctionPort)
            c.home()
            size = c.window_size()
            scale = c.scale
            LogManager.method_info(f"屏幕信息：{int(size.width)}x{int(size.height)}@{float(scale)}", method, udid=udid)
            return int(size.width), int(size.height), float(scale)
        except Exception as e:
            LogManager.method_warning(f"获取屏幕信息异常：{e}", method, udid=udid)
            return 0, 0, 0

    # ---------------- 端口/进程：不复用端口 ----------------
    def _is_port_free(self, port: int, host: str = "127.0.0.1") -> bool:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            s.bind((host, port))
            return True
        except OSError:
            return False
        finally:
            s.close()

    def _pick_new_port(self, tries: int = 40) -> int:
        method = "_pick_new_port"
        for _ in range(max(1, tries // 2)):
            p = random.randint(self.PORT_RAND_LOW_1, self.PORT_RAND_HIGH_1)
            if self._is_port_free(p):
                LogManager.method_info(f"端口候选可用(首段)：{p}", method, udid="system")
                return p
            else:
                LogManager.method_info(f"端口候选占用(首段)：{p}", method, udid="system")
        for _ in range(tries):
            p = random.randint(self.PORT_RAND_LOW_2, self.PORT_RAND_HIGH_2)
            if self._is_port_free(p):
                LogManager.method_info(f"端口候选可用(次段)：{p}", method, udid="system")
                return p
            else:
                LogManager.method_info(f"端口候选占用(次段)：{p}", method, udid="system")
        LogManager.method_warning("随机端口尝试耗尽，改顺序扫描", method, udid="system")
        return self._pick_free_port(start=self.PORT_SCAN_START, limit=self.PORT_SCAN_LIMIT)

    def _wait_until_listening(self, port: int, initial_timeout: float = 2.0) -> bool:
        """自适应等待端口监听：2s -> 3s -> 5s（最多约10s）。"""
        method = "_wait_until_listening"
        timeouts = [initial_timeout, 3.0, 5.0]
        for to in timeouts:
            deadline = _monotonic() + to
            while _monotonic() < deadline:
                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                    s.settimeout(0.25)
                    if s.connect_ex(("127.0.0.1", port)) == 0:
                        LogManager.method_info(f"端口已开始监听：{port}", method, udid="system")
                        return True
                time.sleep(0.06)
            LogManager.method_info(f"监听验收阶段超时：{port}，扩展等待", method, udid="system")
        LogManager.method_warning(f"监听验收最终超时：{port}", method, udid="system")
        return False

    def _start_iproxy(self, udid: str, port: Optional[int] = None) -> Optional[subprocess.Popen]:
        method = "_start_iproxy"
        try:
            with self._lock:
                old_pid = self._pid_by_udid.get(udid)
                if old_pid:
                    LogManager.method_info(f"发现旧 iproxy，准备结束：pid={old_pid}", method, udid=udid)
                    self._kill_pid_gracefully(old_pid)
                    self._pid_by_udid.pop(udid, None)

            attempts = 0
            while attempts < 3:
                attempts += 1
                local_port = port if (attempts == 1 and port is not None) else self._pick_new_port()
                if not self._is_port_free(local_port):
                    LogManager.method_info(f"[attempt {attempts}] 端口竞争，换候选：{local_port}", method, udid=udid)
                    continue

                LogManager.method_info(f"[attempt {attempts}] 启动 iproxy，port={local_port}", method, udid=udid)

                creationflags = 0
                startupinfo = None
                if os.name == "nt":
                    creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000) | \
                                    getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200)
                    si = subprocess.STARTUPINFO()
                    si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                    si.wShowWindow = 0
                    startupinfo = si

                cmd = [self._iproxy_path, "-u", udid, str(local_port), str(wdaScreenPort)]
                try:
                    proc = subprocess.Popen(
                        cmd,
                        stdout=subprocess.DEVNULL,
                        stderr=subprocess.DEVNULL,
                        creationflags=creationflags,
                        startupinfo=startupinfo
                    )
                except Exception as e:
                    LogManager.method_warning(f"创建 iproxy 进程失败：{e}", method, udid=udid)
                    continue

                if not self._wait_until_listening(local_port, initial_timeout=2.0):
                    LogManager.method_warning(f"[attempt {attempts}] iproxy 未监听，重试换端口", method, udid=udid)
                    self._kill(proc)
                    continue

                with self._lock:
                    self._procs[udid] = proc
                    self._pid_by_udid[udid] = proc.pid
                    self._port_by_udid[udid] = local_port

                LogManager.method_info(f"iproxy 启动成功并监听，pid={proc.pid}, port={local_port}", method, udid=udid)
                return proc

            LogManager.method_error("iproxy 启动多次失败", method, udid=udid)
            return None

        except Exception as e:
            LogManager.method_error(f"_start_iproxy 异常：{e}", method, udid=udid)
            return None

    def _kill(self, proc: Optional[subprocess.Popen]):
        method = "_kill"
        if not proc:
            return
        try:
            p = psutil.Process(proc.pid)
            p.terminate()
            try:
                p.wait(timeout=2.0)
                LogManager.method_info("进程已正常终止", method, udid="system")
            except psutil.TimeoutExpired:
                p.kill()
                LogManager.method_warning("进程被强制杀死", method, udid="system")
        except Exception as e:
            LogManager.method_warning(f"结束进程异常：{e}", method, udid="system")

    # ---------------- 自愈：直接换新端口重启 + 指数退避 ----------------
    def _next_backoff(self, prev_backoff: float) -> float:
        if prev_backoff <= 0:
            return self.BACKOFF_MIN_SEC
        return min(prev_backoff * self.BACKOFF_GROWTH, self.BACKOFF_MAX_SEC)

    def _restart_iproxy(self, udid: str):
        method = "_restart_iproxy"
        now = _monotonic()
        next_allowed = self._heal_backoff.get(udid, 0.0)
        if now < next_allowed:
            delta = round(next_allowed - now, 2)
            LogManager.method_info(f"自愈被退避抑制，剩余 {delta}s", method, udid=udid)
            return

        old_port = None
        with self._lock:
            proc = self._procs.get(udid)
            if proc:
                LogManager.method_info(f"为重启准备清理旧 iproxy，pid={proc.pid}", method, udid=udid)
                self._kill(proc)
            model = self._models.get(udid)
            if not model:
                LogManager.method_warning("模型不存在，取消自愈", method, udid=udid)
                return
            old_port = model.screenPort

        proc2 = self._start_iproxy(udid, port=None)
        if not proc2:
            prev = max(0.0, next_allowed - now)
            backoff = self._next_backoff(prev)
            self._heal_backoff[udid] = now + backoff
            LogManager.method_warning(f"重启失败，扩展退避 {round(backoff,2)}s", method, udid=udid)
            return

        # 成功后短退避（抑制频繁重启）
        self._heal_backoff[udid] = now + 1.2

        # 通知前端新端口
        with self._lock:
            model = self._models.get(udid)
            if model:
                model.screenPort = self._port_by_udid.get(udid, model.screenPort)
                self._models[udid] = model
                self._manager_send(model)
        LogManager.method_info(f"[PORT-SWITCH] {udid} {old_port} -> {self._port_by_udid.get(udid)}", method, udid=udid)
        LogManager.method_info(f"重启成功，使用新端口 {self._port_by_udid.get(udid)}", method, udid=udid)

    # ---------------- 健康检查 ----------------
    def _health_check_mjpeg(self, port: int, timeout: float = 1.8) -> bool:
        """使用 GET 真实探测 MJPEG：校验 Content-Type 和 boundary。尝试 /mjpeg -> /mjpegstream -> /"""
        method = "_health_check_mjpeg"
        paths = ["/mjpeg", "/mjpegstream", "/"]
        for path in paths:
            try:
                conn = http.client.HTTPConnection("127.0.0.1", port, timeout=timeout)
                conn.request("GET", path, headers={"Connection": "close"})
                resp = conn.getresponse()
                ctype = (resp.getheader("Content-Type") or "").lower()
                ok_hdr = (200 <= resp.status < 300) and ("multipart/x-mixed-replace" in ctype)
                # 仅读少量字节，不阻塞
                chunk = resp.read(1024)
                try:
                    conn.close()
                except Exception:
                    pass
                if ok_hdr and (b"--" in chunk):
                    return True
            except Exception:
                pass
        return False

    def _health_check_wda(self, udid: str) -> bool:
        """使用 HTTP 探测（带短缓存），避免触发 xctest。"""
        # 加一次重试，减少瞬态波动
        if self._wda_is_running(udid, cache_sec=1.0):
            return True
        time.sleep(0.2)
        return self._wda_is_running(udid, cache_sec=1.0)

    def _maybe_reset_wda_lightweight(self, udid: str) -> bool:
        """在 MJPEG 多次异常但 /status 正常时，做 WDA 轻量复位。成功返回 True。"""
        method = "_maybe_reset_wda_lightweight"
        now = _monotonic()
        last = self._last_wda_reset.get(udid, 0.0)
        if now - last < self.WDA_RESET_COOLDOWN:
            return False

        LogManager.method_warning("MJPEG 连续异常，尝试 WDA 轻量复位", method, udid=udid)
        try:
            dev = tidevice.Device(udid)
            # 先尝试 stop/start
            try:
                dev.app_stop(WdaAppBundleId)
                time.sleep(1.0)
            except Exception:
                pass
            dev.app_start(WdaAppBundleId)
            # 等待就绪（缩短等待）
            if self._wait_wda_ready_http(udid, total_timeout_sec=12.0):
                self._last_wda_reset[udid] = _monotonic()
                return True
        except Exception as e:
            LogManager.method_warning(f"WDA stop/start 失败：{e}", method, udid=udid)

        # 兜底：iOS18+ 用 IOSActivator 再尝试
        try:
            ios = IOSActivator()
            ios.activate(udid)
            if self._wait_wda_ready_http(udid, total_timeout_sec=12.0):
                self._last_wda_reset[udid] = _monotonic()
                return True
        except Exception as e:
            LogManager.method_warning(f"IOSActivator 复位失败：{e}", method, udid=udid)

        return False

    def _check_and_heal_tunnels(self, interval: float = 5.0):
        method = "_check_and_heal_tunnels"
        now = _monotonic()
        if now - self._last_heal_check_ts < interval:
            return
        self._last_heal_check_ts = now

        if (now - self._last_topology_change_ts) < max(self.ORPHAN_COOLDOWN, 6.0):
            LogManager.method_info("拓扑变更冷却中，本轮跳过自愈", method, udid="system")
            return

        with self._lock:
            items = list(self._models.items())

        for udid, model in items:
            port = model.screenPort
            if port <= 0:
                continue

            ok_local = self._health_check_mjpeg(port, timeout=1.8)
            ok_wda   = self._health_check_wda(udid)
            LogManager.method_info(f"健康检查：mjpeg={ok_local}, wda={ok_wda}, port={port}", method, udid=udid)

            if ok_local and ok_wda:
                self._mjpeg_bad_count[udid] = 0
                continue

            # 分层自愈：MJPEG 连续异常而 WDA 正常 → 优先复位 WDA
            if (not ok_local) and ok_wda:
                cnt = self._mjpeg_bad_count.get(udid, 0) + 1
                self._mjpeg_bad_count[udid] = cnt
                if cnt >= self.MJPEG_BAD_THRESHOLD:
                    if self._maybe_reset_wda_lightweight(udid):
                        # 复位成功后重启 iproxy，确保新流映射
                        self._restart_iproxy(udid)
                        self._mjpeg_bad_count[udid] = 0
                        continue  # 下一个设备
                # 若未达门槛或复位失败，仍执行 iproxy 重启
                LogManager.method_warning(f"检测到不健康，触发重启；port={port}", method, udid=udid)
                self._restart_iproxy(udid)
                continue

            # 其他情况（wda 不健康或两者都不健康）：先重启 iproxy
            LogManager.method_warning(f"检测到不健康，触发重启；port={port}", method, udid=udid)
            self._restart_iproxy(udid)

    # ---------------- 进程枚举（结构化返回） ----------------
    def _get_all_iproxy_entries(self) -> List[Dict[str, Any]]:
        """
        返回结构化 iproxy 进程项：
        { 'pid': int, 'name': str, 'cmdline': List[str], 'udid': str|None, 'local_port': int|None, 'remote_port': int|None }
        """
        method = "_get_all_iproxy_entries"
        entries: List[Dict[str, Any]] = []
        is_windows = os.name == "nt"
        target_name = "iproxy.exe" if is_windows else "iproxy"

        for p in psutil.process_iter(attrs=["name", "cmdline", "pid"]):
            try:
                name = (p.info.get("name") or "").lower()
                if name != target_name:
                    continue
                cmdline = p.info.get("cmdline") or []
                if not cmdline:
                    continue

                udid = None
                local_port = None
                remote_port = None

                # 解析 -u <udid> 与后续的两个端口（LOCAL_PORT, REMOTE_PORT）
                if "-u" in cmdline:
                    try:
                        i = cmdline.index("-u")
                        if i + 1 < len(cmdline):
                            udid = cmdline[i + 1]
                        # 在 -u udid 之后扫描数字端口
                        ints = []
                        for token in cmdline[i + 2:]:
                            if token.isdigit():
                                ints.append(int(token))
                            # 停止条件：拿到两个
                            if len(ints) >= 2:
                                break
                        if len(ints) >= 2:
                            local_port, remote_port = ints[0], ints[1]
                        else:
                            # 兜底：全局找两个数字
                            ints2 = [int(t) for t in cmdline if t.isdigit()]
                            if len(ints2) >= 2:
                                local_port, remote_port = ints2[-2], ints2[-1]
                    except Exception:
                        pass

                entries.append({
                    "pid": p.info["pid"],
                    "name": name,
                    "cmdline": cmdline,
                    "udid": udid,
                    "local_port": local_port,
                    "remote_port": remote_port
                })
            except (psutil.NoSuchProcess, psutil.AccessDenied):
                continue

        LogManager.method_info(f"扫描到候选 iproxy 进程数={len(entries)}", method, udid="system")
        return entries

    # ---------------- 杀孤儿（含“同 UDID 的非当前实例”清理） ----------------
    def _cleanup_orphan_iproxy(self):
        method = "_cleanup_orphan_iproxy"

        with self._lock:
            live_udids = set(self._models.keys())
            live_pid_by_udid = dict(self._pid_by_udid)
            live_port_by_udid = dict(self._port_by_udid)

        cleaned = 0
        for ent in self._get_all_iproxy_entries():
            pid = ent["pid"]
            udid = ent.get("udid")
            local_port = ent.get("local_port")
            # 完全不认识的进程（无法解析 udid），跳过
            if not udid:
                continue

            # 1) 完全孤儿：udid 不在活跃设备集，且 pid 不是任何已跟踪 pid → 杀
            if udid not in live_udids and pid not in live_pid_by_udid.values():
                self._kill_pid_gracefully(pid, silent=True)
                cleaned += 1
                LogManager.method_info(f"孤儿 iproxy 已清理：udid={udid}, pid={pid}", method)
                continue

            # 2) 同 UDID 的非当前实例：udid 活跃，但 pid != 当前 pid，且本地端口也不是当前端口 → 杀
            live_pid = live_pid_by_udid.get(udid)
            live_port = live_port_by_udid.get(udid)
            if udid in live_udids and pid != live_pid:
                if (local_port is None) or (live_port is None) or (local_port != live_port):
                    self._kill_pid_gracefully(pid, silent=True)
                    cleaned += 1
                    LogManager.method_info(f"清理同UDID旧实例：udid={udid}, pid={pid}, local_port={local_port}", method)

        if cleaned:
            LogManager.method_info(f"孤儿清理完成，数量={cleaned}", method)

    # ---------------- 按 PID 强杀 ----------------
    def _kill_pid_gracefully(self, pid: int, silent: bool = False):
        """优雅地结束进程，不弹出cmd窗口"""
        try:
            if platform.system() == "Windows":
                # 不弹窗方式
                subprocess.run(
                    ["taskkill", "/PID", str(pid), "/F", "/T"],
                    stdout=subprocess.DEVNULL if silent else None,
                    stderr=subprocess.DEVNULL if silent else None,
                    creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0x08000000),
                )
            else:
                # Linux / macOS
                os.kill(pid, signal.SIGTERM)
        except Exception as e:
            LogManager.method_error(f"结束进程 {pid} 失败: {e}", "_kill_pid_gracefully")

    # ---------------- 端口工具（兜底） ----------------
    def _pick_free_port(self, start: int = None, limit: int = 2000) -> int:
        method = "_pick_free_port"
        p = self._port if start is None else start
        tried = 0
        while tried < limit:
            p += 1
            tried += 1
            if self._is_port_free(p):
                LogManager.method_info(f"顺序扫描找到端口：{p}", method, udid="system")
                return p
        LogManager.method_error("顺序扫描未找到可用端口（范围内）", method, udid="system")
        raise RuntimeError("未找到可用端口（扫描范围内）")

    # ---------------- 其他 ----------------
    def _manager_send(self, model: DeviceModel):
        method = "_manager_send"
        try:
            self._manager.send(model.toDict())
            LogManager.method_info("已通知管理器（前端）", method, udid=model.deviceId)
        except Exception as e:
            LogManager.method_warning(f"通知管理器异常：{e}", method, udid=model.deviceId)

    def _find_iproxy(self) -> str:
        """优先环境变量 IPROXY_PATH；否则按平台在 resources/iproxy 查找。"""
        method = "_find_iproxy"

        env_path = os.getenv("IPROXY_PATH")
        if env_path and Path(env_path).is_file():
            LogManager.method_info(f"使用环境变量指定的 iproxy 路径：{env_path}", method, udid="system")
            return env_path

        base = Path(__file__).resolve().parent.parent
        is_windows = os.name == "nt"
        name = "iproxy.exe" if is_windows else "iproxy"
        path = base / "resources" / "iproxy" / name
        LogManager.method_info(f"查找 iproxy 路径：{path}", method, udid="system")
        if path.is_file():
            return str(path)

        err = f"iproxy 不存在: {path}"
        LogManager.method_error(err, method, udid="system")
        raise FileNotFoundError(err)