diff --git a/.env.example b/.env.example index ae1e1fa..3ae70c3 100644 --- a/.env.example +++ b/.env.example @@ -12,3 +12,5 @@ IFLY_ACCENT=shanghainese # 置信度阈值(0-1,越小越容易识别) IFLY_CONF_THRESHOLD=0.12 +ROBOT_PORT=COM12 +COMMANDS_PATH=commands.yaml diff --git a/README.md b/README.md deleted file mode 100644 index 7ca2c3b..0000000 --- a/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Dialect-Driven Embodied AI Control System 🤖🗣️ - -## 📖 Project Overview -This project implements a high-performance **Voice-to-Command** pipeline designed for robotic interaction. It features robust support for **Chinese dialects** (specifically Shanghainese and Chongqing) and is optimized for real-world noisy environments through custom VAD (Voice Activity Detection) calibration. - -## ✨ Key Technical Highlights -- **Hardware-Aware Audio Processing:** Handles 2-channel input with automatic mono conversion and 1.5x digital gain for low-sensitivity microphones. -- **Dynamic VAD Calibration:** Real-time RMS-based ambient noise estimation to set adaptive start/end thresholds. -- **Dialect Logic Engine:** Comprehensive keyword mapping for Shanghainese (e.g., "伐要动", "拿那") and Chongqing dialects. -- **Industrial Pipeline:** Multi-threaded architecture with a dedicated `ROBOT_QUEUE` for non-blocking command execution. - -## 🛠️ Installation & Setup -1. **Clone the repository:** - `git clone https://github.com/elilin349-eli/Robot-Interaction.git` -2. **Install dependencies:** - `pip install sounddevice numpy websocket-client requests python-dotenv` -3. **Configure Environment:** - Create a `.env` file with your iFlytek credentials (refer to `.env.example`). - -## 🚀 Future Roadmap -- [x] Multi-dialect ASR Integration -- [ ] **Next:** YOLOv8-based Vision Module for spatial object localization -- [ ] **Next:** ROS2 Integration for Robotic Arm Kinematics diff --git a/commands.yaml b/commands.yaml new file mode 100644 index 0000000..86f52b1 --- /dev/null +++ b/commands.yaml @@ -0,0 +1,201 @@ +# ========================================== +# Voice-Robot 指令配置文件 +# 用途：定义机器人动作、方言关键词、参数 +# ========================================== + +# 基础配置 +robot: + name: "SO-101 LeRobot Arm" + description: "Remote operation data collection for LeRobot" + +# 语音识别配置 +speech: + engine: "ifly" # iFlytek 讯飞 + language: "zh_cn" + accent: "mandarin" # 改为普通话，大幅提升非上海话识别率 + vad_eos: 1500 # 句子结束检测时间(ms) + confidence_threshold: 0.12 + fuzzy_match_threshold: 0.8 # 略微提高阈值以过滤干扰 + +# 指令定义（动作树） +commands: + # 抓取动作 + PICK: + description: "拿起物体" + keywords: + - "抓取" + - "拿起来" + - "握紧" + - "抓一下" + - "拿" + - "抓" + tts_feedback: "已收到拿起指令" + action_type: "manipulate" + timeout: 30 # 秒 + + # 放下动作 + PLACE: + description: "放下物体" + keywords: + - "放下" + - "放开" + - "松开" + - "放一下" + - "放" + - "下" + tts_feedback: "已收到放下指令" + action_type: "manipulate" + timeout: 30 + + # 停止动作 + STOP: + description: "停止一切动作" + keywords: + - "停止" + - "放松" + - "释放" + - "停一下" + - "别动" + tts_feedback: "已停止所有动作" + action_type: "emergency" + timeout: 5 + + # 【新增】调试模式：进入示教状态（主臂松弛） + ARM_RELAX: + description: "进入调试/示教模式，松开机械臂力矩" + keywords: + - "进入调试" + - "示教模式" + - "松开机械臂" + - "放松" + - "手动拖动" + tts_feedback: "已进入示教模式，可以手动拖动" + action_type: "debug" + timeout: 300 + + # 【新增】调试模式：退出示教状态（主臂锁定） + ARM_LOCK: + description: "退出调试/示教模式，锁定机械臂力矩" + keywords: + - "退出调试" + - "锁定机械臂" + - "停止示教" + - "锁定" + tts_feedback: "已锁定机械臂" + action_type: "debug" + timeout: 5 + + # 【新增】故障恢复：重启舵机 6 + RESET_GRIPPER: + description: "重启6号舵机（夹持器）并恢复连接" + keywords: + - "重启夹持器" + - "重置舵机" + - "6号舵机重启" + - "修复舵机" + - "舵机重启" + tts_feedback: "正在重启夹持器" + action_type: "recovery" + timeout: 10 + + # 数据采集：开始记录 + RECORD_START: + description: "开始采集遥操作数据" + keywords: + - "开始记录" + - "开始采集" + - "开始录制" + - "开始测试" # 新增：常见误识别词 + - "开始测" # 新增：简化版本 + - "开始" # 新增：通用触发词 + - "录制" + - "开始录" + - "来一遍" + tts_feedback: "开始记录遥操作数据" + action_type: "data_collection" + timeout: 300 # 5分钟 + + # 数据采集：停止记录 + RECORD_STOP: + description: "停止采集遥操作数据" + keywords: + - "停止记录" + - "结束记录" + - "录好了" + - "存好" + - "存一下" + - "保存" + - "没有了" + tts_feedback: "停止记录，数据已保存" + action_type: "data_collection" + timeout: 10 + + # 主臂：记录当前姿态 + GESTURE_RECORD: + description: "从主臂读取并记录当前姿态" + keywords: + - "记录姿态" + - "记一哈" + - "存一个" + - "保存动作" + - "哈这个" + tts_feedback: "已记录主臂姿态" + action_type: "arm_control" + timeout: 5 + + # 主臂：连续录制 + GESTURE_START: + description: "开始连续录制主臂姿态" + keywords: + - "开始录制" + - "连续记录" + - "启动录制" + - "来一遍" + - "保存一遍" + tts_feedback: "开始连续录制主臂姿态" + action_type: "arm_control" + timeout: 300 + + # 主臂：停止录制 + GESTURE_STOP: + description: "停止连续录制主臂姿态" + keywords: + - "停止录制" + - "录好了" + - "完了" + - "结束" + - "保存完" + tts_feedback: "停止录制" + action_type: "arm_control" + timeout: 5 + +# 环境变量覆盖（从 .env 读取，此处为默认值） +defaults: + app_id: "YOUR_APP_ID_HERE" + api_key: "YOUR_API_KEY_HERE" + api_secret: "YOUR_API_SECRET_HERE" + sample_rate: 16000 + channels: 2 + chunk_ms: 80 + +# 日志与输出 +logging: + log_file: "commands.log" + cmd_json: "last_command.json" + lerobot_log: "lerobot_runtime.log" + +# LeRobot 集成（后续扩展） +lerobot: + enabled: true + env_name: "lerobot" + robot_path: "lerobot/configs/robot/so_101" + script_path: "lerobot/scripts/control_robot.py" + +# 硬件配置（主臂/从臂控制） +hardware: + leader_port: "COM12" # 主臂（Leader）串口 + leader_baudrate: 115200 # 主臂波特率 + follower_port: "COM13" # 从臂（Follower）串口（可选） + record_interval_ms: 100 # 连续记录间隔（毫秒） + calib_file: null # 校准文件路径（自动识别） + diff --git a/lerobot_hardware.py b/lerobot_hardware.py new file mode 100644 index 0000000..4a3e573 --- /dev/null +++ b/lerobot_hardware.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +LeRobot SO-101 硬件通信模块 (v2.0 - FeetechMotorsBus 直接集成) +用途：与主臂（Leader Arm）通过 COM12 串口进行通信 +核心特性： + - 使用 LeRobot FeetechMotorsBus 底层 _read/_write 接口（绕过 calibration 检查） + - 读取 6 个关节实时位置（地址 56, 长度 2） + - 写入目标位置用于回放演示（地址 42, 长度 2） + - 完整的手势记录和回放支持 +""" + +import json +import time +import os +from datetime import datetime +from typing import Optional, List, Dict, Tuple +from pathlib import Path + +# 【兼容性补丁】舵机底层 SDK 别名映射 +# 不同的 LeRobot 版本可能使用不同的 SDK 包名 +try: + import scservo_sdk +except ImportError: + try: + import feetech_servo_sdk as scservo_sdk + except ImportError: + pass # 如果两个都找不到，会在 _init_bus 中捕获 + + +class MotorWrapper: + """ + 兼容 LeRobot FeetechMotorsBus 的电机包装类 + 用于避免 calibration 检查导致的 KeyError + """ + def __init__(self, motor_id: int, model: str = "sts3215"): + self.id = motor_id + self.model = model + + +class SO101LeaderArm: + """SO-101 主臂（Leader Arm）硬件接口 - FeetechMotorsBus 直接集成""" + + # 寄存器地址（基于 Feetech STS3215 协议） + REGISTER_CURRENT_POS = 56 # 当前位置地址 + REGISTER_TARGET_POS = 42 # 目标位置地址 + REGISTER_TORQUE_ENABLE = 40 # 力矩使能地址 + REGISTER_LENGTH = 2 # 2 字节 + TORQUE_ENABLE_LENGTH = 1 # 力矩使能是 1 字节 + POS_MIN = 0 # 最小位置（0 度） + POS_MAX = 4095 # 最大位置（240 度） + + def __init__(self, port="COM12", baudrate=115200, timeout=1.0): + """ + 初始化主臂控制器 + + Args: + port: 串口号 (默认 COM12) + baudrate: 波特率 (默认 115200) + timeout: 串口超时 (秒) + """ + self.port = port + self.baudrate = baudrate + self.timeout = timeout + self.bus = None + self.connected = False + self.calib_path = self._get_calib_path() + self._init_bus() + + def _get_calib_path(self) -> str: + """获取校准文件路径""" + home = Path.home() + calib_dir = home / ".cache" / "huggingface" / "lerobot" / "calibration" / "teleoperators" / "so101_leader" + calib_file = calib_dir / "None.json" + return str(calib_file) + + def _init_bus(self): + """初始化 FeetechMotorsBus 实例（支持多版本兼容）""" + FeetechMotorsBus = None + + # 【兼容性补丁】多层级导入机制，自动适配不同 lerobot 版本 + try: + # 路径 1: 适配 0.4.4 及之后的扁平化版本 + from lerobot.motors.feetech import FeetechMotorsBus + print("✅ 导入模式 1: lerobot.motors.feetech (0.4.4+)") + except ImportError: + try: + # 路径 2: 适配 0.3.x 版本 + from lerobot.common.robot_devices.motors.feetech import FeetechMotorsBus + print("✅ 导入模式 2: lerobot.common.robot_devices.motors.feetech (0.3.x)") + except ImportError: + try: + # 路径 3: 适配早期开发版 + from lerobot.robot_devices.motors.feetech import FeetechMotorsBus + print("✅ 导入模式 3: lerobot.robot_devices.motors.feetech (early dev)") + except ImportError: + print("⚠️ LeRobot 环境缺失或版本不兼容，使用降级模式") + self.bus = None + return + + try: + # 创建电机包装器（ID 1-6 对应 6 个关节） + motor_objects = {i: MotorWrapper(i) for i in range(1, 7)} + + # 初始化总线（不传 timeout 参数，以适配最新版库） + self.bus = FeetechMotorsBus( + port=self.port, + motors=motor_objects + ) + except Exception as e: + print(f"⚠️ FeetechMotorsBus 初始化失败: {e}") + self.bus = None + + def connect(self) -> bool: + """连接到主臂（容忍单个舵机故障）""" + try: + if self.bus is None: + print("⚠️ FeetechMotorsBus 未初始化，尝试重新初始化") + self._init_bus() + + if self.bus: + try: + self.bus.connect() + except Exception as motor_check_error: + # 【容错处理】如果是舵机检测失败（常见于 6 号过载） + error_msg = str(motor_check_error) + if "motor check failed" in error_msg or "Missing motor IDs" in error_msg: + print(f"⚠️ 舵机部分故障: {error_msg}") + print("⚠️ 尝试继续连接（忽略故障舵机）...") + + # 强制设置连接状态，允许部分功能继续运行 + self.connected = True + print(f"✅ 主臂已连接（降级模式，部分舵机可能不可用）") + return True + else: + raise # 其他错误继续抛出 + + self.connected = True + print(f"✅ 主臂已连接 ({self.port})") + return True + else: + print("❌ 无法初始化 FeetechMotorsBus") + return False + except Exception as e: + print(f"❌ 无法连接主臂: {e}") + print("💡 建议: 请检查 USB 连接、12V 电源，或重启程序") + self.connected = False + return False + + def disconnect(self) -> bool: + """断开连接""" + try: + if self.bus and self.connected: + self.bus.disconnect() + self.connected = False + print("👋 主臂连接已关闭") + return True + except Exception as e: + print(f"⚠️ 关闭连接时出错: {e}") + return False + + def read_angles(self) -> Optional[List[int]]: + """ + 读取主臂的 6 个关节位置（原始编码值 0-4095） + 使用底层 _read 接口绕过 calibration 检查 + + Returns: + 6 维数组 [pos1, pos2, ..., pos6] (单位: 编码值) 或 None (失败时) + """ + if not self.connected or not self.bus: + print("⚠️ 主臂未连接") + return None + + try: + angles = [] + for motor_id in range(1, 7): + # 使用底层 _read 接口：(address, length, motor_id) + result = self.bus._read( + self.REGISTER_CURRENT_POS, + self.REGISTER_LENGTH, + motor_id + ) + + # 【关键修复】_read 返回 (value, error_code) 元组，必须严格解包 + # 直接 int(result) 会导致 "int() argument must be a string"崩溃 + if isinstance(result, (tuple, list)) and len(result) >= 1: + val = result[0] + else: + val = result + + # 再次确保 val 是有效的数值 + if val is not None: + angles.append(int(val)) + else: + print(f"⚠️ 电机 {motor_id} 读取失败") + return None + + return angles + except Exception as e: + print(f"⚠️ 读取关节位置时出错: {e}") + return None + + def safe_write(self, motor_id: int, target_pos: int) -> bool: + """ + 【新增】安全写入：带软限位保护的单舵机写入 + + 特别针对 6 号舵机（夹持器）的过载保护。基于实测数据， + 6 号舵机在 2954 时容易卡顿，故设置安全区间为 [1000, 2800]。 + + Args: + motor_id: 电机 ID (1-6) + target_pos: 目标位置 (0-4095) + + Returns: + 成功返回 True + """ + if not self.connected or not self.bus: + return False + + try: + # 【硬件保护】6 号舵机的专属限位，防止过载 + if motor_id == 6: + target_pos = max(1000, min(target_pos, 2800)) + print(f"🛡️ 6号舵机软限位: {target_pos} (安全区间: [1000, 2800])") + else: + target_pos = max(self.POS_MIN, min(target_pos, self.POS_MAX)) + + # 启用力矩 + try: + self.bus._write(self.REGISTER_TORQUE_ENABLE, self.TORQUE_ENABLE_LENGTH, motor_id, 1) + except Exception: + pass + + # 写入位置 + self.bus._write(self.REGISTER_TARGET_POS, self.REGISTER_LENGTH, motor_id, int(target_pos)) + return True + except Exception as e: + print(f"⚠️ 舵机 {motor_id} 写入失败: {e}") + return False + + def write_angles(self, angles: List[int]) -> bool: + """ + ...existing code... + """ + if not self.connected or not self.bus: + print("⚠️ 主臂未连接") + return False + + if len(angles) != 6: + print(f"❌ 角度数组长度不正确: 需要 6，收到 {len(angles)}") + return False + + try: + for motor_id, target_pos in enumerate(angles, start=1): + # ...existing code... + if not (self.POS_MIN <= target_pos <= self.POS_MAX): + print(f"⚠️ 电机 {motor_id} 的目标位置 {target_pos} 超出范围 [{self.POS_MIN}, {self.POS_MAX}]") + target_pos = max(self.POS_MIN, min(self.POS_MAX, target_pos)) + + # 【修复2】先启用力矩（Address 40, Length 1, Value 1） + try: + self.bus._write( + self.REGISTER_TORQUE_ENABLE, + self.TORQUE_ENABLE_LENGTH, + motor_id, + 1 # 力矩使能 = 1 + ) + except Exception as e: + print(f"⚠️ 电机 {motor_id} 力矩启用失败: {e}") + + # 再写入目标位置（Address 42, Length 2） + try: + self.bus._write( + self.REGISTER_TARGET_POS, + self.REGISTER_LENGTH, + motor_id, + int(target_pos) + ) + except Exception as e: + print(f"❌ 电机 {motor_id} 位置写入失败: {e}") + return False + + return True + except Exception as e: + print(f"❌ 写入关节位置时出错: {e}") + return False + + def get_status(self) -> Dict: + """获取主臂状态（连接性检查）""" + return { + "port": self.port, + "connected": self.connected, + "timestamp": datetime.now().isoformat(), + "calib_file": self.calib_path, + "register_current_pos": self.REGISTER_CURRENT_POS, + "register_target_pos": self.REGISTER_TARGET_POS + } + + +class GestureRecorder: + """手势/姿态记录器 - 支持原始编码值和回放""" + + def __init__(self, output_file="gestures.json"): + """ + 初始化记录器 + + Args: + output_file: 输出 JSON 文件路径 + """ + self.output_file = output_file + self.gestures = [] + self._load_existing() + + def _load_existing(self): + """加载现有的记录""" + try: + if os.path.exists(self.output_file): + with open(self.output_file, 'r', encoding='utf-8') as f: + self.gestures = json.load(f) + except Exception as e: + print(f"⚠️ 加载现有记录失败: {e}") + self.gestures = [] + + def record_gesture(self, angles: List[int], label: str = "", metadata: Optional[Dict] = None): + """ + 记录一个姿态 + + Args: + angles: 6 维关节位置数组 (编码值 0-4095) + label: 标签（比如 "拿起球"、"放下球"） + metadata: 可选的额外元数据 + """ + if len(angles) != 6: + print(f"❌ 角度数组长度不正确: 需要 6，收到 {len(angles)}") + return False + + record = { + "timestamp": datetime.now().isoformat(), + "angles": list(angles), # 确保是列表 + "label": label, + "metadata": metadata or {} + } + self.gestures.append(record) + self._save() + print(f"✅ 已记录姿态 (当前 {len(self.gestures)} 条): {label}") + return True + + def _save(self): + """保存到文件""" + try: + with open(self.output_file, 'w', encoding='utf-8') as f: + json.dump(self.gestures, f, indent=2, ensure_ascii=False) + except Exception as e: + print(f"❌ 保存姿态失败: {e}") + + def get_count(self) -> int: + """获取已记录的姿态数量""" + return len(self.gestures) + + def get_gesture(self, index: int) -> Optional[Dict]: + """获取指定索引的姿态""" + if 0 <= index < len(self.gestures): + return self.gestures[index] + return None + + def export_csv(self, csv_file="gestures.csv"): + """导出为 CSV 格式""" + try: + import csv + with open(csv_file, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + # 表头 + writer.writerow([ + "timestamp", "angle_1", "angle_2", "angle_3", + "angle_4", "angle_5", "angle_6", "label" + ]) + # 数据 + for g in self.gestures: + angles = g.get('angles', [0]*6) + writer.writerow([ + g['timestamp'], + *angles, + g.get('label', '') + ]) + print(f"✅ 已导出 CSV: {csv_file}") + return True + except Exception as e: + print(f"❌ 导出 CSV 失败: {e}") + return False + + +if __name__ == "__main__": + """测试和演示脚本""" + import sys + + print(""" +╔════════════════════════════════════════════════════════════╗ +║ LeRobot SO-101 硬件通信模块测试 ║ +╚════════════════════════════════════════════════════════════╝ + """) + + # 1. 初始化主臂和记录器 + arm = SO101LeaderArm(port="COM12") + recorder = GestureRecorder("test_gestures.json") + + # 2. 尝试连接 + print("\n【连接测试】") + if not arm.connect(): + print("❌ 无法连接主臂，请检查：") + print(" 1. USB 数据线是否已接入 COM12") + print(" 2. 12V 电源是否已打开") + print(" 3. LeRobot 环境是否已正确安装") + sys.exit(1) + + # 3. 读取关节位置 + print("\n【关节位置读取】") + for i in range(3): + angles = arm.read_angles() + if angles: + print(f" 第 {i+1} 次: {angles}") + recorder.record_gesture(angles, label=f"sample_{i+1}") + else: + print(f" 第 {i+1} 次: 读取失败") + time.sleep(0.5) + + # 4. 关闭连接 + print("\n【关闭连接】") + arm.disconnect() + + # 5. 导出数据 + print("\n【导出数据】") + recorder.export_csv("test_gestures.csv") + print(f"✅ 共记录 {recorder.get_count()} 个姿态") + print(f"✅ 已保存到 test_gestures.json 和 test_gestures.csv") + + diff --git a/requirements.txt b/requirements.txt index 492eabe..6170d99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,163 @@ -pip freeze > requirements.txt +accelerate==1.13.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.5 +aiosignal==1.4.0 +aliyun-python-sdk-core==2.16.0 +aliyun-python-sdk-kms==2.16.5 +annotated-doc==0.0.4 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.12.1 +async-timeout==5.0.1 +attrs==26.1.0 +audioread==3.1.0 +av==15.1.0 +certifi==2026.2.25 +cffi==2.0.0 +charset-normalizer==3.4.6 +click==8.3.1 +cloudpickle==3.1.2 +cmake==4.1.3 +colorama==0.4.6 +coloredlogs==15.0.1 +comtypes==1.4.16 +crcmod==1.7 +cryptography==46.0.5 +ctranslate2==4.7.1 +datasets==4.8.4 +decorator==5.2.1 +deepdiff==8.6.2 +diffusers==0.35.2 +dill==0.4.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.2 +exceptiongroup==1.3.1 +Farama-Notifications==0.0.4 +faster-whisper==1.2.1 +feetech-servo-sdk==1.0.0 +filelock==3.25.2 +flatbuffers==25.12.19 +frozenlist==1.8.0 +fsspec==2026.2.0 +funasr==1.3.1 +gitdb==4.0.12 +GitPython==3.1.46 +gymnasium==1.2.3 +h11==0.16.0 +hf-xet==1.4.2 +hf_transfer==0.1.9 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.3 +humanfriendly==10.0 +hydra-core==1.3.2 +idna==3.11 +ImageIO==2.37.3 +imageio-ffmpeg==0.6.0 +importlib_metadata==9.0.0 +inquirerpy==0.3.4 +jaconv==0.5.0 +jamo==0.4.1 +jieba==0.42.1 +Jinja2==3.1.6 +jmespath==0.10.0 +joblib==1.5.3 +jsonlines==4.0.0 +kaldiio==2.18.1 +lazy-loader==0.5 +lerobot==0.4.4 +librosa==0.11.0 +llvmlite==0.46.0 +markdown-it-py==4.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +mergedeep==1.3.4 +modelscope==1.35.1 +more-itertools==11.0.2 +mpmath==1.3.0 +msgpack==1.1.2 +multidict==6.7.1 +multiprocess==0.70.19 +mypy_extensions==1.1.0 +networkx==3.4.2 +numba==0.64.0 +numpy==2.2.6 +omegaconf==2.3.0 +onnxruntime==1.23.2 +openai-whisper==20250625 +opencc-python-reimplemented==0.1.7 +opencv-contrib-python==4.13.0.92 +opencv-python==4.13.0.92 +opencv-python-headless==4.12.0.88 +orderly-set==5.5.0 +oss2==2.19.1 +packaging @ file:///C:/miniconda3/conda-bld/packaging_1761049101700/work +pandas==2.3.3 +pfzy==0.3.4 +pillow==12.1.1 +platformdirs==4.9.4 +pooch==1.9.0 +prompt_toolkit==3.0.52 +propcache==0.4.1 +protobuf==6.33.6 +psutil==7.2.2 +pyarrow==23.0.1 +pycparser==3.0 +pycryptodome==3.23.0 +pydantic==2.13.0 +pydantic_core==2.46.0 +Pygments==2.19.2 +pynndescent==0.6.0 +pynput==1.8.1 +pypiwin32==223 +pyreadline3==3.5.4 +pyserial==3.5 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.2 +pytorch-wpe==0.0.1 +pyttsx3==2.99 +pytz==2026.1.post1 +pywin32==311 +PyYAML==6.0.3 +pyyaml-include==1.4.1 +regex==2026.4.4 +requests==2.32.5 +rerun-sdk==0.26.2 +rich==14.3.3 +safetensors==0.7.0 +scikit-learn==1.7.2 +scipy==1.15.3 +sentencepiece==0.2.1 +sentry-sdk==2.58.0 +shellingham==1.5.4 +six==1.17.0 +smmap==5.0.3 +sounddevice==0.5.5 +soundfile==0.13.1 +soxr==1.0.0 +sympy==1.14.0 +tensorboardX==2.6.4 +termcolor==3.3.0 +threadpoolctl==3.6.0 +tiktoken==0.12.0 +tokenizers==0.22.2 +toml==0.10.2 +torch==2.10.0 +torch-complex==0.4.4 +torchaudio==2.11.0+cpu +torchvision==0.25.0 +tqdm==4.67.3 +typer==0.24.1 +typing-inspect==0.9.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2026.1 +umap-learn==0.5.11 +urllib3==2.6.3 +wandb==0.24.2 +wcwidth==0.6.0 +websocket-client==1.9.0 +xxhash==3.6.0 +yarl==1.23.0 +zipp==3.23.1 diff --git a/voice_robot.py b/voice_robot.py index 1aeef65..856ea01 100644 --- a/voice_robot.py +++ b/voice_robot.py @@ -1,23 +1,26 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -精简版 - iFlytek IAT 实时语音识别脚本 +Voice-Robot: 语音控制机械臂系统 (LeRobot SO-101) 核心特性： - - 双声道→单声道正确转换 - - 快速响应 Ctrl+C - - 多方言关键词识别 - - 机器人指令队列 + - 配置驱动架构（commands.yaml） + - iFlytek 多方言实时识别 + 动态修正 + - 模糊匹配（difflib）+ 鲁棒性增强 + - TTS 双向反馈 (pyttsx3) + - 进程安全守护（atexit） + - WebSocket 稳定重连 + PCM 音频处理 """ -import os, sys, time, ssl, hmac, base64, json, queue, threading, signal, hashlib +import os, sys, time, ssl, hmac, base64, json, queue, threading, signal, hashlib, atexit, difflib, subprocess import numpy as np from datetime import datetime from time import mktime from urllib.parse import urlencode from wsgiref.handlers import format_date_time +from typing import Dict, List, Optional, Tuple # ========================================== -# 依赖导入与配置加载 +# 依赖导入 # ========================================== try: import sounddevice as sd @@ -26,39 +29,109 @@ print(f"❌ 缺失依赖: {e}\n请运行: pip install sounddevice websocket-client") sys.exit(1) -# 从 .env 读取凭据 -env_path = os.path.join(os.path.dirname(__file__), '.env') -if os.path.exists(env_path): - try: - # 尝试 UTF-8，失败则用 GBK +# Whisper 本地语音识别（备用方案） +try: + import whisper + WHISPER_AVAILABLE = True + print("✅ Whisper 库已加载（可用于本地语音识别）") +except ImportError: + WHISPER_AVAILABLE = False + print("⚠️ Whisper 库缺失。运行: pip install openai-whisper") + +# 导入 LeRobot 硬件模块 +try: + from lerobot_hardware import SO101LeaderArm, GestureRecorder + LEROBOT_AVAILABLE = True +except ImportError: + print("⚠️ LeRobot 硬件模块缺失，主臂控制不可用") + LEROBOT_AVAILABLE = False + +try: + import yaml +except ImportError: + print("⚠️ yaml 库缺失，使用备用 JSON 加载器") + yaml = None + +try: + import pyttsx3 + TTS_AVAILABLE = True +except ImportError: + print("⚠️ pyttsx3 库缺失，TTS 反馈不可用。运行: pip install pyttsx3") + TTS_AVAILABLE = False + +# ========================================== +# 配置加载器（多层级：yaml > .env > 默认值） +# ========================================== +class ConfigLoader: + """从 yaml/json/.env 加载配置的通用加载器""" + def __init__(self, config_file='commands.yaml', env_file='.env'): + self.config = {} + self.load_config_file(config_file) + self.load_env_file(env_file) + + def load_config_file(self, filename): + """加载 YAML 配置文件""" + fpath = os.path.join(os.path.dirname(__file__), filename) + if not os.path.exists(fpath): + print(f"⚠️ 配置文件不存在: {fpath}，使用默认值") + return + try: - with open(env_path, 'r', encoding='utf-8') as f: - content = f.read() - except UnicodeDecodeError: - with open(env_path, 'r', encoding='gbk') as f: - content = f.read() - - for line in content.split('\n'): - line = line.strip() - if not line or line.startswith('#') or '=' not in line: - continue - k, v = line.split('=', 1) - k, v = k.strip(), v.strip().strip('"').strip("'") - if k and v and k not in os.environ: - os.environ[k] = v - except Exception: - pass + if yaml: + with open(fpath, 'r', encoding='utf-8') as f: + self.config = yaml.safe_load(f) or {} + else: + # 备用：简单的 JSON 解析（如果 YAML 不可用） + with open(fpath, 'r', encoding='utf-8') as f: + content = f.read() + if content.startswith('{'): + self.config = json.loads(content) + except Exception as e: + print(f"❌ 加载配置文件失败: {e}") + + def load_env_file(self, filename): + """从 .env 覆盖配置""" + fpath = os.path.join(os.path.dirname(__file__), filename) + if not os.path.exists(fpath): + return + + try: + with open(fpath, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + k, v = line.split('=', 1) + k, v = k.strip(), v.strip().strip('"').strip("'") + os.environ[k] = v + except Exception: + pass + + def get(self, key, default=None): + """获取配置值（支持嵌套 key: "commands.PICK.keywords"）""" + keys = key.split('.') + val = self.config + for k in keys: + if isinstance(val, dict): + val = val.get(k) + else: + return default + return val if val is not None else default + -APP_ID = os.environ.get('IFLY_APP_ID', 'REMOVED') -API_KEY = os.environ.get('IFLY_API_KEY', 'REMOVED') -API_SECRET = os.environ.get('IFLY_API_SECRET', 'MWNlODE0YzRhODEyYTFjNmMwNTVkZmFh') +# 加载配置 +CONFIG = ConfigLoader() -ACCENT = os.environ.get('IFLY_ACCENT', 'mandarin') -RATE = 16000 -CHANNELS = 2 # 双声道：Realtek 驱动需要 -CHUNK_MS = 80 +APP_ID = os.environ.get('IFLY_APP_ID', CONFIG.get('defaults.app_id', 'YOUR_APP_ID_HERE')) +API_KEY = os.environ.get('IFLY_API_KEY', CONFIG.get('defaults.api_key', 'YOUR_API_KEY_HERE')) +API_SECRET = os.environ.get('IFLY_API_SECRET', CONFIG.get('defaults.api_secret', 'YOUR_API_SECRET_HERE')) + +ACCENT = os.environ.get('IFLY_ACCENT', CONFIG.get('speech.accent', 'mandarin')) +RATE = CONFIG.get('speech.sample_rate', 16000) +CHANNELS = CONFIG.get('speech.channels', 2) +CHUNK_MS = CONFIG.get('speech.chunk_ms', 80) CHUNK = int(RATE * CHUNK_MS / 1000) -CONFIDENCE_THRESHOLD = float(os.environ.get('IFLY_CONF_THRESHOLD', '0.12')) +FUZZY_THRESHOLD = CONFIG.get('speech.fuzzy_match_threshold', 0.75) # 全局控制 EXIT_EVENT = threading.Event() @@ -66,20 +139,184 @@ ROBOT_QUEUE = queue.Queue() LATEST_RMS = 0.0 RMS_LOCK = threading.Lock() -LAST_TEXT = "" # 【关键】用于跨函数记录最后一次识别到的文本（断尾兜底） +LAST_TEXT = "" -# VAD 阈值（校准时设置） +# VAD 阈值 START_THRESHOLD = 6.0 END_THRESHOLD = 3.0 CALIBRATE_SECONDS = 1.5 -# 日志与输出 -LOG_FILE = os.path.join(os.path.dirname(__file__), 'commands.log') -CMD_FILE = os.path.join(os.path.dirname(__file__), 'last_command.json') +# 日志输出 +LOG_FILE = os.path.join(os.path.dirname(__file__), CONFIG.get('logging.log_file', 'commands.log')) +CMD_FILE = os.path.join(os.path.dirname(__file__), CONFIG.get('logging.cmd_json', 'last_command.json')) + + +# ========================================== +# TTS 语音反馈引擎 +# ========================================== +class TTSEngine: + """文字转语音（中文）""" + _lock = threading.Lock() + + def __init__(self): + self.enabled = TTS_AVAILABLE + self.engine = None + self.speak_thread = None + if TTS_AVAILABLE: + try: + self.engine = pyttsx3.init() + self.engine.setProperty('rate', 150) # 语速 + self.engine.setProperty('volume', 0.8) # 音量 + # 注册退出钩子，防止 GC 时模块已被销毁导致的 NoneType 崩溃 + atexit.register(self.stop) + except Exception as e: + print(f"⚠️ TTS 初始化失败: {e}") + self.enabled = False + + def stop(self): + """显式停止引擎""" + if self.enabled and self.engine: + try: + self.engine.stop() + except: + pass + + def speak(self, text): + """播放语音反馈（异步，不阻塞主线程）""" + if not self.enabled or not text: + return + + # 【修复】使用独立线程运行 TTS，避免 run loop 冲突 + def _speak_async(): + try: + with self.__class__._lock: + if self.engine: + self.engine.say(text) + self.engine.runAndWait() + except Exception as e: + # 静默捕获 TTS 错误，防止程序崩溃 + print(f"⚠️ [后台] TTS 播放异常: {e}") + + # 启动后台线程，不阻塞主程序 + self.speak_thread = threading.Thread(target=_speak_async, daemon=True) + self.speak_thread.start() + +tts = TTSEngine() # ========================================== -# 实用工具 +# 命令识别引擎（配置驱动 + 模糊匹配） +# ========================================== +class CommandMatcher: + """配置驱动的指令匹配器，支持模糊匹配""" + def __init__(self, config: ConfigLoader): + self.config = config + self.commands = config.get('commands', {}) + self.fuzzy_threshold = config.get('speech.fuzzy_match_threshold', 0.75) + + def match(self, text: str) -> Tuple[Optional[str], float]: + """ + 匹配指令。返回 (action_name, similarity_score) + similarity_score: 0-1，表示置信度 + """ + if not text: + return None, 0.0 + + best_action = None + best_score = 0.0 + + for action_name, action_config in self.commands.items(): + if not isinstance(action_config, dict): + continue + + keywords = action_config.get('keywords', []) + + # 【精确匹配】：只要包含任何关键词就立刻返回 + for kw in keywords: + if kw in text: + return action_name, 1.0 + + # 【模糊匹配】：用 difflib 计算相似度 + for kw in keywords: + score = difflib.SequenceMatcher(None, text, kw).ratio() + if score > best_score: + best_score = score + best_action = action_name + + # 只返回超过阈值的结果 + if best_score >= self.fuzzy_threshold: + return best_action, best_score + + return None, best_score + + def get_tts_feedback(self, action_name: str) -> str: + """获取指令对应的 TTS 反馈文本""" + action = self.commands.get(action_name, {}) + return action.get('tts_feedback', '') + + +# 全局指令匹配器 +matcher = CommandMatcher(CONFIG) + + +# ========================================== +# Whisper 本地语音识别（单例） +# ========================================== +class WhisperEngine: + """Whisper 本地语音识别引擎""" + _instance = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + self.model = None + self.enabled = False + if WHISPER_AVAILABLE: + self._load_model() + + def _load_model(self): + """加载 Whisper 模型（首次加载会下载 ~140MB）""" + try: + print("\n🔄 加载 Whisper 本地语音识别模型（首次约需 30s）...") + self.model = whisper.load_model("base") + self.enabled = True + print("✅ Whisper 模型已加载") + except Exception as e: + print(f"⚠️ Whisper 模型加载失败: {e}") + self.enabled = False + + def transcribe(self, audio_file: str) -> Optional[str]: + """ + 转写音频文件（使用 Whisper） + + Args: + audio_file: WAV 文件路径 + + Returns: + 转写的中文文本，失败时返回 None + """ + if not self.enabled or not self.model: + return None + + try: + result = self.model.transcribe(audio_file, language="zh") + return result.get("text", "").strip() + except Exception as e: + print(f"⚠️ Whisper 转写失败: {e}") + return None + +whisper_engine = WhisperEngine() + + + +# ========================================== +# 日志工具 # ========================================== def log_cmd(text, cmd_label, score=None): """写入命令日志""" @@ -110,62 +347,395 @@ def save_cmd_json(action, text, score=None): pass -def parse_command(text): - """多方言关键词识别""" - if not text: - return None - - rules = [ - # 停止动作（普通话、重庆话、上海话） - ('STOP', [ - '停止', '莫动', '不要', '煞', '停下', '别动', '煞车', '拿停', '勒停', - '伐要动', '别动', '不要拿' # 上海话 - ]), - # 抓取动作（普通话、重庆话、上海话） - ('PICK', [ - '抓', '拿', '起来', '拿起', '抓起', '捉', '物事', '搿只', '把那', '拿那', - '侬拿', '帮吾拿', '拿起来', '搿个物事' # 上海话特有 - ]), - # 放下动作（普通话、重庆话、上海话） - ('PLACE', [ - '放', '摆', '下去', '放下', '放倒', '下', '摆下', '落去', '搁下', - '摆辣海', '摆下去', '伐要', '落地' # 上海话特有 - ]) - ] - - for action, keywords in rules: - for kw in keywords: - if kw in text: - return action - return None + +# ========================================== +# 进程安全管理（LeRobot 子进程保护） +# ========================================== +class ProcessManager: + """管理 LeRobot 子进程的生命周期""" + def __init__(self): + self.process = None + self.lock = threading.Lock() + + # 主臂硬件接口 + self.leader_arm = None + self.gesture_recorder = None + self.recording_gestures = False + + # 注册清理钩子，程序退出时强制杀死子进程 + atexit.register(self.cleanup) + + # 初始化主臂（如果可用） + if LEROBOT_AVAILABLE: + self._init_leader_arm() + + def start_recording(self, env_name='lerobot', robot_path='lerobot/configs/robot/so_101'): + """启动 LeRobot 数据采集""" + with self.lock: + if self.process is not None: + print("⚠️ LeRobot 进程已在运行") + return False + + script_path = CONFIG.get('lerobot.script_path', 'lerobot/scripts/control_robot.py') + cmd = f"conda run -n {env_name} python {script_path} teleoperate --robot-path {robot_path} --record" + + try: + # 日志重定向到文件，防止管道缓冲区溢出导致进程卡死 + log_file = open(CONFIG.get('logging.lerobot_log', 'lerobot_runtime.log'), 'a', encoding='utf-8') + self.process = subprocess.Popen( + cmd, + shell=True, + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == 'nt' else 0 + ) + print(f"📊 LeRobot 采集已启动 (PID: {self.process.pid})") + tts.speak("开始记录遥操作数据") + return True + except Exception as e: + print(f"❌ 启动 LeRobot 失败: {e}") + return False + + def stop_recording(self): + """停止 LeRobot 数据采集""" + with self.lock: + if self.process is None: + return True + + try: + if os.name == 'nt': # Windows + subprocess.run(f"taskkill /F /T /PID {self.process.pid}", shell=True, timeout=5) + else: # Linux/Mac + self.process.terminate() + self.process.wait(timeout=5) + print("✅ LeRobot 采集已停止") + tts.speak("停止记录，数据已保存") + self.process = None + return True + except Exception as e: + print(f"⚠️ 停止 LeRobot 时出错: {e}") + self.process = None + return False + + def _init_leader_arm(self): + """初始化主臂硬件接口""" + try: + port = CONFIG.get('hardware.leader_port', 'COM12') + self.leader_arm = SO101LeaderArm(port=port) + self.gesture_recorder = GestureRecorder("gestures.json") + + # 【关键】立即尝试连接硬件 + # 这样可以在程序启动时就发现问题，而不是等到指令时才发现 + print(f"📦 主臂模块已初始化 (端口: {port})") + print("🔗 正在建立硬件连接...") + + if not self.leader_arm.connect(): + print("⚠️ 硬件连接失败，请检查：") + print(" 1. SO-101 机械臂是否接入 COM12") + print(" 2. 12V 电源是否已打开") + print(" 3. USB 数据线连接是否正常") + print(" 进程将继续运行，但硬件指令会被忽略") + else: + print("✅ 硬件连接已建立") + + except Exception as e: + print(f"⚠️ 主臂初始化失败: {e}") + + def record_gesture_from_leader(self, label: str = ""): + """从主臂读取当前姿态并记录""" + if not LEROBOT_AVAILABLE or not self.leader_arm: + print("⚠️ 主臂硬件不可用") + return False + + try: + if not self.leader_arm.connected: + if not self.leader_arm.connect(): + return False + + angles = self.leader_arm.read_angles() + if angles is None: + print("⚠️ 无法读取关节角度") + return False + + # 记录姿态 + self.gesture_recorder.record_gesture(angles, label) + tts.speak(f"已记录姿态：{label}") + return True + except Exception as e: + print(f"❌ 记录姿态失败: {e}") + return False + + def start_gesture_recording(self): + """开始连续记录主臂姿态""" + if not LEROBOT_AVAILABLE or not self.leader_arm: + print("⚠️ 主臂硬件不可用") + return False + + self.recording_gestures = True + print("🎬 开始连续记录主臂姿态（后台线程）") + tts.speak("开始记录主臂姿态") + + def record_loop(): + interval = CONFIG.get('hardware.record_interval_ms', 100) / 1000.0 + frame_count = 0 + while self.recording_gestures and not EXIT_EVENT.is_set(): + try: + if not self.leader_arm.connected: + if not self.leader_arm.connect(): + time.sleep(1) + continue + + angles = self.leader_arm.read_angles() + if angles: + self.gesture_recorder.record_gesture(angles, f"frame_{frame_count}") + frame_count += 1 + + time.sleep(interval) + except Exception as e: + print(f"⚠️ 记录循环异常: {e}") + time.sleep(interval) + + thread = threading.Thread(target=record_loop, daemon=True) + thread.start() + return True + + def stop_gesture_recording(self): + """停止连续记录主臂姿态""" + self.recording_gestures = False + print("⏹️ 已停止记录主臂姿态") + tts.speak(f"停止记录，共记录 {self.gesture_recorder.get_count()} 个姿态") + return True + + def cleanup(self): + """清理钩子：程序退出时调用""" + self.stop_recording() + if self.gesture_recorder: + print(f"📊 共记录 {self.gesture_recorder.get_count()} 个姿态") + + # 【舵机过载保护】尝试释放所有舵机的力矩，即使某个舵机已过载也不中断清理流程 + if self.leader_arm and self.leader_arm.connected: + try: + # 遍历 6 个舵机逐个释放力矩（地址 40 写入 0） + for motor_id in range(1, 7): + try: + self.leader_arm.bus._write(40, 1, motor_id, 0) + except Exception as e: + # 某个舵机可能已过载，记录但继续释放其他舵机 + print(f"⚠️ 释放电机 {motor_id} 力矩时出错（可能已过载）: {e}") + continue + except Exception as e: + print(f"⚠️ 清理舵机力矩时出错: {e}") + + # 最后断开连接 + try: + self.leader_arm.disconnect() + except Exception as e: + print(f"⚠️ 断开连接时出错: {e}") + +pm = ProcessManager() +# ========================================== +# 机器人指令处理器 +# ========================================== def robot_worker(): - """处理机器人指令队列""" + """ + 【关键修复】处理机器人指令队列 + + 核心特性： + - 独立线程运行，不受 WebSocket 超时影响 + - 即使语音连接断开也持续工作 + - 包含异常捕获，防止线程意外退出 + """ + print("🤖 robot_worker 已启动（后台持续监听指令）") + + error_count = 0 while not EXIT_EVENT.is_set(): try: + # 非阻塞读队列，避免卡死 cmd = ROBOT_QUEUE.get(timeout=0.5) except queue.Empty: + # 定期检查 EXIT_EVENT，但不打印日志避免噪音 + continue + except Exception as e: + error_count += 1 + if error_count > 10: + print(f"⚠️ robot_worker 异常多次: {e}") + error_count = 0 continue - action = cmd.get('action') - text = cmd.get('text', '') - print(f"▶️ 执行: {action} | '{text}'") - save_cmd_json(action, text, cmd.get('score')) - time.sleep(0.2) - - -def process_final_text(text): - """【新增】统一的指令解析与入队逻辑（供 on_message 和 on_close 共同调用）""" - action = parse_command(text) - if action: - print(f"🤖 指令匹配成功: {action}") try: - ROBOT_QUEUE.put_nowait({'action': action, 'text': text}) - except Exception: - pass - else: - print(f"⚠️ 未能识别指令关键词: {text}") + action = cmd.get('action') + text = cmd.get('text', '') + score = cmd.get('score', 0.0) + + print(f"▶️ 执行: {action} | '{text}' (置信度: {score:.2f})") + save_cmd_json(action, text, score) + log_cmd(text, action, score) + + # 【关键】根据动作类型分发处理 + if action == 'RECORD_START': + pm.start_recording() + elif action == 'RECORD_STOP': + pm.stop_recording() + elif action == 'GESTURE_RECORD': + pm.record_gesture_from_leader(label=text) + elif action == 'GESTURE_START': + pm.start_gesture_recording() + elif action == 'GESTURE_STOP': + pm.stop_gesture_recording() + + # 【新增】机械臂实时控制逻辑（使用 safe_write 保护） + elif action == 'PICK': + try: + print("🦾 执行实时抓取动作 (6 号舵机)...") + # 【优化】异步TTS反馈，不阻塞动作执行 + threading.Thread(target=lambda: tts.speak("收到抓取指令"), daemon=True).start() + + if pm.leader_arm and pm.leader_arm.connected: + # 【关键】实时反馈机制：读取当前位置 + current_angles = pm.leader_arm.read_angles() + if current_angles: + print(f"🔍 调试: 6号舵机当前位置 {current_angles[5]} (范围: 1000-2800)") + + # 执行抓取：使用 safe_write 确保在安全范围内 + pm.leader_arm.safe_write(6, 2800) # 闭合位置（安全范围内） + print("✅ 抓取动作已执行") + else: + print("⚠️ 主臂未连接，无法执行抓取") + threading.Thread(target=lambda: tts.speak("主臂未连接"), daemon=True).start() + + except Exception as e: + print(f"⚠️ 抓取动作执行失败: {e}") + threading.Thread(target=lambda: tts.speak("操作执行遇到异常"), daemon=True).start() + + elif action == 'PLACE': + try: + print("🦾 执行实时放下动作 (6 号舵机)...") + threading.Thread(target=lambda: tts.speak("收到放下指令"), daemon=True).start() + + if pm.leader_arm and pm.leader_arm.connected: + current_angles = pm.leader_arm.read_angles() + if current_angles: + print(f"🔍 调试: 6号舵机当前位置 {current_angles[5]} (范围: 1000-2800)") + + # 执行放下：使用 safe_write 确保在安全范围内 + pm.leader_arm.safe_write(6, 1000) # 打开位置（安全范围内） + print("✅ 放下动作已执行") + else: + print("⚠️ 主臂未连接，无法执行放下") + threading.Thread(target=lambda: tts.speak("主臂未连接"), daemon=True).start() + + except Exception as e: + print(f"⚠️ 放下动作执行失败: {e}") + threading.Thread(target=lambda: tts.speak("操作执行遇到异常"), daemon=True).start() + + elif action == 'STOP': + try: + print("🛑 紧急停止：释放所有舵机力矩...") + threading.Thread(target=lambda: tts.speak("已紧急停止"), daemon=True).start() + + if pm.leader_arm and pm.leader_arm.connected: + # 逐个释放舵机，忽略过载错误 + for motor_id in range(1, 7): + try: + pm.leader_arm.bus._write(40, 1, motor_id, 0) + except Exception: + pass # 过载舵机可能无法释放，忽略 + print("✅ 所有舵机已释放") + else: + print("⚠️ 主臂未连接，无法释放舵机") + + except Exception as e: + print(f"⚠️ 释放舵机失败: {e}") + threading.Thread(target=lambda: tts.speak("释放操作失败"), daemon=True).start() + + elif action == 'ARM_RELAX': + # 【新增】进入调试模式：松开所有舵机力矩 + try: + print("🎯 进入调试模式，松开机械臂...") + if pm.leader_arm and pm.leader_arm.connected: + for motor_id in range(1, 7): + try: + pm.leader_arm.bus._write(40, 1, motor_id, 0) + except Exception: + pass + print("✅ 机械臂已松开，可以手动拖动进行示教") + tts.speak("机械臂已进入示教模式") + except Exception as e: + print(f"⚠️ 松开失败: {e}") + + elif action == 'ARM_LOCK': + # 【新增】退出调试模式：锁定所有舵机 + try: + print("🔒 退出调试模式，锁定机械臂...") + if pm.leader_arm and pm.leader_arm.connected: + for motor_id in range(1, 7): + try: + pm.leader_arm.bus._write(40, 1, motor_id, 1) + except Exception: + pass + print("✅ 机械臂已锁定") + tts.speak("机械臂已锁定") + except Exception as e: + print(f"⚠️ 锁定失败: {e}") + + elif action == 'RESET_GRIPPER': + # 【新增】故障恢复：重启 6 号舵机 + try: + print("🔄 正在重启 6 号舵机（夹持器）...") + threading.Thread(target=lambda: tts.speak("正在重启夹持器"), daemon=True).start() + + if pm.leader_arm and pm.leader_arm.connected: + # 1. 先释放力矩 + try: + pm.leader_arm.bus._write(40, 1, 6, 0) + print(" 1️⃣ 已释放舵机力矩") + except Exception as e: + print(f" ⚠️ 释放力矩失败: {e}") + + # 2. 等待 1 秒 + time.sleep(1) + + # 3. 重新启用力矩 + try: + pm.leader_arm.bus._write(40, 1, 6, 1) + print(" 2️⃣ 已重新启用力矩") + except Exception as e: + print(f" ⚠️ 启用力矩失败: {e}") + + # 4. 读取当前位置确认 + angles = pm.leader_arm.read_angles() + if angles and angles[5]: + print(f"✅ 舵机已恢复，当前位置: {angles[5]}") + threading.Thread(target=lambda: tts.speak("夹持器已恢复"), daemon=True).start() + else: + print("❌ 舵机仍未响应") + threading.Thread(target=lambda: tts.speak("舵机恢复失败，请检查硬件"), daemon=True).start() + else: + print("⚠️ 主臂未连接，无法重启舵机") + # 尝试重新连接 + if pm.leader_arm: + print("🔗 尝试重新连接硬件...") + if pm.leader_arm.connect(): + print("✅ 硬件重新连接成功") + threading.Thread(target=lambda: tts.speak("硬件已重新连接"), daemon=True).start() + + except Exception as e: + print(f"❌ 舵机重启失败: {e}") + threading.Thread(target=lambda: tts.speak("重启失败"), daemon=True).start() + + + # 短暂延迟，避免快速连续指令冲突 + time.sleep(0.2) + + except Exception as e: + print(f"❌ robot_worker 处理指令失败: {e}") + # 不 sys.exit()，继续等待下一个指令 + continue + + print("🤖 robot_worker 已退出") # ========================================== @@ -180,7 +750,6 @@ def calibrate_mic(device_id=None): print(f"\n🔧 校准麦克风 ({CALIBRATE_SECONDS}s)，请保持安静...") frames = [] - def _cb(indata, frames_count, time_info, status): frames.append(indata.copy()) @@ -191,19 +760,17 @@ def _cb(indata, frames_count, time_info, status): print("⚠️ 无法采集校准数据") return 0.0, 6.0, 3.0 - # 合并帧，转单声道（取第一个声道，避免相位抵消） + # 合并帧，只取左声道，应用增益 arr = np.concatenate(frames, axis=0) if arr.ndim > 1 and arr.shape[1] > 1: - arr = arr[:, 0] # 只取左声道 - # 应用同样的增益（1.5倍），使校准更准确 + arr = arr[:, 0] arr = np.clip(arr * 1.5, -1.0, 1.0) else: arr = arr.reshape(-1) rms = float(np.sqrt(np.mean(np.square(arr)))) * 1000 - # 由于增益后的数值更大，调整阈值倍数 - start_th = max(rms * 1.5, 5.0) # 更宽松，更容易触发 - end_th = max(rms * 0.8, 2.0) # 更灵敏，更快结束句子 + start_th = max(rms * 1.5, 5.0) + end_th = max(rms * 0.8, 2.0) print(f"✅ 校准完成: 环境={rms:.1f}, start_th={start_th:.1f}, end_th={end_th:.1f}") return rms, start_th, end_th @@ -221,23 +788,11 @@ def audio_callback(indata, frames, time_info, status): return try: - # 【修复1】核心改动：绝对不能取平均值！强制只取左声道 - # 这彻底解决相位抵消（+1 + (-1) = 0）的问题 - if indata.shape[1] > 1: - mono = indata[:, 0:1].copy() # 只取左声道 - else: - mono = indata.copy() - - # 计算原始 RMS（用于VAD判定） - rms_raw = float(np.sqrt(np.mean(np.square(mono)))) - rms_display = rms_raw * 1000 + # 【修复1】只取左声道，避免相位抵消 + mono = indata[:, 0:1].copy() if indata.shape[1] > 1 else indata.copy() - # 【修复2】暴力增加数字增益：振幅放大 1.5 倍 - # 这解决"音量大但讯飞收不到"的问题 - # 目标：使得大声说话时RMS能到 20-50（而不是目前的 0.2） + # 【修复2】应用增益：1.5 倍放大 mono_amplified = np.clip(mono * 1.5, -1.0, 1.0) - - # 转为 int16 bytes（音量已被放大） audio_bytes = (mono_amplified.reshape(-1) * 32768).astype(np.int16).tobytes() # 非阻塞写队列 @@ -246,14 +801,16 @@ def audio_callback(indata, frames, time_info, status): except queue.Full: pass - # 实时音量显示 + # 【优化】只在必要时更新显示 with RMS_LOCK: global LATEST_RMS - LATEST_RMS = 0.7 * LATEST_RMS + 0.3 * rms_display - vol_display = LATEST_RMS + rms_raw = float(np.sqrt(np.mean(np.square(mono)))) * 1000 + LATEST_RMS = 0.7 * LATEST_RMS + 0.3 * rms_raw - bar = '█' * int(min(vol_display / 2, 20)) + '-' * (20 - int(min(vol_display / 2, 20))) - print(f"\r🎙️ [{bar}] {vol_display:5.1f}", end='', flush=True) + # 实时音量显示（简化计算） + vol_bar_len = min(int(LATEST_RMS / 2), 20) + bar = '█' * vol_bar_len + '-' * (20 - vol_bar_len) + print(f"\r🎙️ [{bar}] {LATEST_RMS:5.1f}", end='', flush=True) except Exception: pass @@ -282,7 +839,7 @@ def create_url(self): def on_message(ws, msg): - """处理识别结果（支持动态修正与 LS 兜底触发）""" + """处理识别结果（支持动态修正与断尾兜底）""" global LAST_TEXT try: data = json.loads(msg) @@ -292,45 +849,38 @@ def on_message(ws, msg): if data.get('code') != 0: return - # 1. 提取识别文本 + # 提取识别文本 result = data.get('data', {}).get('result', {}) - ws_data = result.get('ws', []) - text = "" - for seg in ws_data: + for seg in result.get('ws', []): cw_list = seg.get('cw', []) if cw_list: - text += cw_list[0]['w'] # 只拿最高置信度的候选 + text += cw_list[0]['w'] + text = text.strip() - if not text or text in ['。', '？', '！']: return - # 【关键】只要有字，就实时更新全局缓存 - if text: - LAST_TEXT = text + # 更新全局缓存 + LAST_TEXT = text - # 2. 获取关键状态标志 - pgs = result.get('pgs') # pgs="rpl" 表示动态修正 + # 获取状态标志 status = data.get('data', {}).get('status', 0) - is_ls = result.get('ls', False) # 句子是否结束（即便 status 不是 2） + is_ls = result.get('ls', False) - # 3. 屏幕反馈 - if pgs == 'rpl': + # 屏幕反馈 + if result.get('pgs') == 'rpl': print(f"✨ 修正: {text}", end='\r') else: print(f"🎙️ 识别中: {text}", end='\r') - # 4. 【核心修复】双重判定触发逻辑 - # 只要满足 (status=2) 或者 (ls=True)，就代表话说明白了，立刻去解析指令 - is_final_trigger = (status == 2) or is_ls - - if is_final_trigger and text: - print(f"\n🏁 最终识别: {text}") # 换行打印最终定稿，方便调试 + # 双重判定：触发指令解析 + if (status == 2 or is_ls) and text: + print(f"\n🏁 最终识别: {text}") process_final_text(text) - LAST_TEXT = "" # 正常触发后清空，防止 on_close 重复触发 + LAST_TEXT = "" - # 5. 安全关闭连接 + # 安全关闭连接 if is_ls: try: ws.close() @@ -339,26 +889,59 @@ def on_message(ws, msg): def on_error(ws, error): - print(f"\n❗ WS error: {error}") + # 仅当 error 是真正的异常对象时打印，过滤掉整数状态码和字符串消息 + if isinstance(error, Exception): + print(f"\n❗ WS error: {error}") def on_close(ws, close_status_code, close_msg): - """【关键修复】连接关闭时的断尾兜底逻辑""" + """断尾兜底逻辑：连接关闭时处理剩余文本""" global LAST_TEXT - # print(f"\n🔌 连接已关闭 (Code: {close_status_code})") - # 【兜底逻辑】如果连接断开时，LAST_TEXT 还有内容没被处理 + # 如果连接断开时还有未处理的文本，立刻处理 if LAST_TEXT and len(LAST_TEXT.strip()) > 0: print(f"\n🏁 捕获到断开前的最后文本: {LAST_TEXT}") process_final_text(LAST_TEXT) - LAST_TEXT = "" # 处理完立即清空 + LAST_TEXT = "" -def on_open(ws): - """发送音频的线程""" +# ========================================== +# 统一的指令解析与入队逻辑 +# ========================================== +def process_final_text(text): + """统一的指令解析与入队逻辑""" + action, score = matcher.match(text) + if action: + print(f"🎯 指令匹配: {action} (置信度: {score:.2f})") + tts_feedback = matcher.get_tts_feedback(action) + if tts_feedback: + tts.speak(tts_feedback) + try: + ROBOT_QUEUE.put_nowait({'action': action, 'text': text, 'score': score}) + except Exception: + pass + else: + print(f"⚠️ 未能识别指令: {text}") + +def on_open(ws): + """启动音频发送线程""" def send_loop(): status = 0 + payload_base = { + 'common': {'app_id': APP_ID}, + 'business': { + 'language': 'zh_cn', + 'domain': 'iat', + 'accent': ACCENT, + 'vinfo': 1, + 'vad_eos': CONFIG.get('speech.vad_eos', 1500), + 'dwa': 'wpgs', + 'nbest': 1, + 'rlang': 'zh-cn' + } + } + while not EXIT_EVENT.is_set() and ws.sock and ws.sock.connected: try: audio = AUDIO_QUEUE.get(timeout=0.2) @@ -366,25 +949,23 @@ def send_loop(): audio = b'\x00' * (CHUNK * 2) if status == 0: - payload = { - 'common': {'app_id': APP_ID}, - 'business': { - 'language': 'zh_cn', - 'domain': 'iat', - 'accent': 'shanghainese', - 'vinfo': 1, - 'vad_eos': 1500, # 从 3000 缩短到 1500，抢在超时前切断 - 'dwa': 'wpgs', - 'nbest': 1, # 只要第 1 候选，减少冗余数据 - 'rlang': 'zh-cn' - }, - 'data': {'status': 0, 'format': 'audio/L16;rate=16000', 'encoding': 'raw', - 'audio': str(base64.b64encode(audio), 'utf-8')} + payload = dict(payload_base) + payload['data'] = { + 'status': 0, + 'format': 'audio/L16;rate=16000', + 'encoding': 'raw', + 'audio': str(base64.b64encode(audio), 'utf-8') } status = 1 else: - payload = {'data': {'status': 1, 'format': 'audio/L16;rate=16000', 'encoding': 'raw', - 'audio': str(base64.b64encode(audio), 'utf-8')}} + payload = { + 'data': { + 'status': 1, + 'format': 'audio/L16;rate=16000', + 'encoding': 'raw', + 'audio': str(base64.b64encode(audio), 'utf-8') + } + } try: ws.send(json.dumps(payload)) @@ -418,12 +999,10 @@ def test_devices(dev_ids, duration=1.5): for dev_id in dev_ids: try: info = sd.query_devices(dev_id) - # 【修复】强制使用16000Hz（讯飞硬性要求） sr = RATE - print(f"\n测试设备 {dev_id}: {info.get('name')} (使用16000Hz)") + print(f"\n测试设备 {dev_id}: {info.get('name')} (16000Hz)") frames = [] - def cb(indata, frames_count, time_info, status): frames.append(indata.copy()) @@ -436,11 +1015,10 @@ def cb(indata, frames_count, time_info, status): print("❌ 无数据") continue - # 合并帧，只取左声道，并应用增益 + # 合并帧，只取左声道，应用增益 arr = np.concatenate(frames, axis=0) if arr.ndim > 1 and arr.shape[1] > 1: - arr = arr[:, 0] # 只取左声道 - # 应用同样的增益 + arr = arr[:, 0] arr = np.clip(arr * 1.5, -1.0, 1.0) else: arr = arr.reshape(-1) @@ -448,7 +1026,7 @@ def cb(indata, frames_count, time_info, status): rms = np.sqrt(np.mean(np.square(arr))) * 1000 print(f"RMS: {rms:.1f}") - # 保存 WAV（用16000Hz） + # 保存 WAV int16 = (arr * 32767).astype(np.int16) fname = os.path.join(os.path.dirname(__file__), f'test_dev{dev_id}.wav') try: @@ -468,7 +1046,14 @@ def cb(indata, frames_count, time_info, status): def run(device_id): - """主循环""" + """ + 主循环 + + 设计原则： + - WebSocket 重连独立于 robot_worker 线程 + - 语音超时不影响指令执行 + - 异常捕获确保持续监听 + """ print("\n启动中...") sd.default.device = device_id @@ -476,10 +1061,7 @@ def run(device_id): global START_THRESHOLD, END_THRESHOLD ambient, START_THRESHOLD, END_THRESHOLD = calibrate_mic(device_id) - # 【修复】强制采样率为16000Hz（讯飞硬性要求） - # 不要用设备的默认采样率（可能是44100），那会导致识别乱码 sr = RATE # 强制 16000Hz - stream = sd.InputStream(samplerate=sr, channels=CHANNELS, dtype='float32', callback=audio_callback, blocksize=int(sr * CHUNK_MS / 1000), device=device_id) @@ -487,11 +1069,29 @@ def run(device_id): print("\n🔊 麦克风已开启，开始监听（按 Ctrl+C 退出）") + # 【关键修复】在启动语音线程之前，强制进行硬件连接握手 + # 这是避免"虚假初始化"导致所有硬件指令被跳过的必要步骤 + print("\n🔗 正在建立硬件连接...") + if pm.leader_arm and not pm.leader_arm.connected: + if pm.leader_arm.connect(): + print(f"✅ 硬件连接已就绪 (Port: {pm.leader_arm.port})") + else: + print(f"⚠️ 硬件连接失败，请检查：") + print(f" 1. SO-101 机械臂是否接入 {pm.leader_arm.port}") + print(f" 2. 12V 电源是否已打开") + print(f" 3. USB 数据线连接是否正常") + print(f" 进程将继续运行，但硬件指令会被忽略") + elif pm.leader_arm and pm.leader_arm.connected: + print(f"✅ 硬件已连接") + else: + print(f"⚠️ 主臂硬件模块不可用") + # 启动机器人指令处理线程 worker = threading.Thread(target=robot_worker, daemon=True) worker.start() # WebSocket 重连循环 + consecutive_errors = 0 try: while not EXIT_EVENT.is_set(): try: @@ -503,8 +1103,6 @@ def run(device_id): on_close=on_close ) ws.on_open = on_open - - # 【关键修复】减小 ping_interval 加快响应 ws.run_forever(sslopt={'cert_reqs': ssl.CERT_NONE}, ping_interval=5, ping_timeout=3) # 清空过期队列 @@ -514,17 +1112,30 @@ def run(device_id): except Exception: break + consecutive_errors = 0 time.sleep(0.05) + + except ConnectionError as e: + consecutive_errors += 1 + if consecutive_errors % 5 == 0: + print(f"⚠️ 连接错误 (第 {consecutive_errors} 次): {type(e).__name__}") + time.sleep(0.5) except Exception as e: - print(f"⚠️ 异常: {e}") + consecutive_errors += 1 + if consecutive_errors % 10 == 0: + print(f"⚠️ 异常 (第 {consecutive_errors} 次): {e}") time.sleep(1) + except KeyboardInterrupt: print("\n⏹️ 停止...") + except Exception as e: + print(f"\n❌ 主循环异常: {e}") finally: EXIT_EVENT.set() stream.stop() stream.close() worker.join(timeout=2) + pm.cleanup() print("✅ 已停止") @@ -549,9 +1160,10 @@ def signal_handler(sig, frame): devs = [i for i, _ in mic_list][:5] test_devices(devs) ans = input("\n输入要使用的设备 ID: ").strip() - device_id = int(ans) if ans else best_id + device_id = int(ans) if (ans and ans.isdigit()) else best_id else: - device_id = int(ans) if ans else best_id + # 增加 isdigit() 判断，防止误输入导致崩溃 + device_id = int(ans) if (ans and ans.isdigit()) else best_id run(device_id)