From 68a7ea4e7d0a9b9234f0db822eb05cfee727e572 Mon Sep 17 00:00:00 2001 From: lynnyulinlin-debug Date: Mon, 23 Mar 2026 20:32:42 +0800 Subject: [PATCH 01/10] feat: add ball motion types and configuration --- src/types.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/types.ts b/src/types.ts index d2919dcf..d51bf672 100644 --- a/src/types.ts +++ b/src/types.ts @@ -13,6 +13,14 @@ export type RobotConfig = string[]; export type SceneType = 'basic' | 'living_room' | 'classroom' | 'tennis_court'; export type SceneSize = 'small' | 'medium' | 'large'; export type SceneComplexity = 'low' | 'medium' | 'high'; +export type BallMotionMode = 'fixed' | 'random' | 'mixed'; + +export interface BallMotionConfig { + mode: BallMotionMode; + radius: number; + speed: number; + randomIntensity: number; +} export interface LogEntry { message: string; From 57ff07051a4d106b9eca5ad1cffbf6170e5d0535 Mon Sep 17 00:00:00 2001 From: lynnyulinlin-debug Date: Mon, 23 Mar 2026 20:33:35 +0800 Subject: [PATCH 02/10] feat: improve neural network architecture for ball tracking --- src/services/actService.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/services/actService.ts b/src/services/actService.ts index a2e1c47d..570e7f8e 100644 --- a/src/services/actService.ts +++ b/src/services/actService.ts @@ -34,24 +34,25 @@ export const actService = { createModel() { const imageInput = tf.input({shape: [64, 64, 3]}); - const stateInput = tf.input({shape: [14]}); + const stateInput = tf.input({shape: [12]}); const h1 = tf.layers.conv2d({filters: 16, kernelSize: 3, activation: 'relu'}).apply(imageInput) as tf.SymbolicTensor; const h2 = tf.layers.maxPooling2d({poolSize: 2}).apply(h1) as tf.SymbolicTensor; const h3 = tf.layers.conv2d({filters: 32, kernelSize: 3, activation: 'relu'}).apply(h2) as tf.SymbolicTensor; const h4 = tf.layers.maxPooling2d({poolSize: 2}).apply(h3) as tf.SymbolicTensor; const h5 = tf.layers.flatten().apply(h4) as tf.SymbolicTensor; - - const s1 = tf.layers.dense({units: 32, activation: 'relu'}).apply(stateInput) as tf.SymbolicTensor; - + + const s1 = tf.layers.dense({units: 64, activation: 'relu'}).apply(stateInput) as tf.SymbolicTensor; + const concatenated = tf.layers.concatenate().apply([h5, s1]) as tf.SymbolicTensor; - + const d1 = tf.layers.dense({units: 128, activation: 'relu'}).apply(concatenated) as tf.SymbolicTensor; - const output = tf.layers.dense({units: CHUNK_SIZE * 2, activation: 'linear'}).apply(d1) as tf.SymbolicTensor; - + const d2 = tf.layers.dense({units: 64, activation: 'relu'}).apply(d1) as tf.SymbolicTensor; + const output = tf.layers.dense({units: CHUNK_SIZE * 2, activation: 'linear'}).apply(d2) as tf.SymbolicTensor; + const model = tf.model({inputs: [imageInput, stateInput], outputs: output}); model.compile({ optimizer: 'adam', loss: 'meanSquaredError' }); - + return model; }, From 59dab03b8fa2f57c3c01ff47e89456fc96a7f4bf Mon Sep 17 00:00:00 2001 From: lynnyulinlin-debug Date: Mon, 23 Mar 2026 20:35:03 +0800 Subject: [PATCH 03/10] feat: implement ball motion control system --- src/App.tsx | 162 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 139 insertions(+), 23 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index fe93810c..24494456 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -36,6 +36,10 @@ export default function App() { }); const [speed, setSpeed] = useState(() => Number(localStorage.getItem('speed')) || 0.1); const [turnSpeed, setTurnSpeed] = useState(() => Number(localStorage.getItem('turnSpeed')) || 0.05); + const [ballMotionMode, setBallMotionMode] = useState<'fixed' | 'random' | 'mixed'>(() => (localStorage.getItem('ballMotionMode') as any) || 'fixed'); + const [ballRadius, setBallRadius] = useState(() => Number(localStorage.getItem('ballRadius')) || 5); + const [ballSpeed, setBallSpeed] = useState(() => Number(localStorage.getItem('ballSpeed')) || 1); + const [ballRandomIntensity, setBallRandomIntensity] = useState(() => Number(localStorage.getItem('ballRandomIntensity')) || 0.5); const [logs, setLogs] = useState<{ message: string, type: string, time: string }[]>([ { message: 'System initialized. Waiting for commands...', type: 'info', time: new Date().toLocaleTimeString() } ]); @@ -51,7 +55,11 @@ export default function App() { localStorage.setItem('lightPos', JSON.stringify(lightPos)); localStorage.setItem('speed', speed.toString()); localStorage.setItem('turnSpeed', turnSpeed.toString()); - }, [sceneType, sceneSize, sceneComplexity, hasArm, lightPos, speed, turnSpeed]); + localStorage.setItem('ballMotionMode', ballMotionMode); + localStorage.setItem('ballRadius', ballRadius.toString()); + localStorage.setItem('ballSpeed', ballSpeed.toString()); + localStorage.setItem('ballRandomIntensity', ballRandomIntensity.toString()); + }, [sceneType, sceneSize, sceneComplexity, hasArm, lightPos, speed, turnSpeed, ballMotionMode, ballRadius, ballSpeed, ballRandomIntensity]); // Cloud Training State const [trainingMode, setTrainingMode] = useState<'frontend' | 'cloud'>('frontend'); @@ -103,7 +111,12 @@ export default function App() { stuckCounter: 0, actionBuffer: [] as any[], lastInferenceLogTime: 0, - lastManualLogTime: 0 + lastManualLogTime: 0, + ballMotionTime: 0, + ballPrevX: 0, + ballPrevZ: 0, + ballVelX: 0, + ballVelZ: 0 }); const isAtBottom = useRef(true); @@ -529,28 +542,36 @@ export default function App() { // state: [x, y, angle, speed, ballDist, isColliding, ...zeros] (14 dims) const x = sim.current.robotState.x; - const z = sim.current.robotState.z; // treating z as y in 2D + const z = sim.current.robotState.z; const angle = sim.current.robotState.rotation; const speed = sim.current.robotState.velocity; - const targetDist = Math.hypot(sim.current.target.position.x - x, sim.current.target.position.z - z); + const targetX = sim.current.target.position.x; + const targetZ = sim.current.target.position.z; + const targetDist = Math.hypot(targetX - x, targetZ - z); - const state = new Array(14).fill(0); + // Calculate relative angle to target + const angleToTarget = Math.atan2(targetX - x, targetZ - z); + let relativeAngle = angleToTarget - angle; + // Normalize angle to [-PI, PI] + while (relativeAngle > Math.PI) relativeAngle -= Math.PI * 2; + while (relativeAngle < -Math.PI) relativeAngle += Math.PI * 2; + + const state = new Array(12).fill(0); state[0] = x; state[1] = z; - state[2] = angle; - state[3] = speed; - state[4] = targetDist; - state[5] = sim.current.isColliding ? 1.0 : 0.0; // Explicitly tell model we are colliding - - const envState = new Array(7).fill(0); - envState[0] = x; - envState[1] = z; - envState[2] = angle; - envState[3] = speed; - envState[4] = sim.current.isColliding ? 1 : 0; - envState[5] = 0; // Forward distance placeholder - envState[6] = targetDist; + state[2] = Math.sin(angle); + state[3] = Math.cos(angle); + state[4] = speed; + state[5] = targetX; + state[6] = targetZ; + state[7] = sim.current.ballVelX || 0; + state[8] = sim.current.ballVelZ || 0; + state[9] = targetDist; + state[10] = relativeAngle; + state[11] = sim.current.isColliding ? 1.0 : 0.0; + + const envState = [...state]; // Use same state for envState placeholder // Action: [up, down, left, right, stop] (one-hot or similar) // Reference uses commandToActionVec which returns 5-dim vector. @@ -725,6 +746,56 @@ export default function App() { if (velocityRef.current) velocityRef.current.textContent = (Math.abs(state.velocity) * 10).toFixed(1); }, []); + const updateBallMotion = useCallback(() => { + if (!sim.current.target) return; + + const dt = 0.016; // ~60fps + sim.current.ballMotionTime += dt; + + const prevX = sim.current.target.position.x; + const prevZ = sim.current.target.position.z; + + let newX = prevX; + let newZ = prevZ; + + if (ballMotionMode === 'fixed') { + // 固定航迹：圆形运动 + const angle = sim.current.ballMotionTime * ballSpeed; + newX = ballRadius * Math.cos(angle); + newZ = ballRadius * Math.sin(angle); + } else if (ballMotionMode === 'random') { + // 随机运动：随机游走 + const randomX = (Math.random() - 0.5) * ballRandomIntensity * ballSpeed; + const randomZ = (Math.random() - 0.5) * ballRandomIntensity * ballSpeed; + newX = prevX + randomX; + newZ = prevZ + randomZ; + + // 限制在场景范围内 + const maxRange = 8; + newX = Math.max(-maxRange, Math.min(maxRange, newX)); + newZ = Math.max(-maxRange, Math.min(maxRange, newZ)); + } else if (ballMotionMode === 'mixed') { + // 混合运动：固定航迹 + 随机扰动 + const angle = sim.current.ballMotionTime * ballSpeed; + const baseX = ballRadius * Math.cos(angle); + const baseZ = ballRadius * Math.sin(angle); + + const randomX = (Math.random() - 0.5) * ballRandomIntensity * 0.5; + const randomZ = (Math.random() - 0.5) * ballRandomIntensity * 0.5; + + newX = baseX + randomX; + newZ = baseZ + randomZ; + } + + // 计算小球速度 + sim.current.ballVelX = (newX - prevX) / dt; + sim.current.ballVelZ = (newZ - prevZ) / dt; + + // 更新小球位置 + sim.current.target.position.x = newX; + sim.current.target.position.z = newZ; + }, [ballMotionMode, ballRadius, ballSpeed, ballRandomIntensity]); + const updateOnboardCamera = useCallback(() => { const { onboardCamera, onboardRenderTarget, renderer, scene, robotState } = sim.current; const canvas = cameraCanvasRef.current; @@ -867,13 +938,14 @@ export default function App() { updateRobotMovement(); updatePhysics(); + updateBallMotion(); updateOnboardCamera(); updateArm(); if (sim.current.renderer && sim.current.scene && sim.current.camera) { sim.current.renderer.render(sim.current.scene, sim.current.camera); } - }, [updateRobotMovement, updatePhysics, updateOnboardCamera, updateArm]); + }, [updateRobotMovement, updatePhysics, updateBallMotion, updateOnboardCamera, updateArm]); useEffect(() => { sim.current.animationFrameId = requestAnimationFrame(animate); @@ -1616,12 +1688,27 @@ export default function App() { const dz = target.position.z - robotState.z; const targetDist = Math.sqrt(dx * dx + dz * dz); - const state = new Array(14).fill(0); + // Calculate relative angle to target + const angleToTarget = Math.atan2(dx, dz); + let relativeAngle = angleToTarget - robotState.rotation; + // Normalize angle to [-PI, PI] + while (relativeAngle > Math.PI) relativeAngle -= Math.PI * 2; + while (relativeAngle < -Math.PI) relativeAngle += Math.PI * 2; + + // Build 12-dimensional state (matching recordFrame structure) + const state = new Array(12).fill(0); state[0] = robotState.x; state[1] = robotState.z; - state[2] = robotState.rotation; - state[3] = robotState.velocity; - state[4] = targetDist; + state[2] = Math.sin(robotState.rotation); + state[3] = Math.cos(robotState.rotation); + state[4] = robotState.velocity; + state[5] = target.position.x; + state[6] = target.position.z; + state[7] = sim.current.ballVelX || 0; + state[8] = sim.current.ballVelZ || 0; + state[9] = targetDist; + state[10] = relativeAngle; + state[11] = sim.current.isColliding ? 1.0 : 0.0; const prediction = actService.predict(model, image, state); @@ -2001,6 +2088,35 @@ export default function App() { {turnSpeed.toFixed(2)} + +
+
小球运动控制
+ + + + {(ballMotionMode === 'random' || ballMotionMode === 'mixed') && ( + + )} +
{hasArm ? (
-
-
小球运动控制
- - - - {(ballMotionMode === 'random' || ballMotionMode === 'mixed') && ( - - )} -
{hasArm ? (
+

使用键盘 WASD 或方向键控制

+ +
+
🎾 小球运动控制
+ + + + {(ballMotionMode === 'random' || ballMotionMode === 'mixed') && ( + + )} + {ballMotionMode === 'manual' && ( +
+ 💡 手动模式：用鼠标拖动小球，可以自由创建训练数据 +
+ )} +

使用键盘 WASD 或方向键控制

diff --git a/src/components/SidebarLeft.tsx b/src/components/SidebarLeft.tsx index 05fb7b50..e1fe774a 100644 --- a/src/components/SidebarLeft.tsx +++ b/src/components/SidebarLeft.tsx @@ -59,6 +59,16 @@ interface SidebarLeftProps { isInferencing: boolean; showAttention: boolean; setShowAttention: (show: boolean) => void; + + // Ball Motion Props + ballMotionMode: 'fixed' | 'random' | 'mixed'; + setBallMotionMode: (mode: 'fixed' | 'random' | 'mixed') => void; + ballRadius: number; + setBallRadius: (radius: number) => void; + ballSpeed: number; + setBallSpeed: (speed: number) => void; + ballRandomIntensity: number; + setBallRandomIntensity: (intensity: number) => void; } export const SidebarLeft: React.FC = ({ @@ -70,7 +80,8 @@ export const SidebarLeft: React.FC = ({ speed, setSpeed, turnSpeed, setTurnSpeed, simRef, sendCommand, isRecording, toggleRecording, episodesCount, frameCount, actionCount, saveDataset, startTraining, isTraining, trainingProgress, trainingStatus, - trainedModel, startInference, isInferencing, showAttention, setShowAttention + trainedModel, startInference, isInferencing, showAttention, setShowAttention, + ballMotionMode, setBallMotionMode, ballRadius, setBallRadius, ballSpeed, setBallSpeed, ballRandomIntensity, setBallRandomIntensity }) => { return (