本仓库的预训练模型使用 Python 3.10、Stable-Baselines3 2.6.0 和
MuJoCo 3.3.2 生成。使用项目内的 uv.lock 可以复现对应版本:
uv python install 3.10
uv sync
.\.venv\Scripts\Activate.ps1
python -c "import mujoco, gymnasium, stable_baselines3, torch; print('OK')"uv sync 会自动创建 .venv。Windows 下会跳过只提供 Linux/macOS
二进制包的 Pinocchio、EigenPy 和 PyKDL 扩展;本文涉及的三个 RL
脚本不依赖这些可选扩展。
# 基础库 & GUI 支持
sudo apt update && sudo apt install -y \
libgl1-mesa-glx libgl1-mesa-dri mesa-utils \
libxcb-xinerama0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 \
libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-sync1 \
libxcb-xfixes0 libxkbcommon-x11-0 libqt5gui5 libqt5widgets5
# 编译 PyKDL / Pinocchio 所需工具
sudo apt install -y make cmake g++ unzip python3-dev libboost-all-devuv python install 3.10
uv sync # 创建 .venv 并安装 pyproject.toml + uv.lock
source .venv/bin/activatebash scripts/install_pinocchio.sh # 源码编译带 CasADi 的 Pinocchio安装后直接使用即可。
# uv sync 会安装 kdl_parser Python 包
uv sync
# 运行脚本:编译 PyKDL C++ 库 + 自动修复兼容性问题
bash scripts/install_pykdl.sh # 第一次需要编译,之后会自动检查跳过之后 import kdl_parser / from kdl_parser.urdf import ... 即可使用。
下面三个命令默认加载仓库内的预训练模型,并打开 MuJoCo 窗口:
python rl_panda_reach_target_high_profile.py
python rl_panda_obstacle_high_profile.py
python rl_panda_pickup_cube.py在无桌面或自动化环境中,可以各跑一个无界面回合:
python -c "from rl_panda_reach_target_high_profile import test_ppo; test_ppo('assets/model/rl_reach_target_checkpoint/panda_ppo_reach_target', 1, render=False)"
python -c "from rl_panda_obstacle_high_profile import test_ppo; test_ppo('assets/model/rl_obstacle_avoidance_checkpoint/panda_obstacle_avoidance_v3', 1, render=False)"
python -c "from rl_panda_pickup_cube import test_ppo; test_ppo('assets/model/rl_pickup_checkpoint/panda_pickup_v1_5', 1, render=False)"需要训练时,将对应脚本底部的 TRAIN_MODE 改为 True。默认训练步数很大,
首次验证建议先把 total_timesteps 和 n_envs 调小;完整学习顺序与参数说明见
RL_LEARNING_GUIDE.md。
Set-Location D:\project\mujoco-learning
C:\Users\playee.venvs\mujoco-learning-gpu\Scripts\python.exe .\rl_so100_pickup_cube_resume.py --mode train --algorithm ppo --n-envs 16 --timesteps 3000000 --model-path assets/model/rl_pickup_checkpoint/so100_resume_ppo_small
C:\Users\playee.venvs\mujoco-learning-gpu\Scripts\python.exe .\rl_so100_pickup_cube_resume.py --mode train --algorithm ppo --n-envs 16 --timesteps 3000000 --model-path assets/model/rl_pickup_checkpoint/so100_resume_ppo_small
Set-Location D:\project\mujoco-learning
.\scripts\start_pickup_tensorboard.ps1 -LogDir tensorboard\so100_pickup_resume -Port 6006
.\scripts\run_so100_resume_training_batch.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_v3pose_place;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4', 'name=ppo_seed0_lr5e4_small_v3pose_place;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=5e-4' )
.\scripts\run_so100_resume_training_batch.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_v4pose_no_stage_but_event_and_bestprogress;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4', 'name=ppo_seed0_lr5e4_small_v4pose_no_stage_but_event_and_bestprogress;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=5e-4' )
.\scripts\run_so100_resume_training_batch.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_v4pose_no_stage_but_event_and_bestprogress_lowgrasp_lateral_lowgrasp_lateral_threshold;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' )
.\scripts\run_so100_resume_training_batch.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_v5pose_no_stage_but_event_and_bestprogress_even_reach_reward;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' )
.\scripts\run_so100_resume_training_batch.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_raw;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' )
.\scripts\run_so100_resume_training_batch.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_raw;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' ) Set-Location D:\project\mujoco-learning
.\scripts\test_so100_resume_models.ps1 -Run @( 'name=ppo_seed0_lr3e4_small;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=3e-4', 'name=ppo_seed0_lr4e4_small;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' ) -Episodes 100 -OutputDir experiments\so100_resume_lr_sweep_analysis
.\scripts\test_so100_resume_models.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_v4pose_no_stage_but_event_and_bestprogress_lowgrasp_lateral;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' ) -Episodes 100 -OutputDir experiments\so100_resume_lr_sweep_analysis
.\scripts\visualize_so100_resume_rollouts.ps1 -Run @( 'name=ppo_seed0_lr3e4_small;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=3e-4', 'name=ppo_seed0_lr4e4_small;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' ) -Episodes 1 -PauseBetweenRuns
.\scripts\visualize_so100_resume_rollouts.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_v3pose_place;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' ) -Episodes 1 -PauseBetweenRuns
.\scripts\visualize_so100_resume_rollouts.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_v7_tcp_delta_fixedjaw_gate;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' ) -Episodes 1 -PauseBetweenRuns
.\scripts\visualize_so100_resume_rollouts.ps1 -Run @( 'name=ppo_seed0_lr4e4_small_raw;algo=ppo;seed=0;network=256,128,64;steps=3000000;lr=4e-4' ) -Episodes 1 -PauseBetweenRuns