diff --git a/CLAUDE.md b/CLAUDE.md index f9cb8c0..18332fd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,12 +1,101 @@ # CLAUDE.md -## Always offer to pull down issues with Github CLI, address issues before beginning +Camera-agnostic ROS2 (Humble) wrapper for ByteDance's Depth Anything 3 monocular depth estimation, targeting real-time performance (>30 FPS) on NVIDIA Jetson Orin AGX. -## DO NOT MAKE ANY COMMITS, USER WILL MAKE COMMITS +## Version Requirements -## Always see if there are specialized agents to helps with tasks and troubleshooting, orchestrate agents to work together +- **ROS2**: Humble Hawksbill (also compatible with Jazzy/Iron) +- **Python**: 3.10+ +- **Target Hardware**: NVIDIA Jetson Orin AGX (also supports desktop GPUs) -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +## Always Follow These Guidelines + +## Environment Detection (Do This First) + +Before attempting Jetson access or system commands, determine your environment: + +### Check Available Tools + +1. **MCP Tools**: Check if OSA Tools, Windows MCP, or SSH MCP tools are available +2. **Local Environment**: Test if `~/.ssh/jetson_j4012` exists and `10.69.7.112` is reachable + +### Environment-Specific Behavior + +| Environment | SSH Access | Method | +|-------------|------------|--------| +| Claude Code (local) | Direct | `ssh -i ~/.ssh/jetson_j4012 gerdsenai@10.69.7.112` | +| Claude Cowork (Mac) | Via osascript MCP | `mcp__Control_your_Mac__osascript` runs commands on host Mac | +| Docker container | Exit first | Run SSH from host, not container | + +### Cowork + osascript MCP (Preferred Method) + +When running in Cowork on a Mac with the "Control your Mac" MCP enabled: + +```applescript +-- SSH command via osascript +do shell script "ssh -i ~/.ssh/jetson_j4012 gerdsenai@10.69.7.112 ''" +``` + +This executes on the user's Mac, which has network access to `10.69.7.112` and the SSH key at `~/.ssh/jetson_j4012`. + +### X11 Display Setup for GUI Apps + +To run GUI apps (viewers, rqt) from Docker container on Jetson display: + +```bash +# 1. Enable X11 access for Docker (run on Jetson host via SSH) +export DISPLAY=:10 +export XAUTHORITY=/run/user/1000/gdm/Xauthority +xhost +local:docker + +# 2. Run GUI app in container with display forwarding +docker exec -e DISPLAY=:10 -e XAUTHORITY=/run/user/1000/gdm/Xauthority \ + da3_ros2_jetson +``` + +## Jetson SSH Quick Reference + +These commands work from Claude Code (direct) or Cowork (via osascript MCP): + +- **Host**: `10.69.7.112` (Jetson device on local network) +- **User**: `gerdsenai` +- **Identity file**: `~/.ssh/jetson_j4012` + +```bash +# Quick connectivity test (run this first) +ping -c 1 10.69.7.112 && ls ~/.ssh/jetson_j4012 + +# SSH to Jetson +ssh -i ~/.ssh/jetson_j4012 gerdsenai@10.69.7.112 +``` + +## Git Workflow (Non-Negotiable) + +- **USER MAKES ALL COMMITS AND PRs** - Claude must NEVER commit or create PRs +- **Always branch off `main`** - Create feature branches from main for all work +- **Never commit directly to `main`** - All changes go through feature branches +- When starting work, create a new branch: `git checkout -b feature/description main` + +## GitHub CLI Usage + +Use `gh` CLI for all GitHub interactions: + +```bash +# View issues +gh issue list +gh issue view + +# View PRs +gh pr list +gh pr view + +# Check repo status +gh repo view +``` + +Always offer to pull down and review issues before beginning work. + +This file provides guidance to Claude Code and Claude Cowork when working with this repository. ## Build & Development Commands @@ -32,25 +121,126 @@ ros2 launch depth_anything_3_ros2 depth_anything_3.launch.py image_topic:=/camer # Docker (GPU) docker-compose up -d depth-anything-3-gpu -# Deploy to Jetson (via SCP) -scp -r . gerdsenai@10.69.7.112:~/depth_anything_3_ros2/ +# Run live demo (one-click from repo root) +./run.sh +``` + +## Jetson Deployment + +Jetson Orin AGX is available at `10.69.7.112`. SSH requires identity file. + +```bash +# SSH to Jetson (identity file required) +ssh -i ~/.ssh/jetson_j4012 gerdsenai@10.69.7.112 + +# Deploy via git clone (preferred - maintains git history) +ssh -i ~/.ssh/jetson_j4012 gerdsenai@10.69.7.112 \ + "git clone https://github.com/GerdsenAI/Depth-Anything-3-ROS2-Wrapper.git ~/depth_anything_3_ros2" + +# Or deploy via SCP (no git history) +scp -i ~/.ssh/jetson_j4012 -r . gerdsenai@10.69.7.112:~/depth_anything_3_ros2/ + +# Run commands remotely +ssh -i ~/.ssh/jetson_j4012 gerdsenai@10.69.7.112 "cd ~/depth_anything_3_ros2 && " +``` + +### One-Click Demo (Recommended) + +Use the `run.sh` script at repo root which handles everything: + +```bash +# On Jetson, after cloning: +cd ~/depth_anything_3_ros2 +./run.sh # Auto-detect camera, build if needed +./run.sh --camera /dev/video0 # Specify camera +./run.sh --no-display # Headless mode (SSH) +./run.sh --rebuild # Force rebuild Docker +``` + +### JetPack / L4T Version Notes + +| L4T Version | OpenCV | cuDNN | Base Image | +|-------------|--------|-------|-----------------------------------------| +| r36.2.0 | 4.8.1 | 8.x | dustynv/ros:humble-desktop-l4t-r36.2.0 | +| r36.4.0 | 4.10.0 | 9.x | dustynv/ros:humble-desktop-l4t-r36.4.0 | + +**Important**: The `humble-pytorch` variant does NOT exist for r36.x. Use `humble-desktop` instead. + +## Docker Build Known Issues (Jetson) + +### 1. pip.conf Points to Unreliable Server + +dustynv base images configure pip to use `jetson.webredirect.org` which may be unreliable. +**Fix**: Use `--index-url https://pypi.org/simple/` explicitly for pip installs. + +### 2. OpenCV Version Check + +The Dockerfile validates OpenCV version. Supported versions: + +- 4.5.x (apt packages) +- 4.8.x (L4T r36.2) +- 4.10.x (L4T r36.4) + +### 3. cuDNN Version Mismatch + +L4T r36.4.0 ships with cuDNN 9.x, but some PyTorch wheels expect cuDNN 8. +**Fix**: For host-container TRT architecture, container doesn't need CUDA-accelerated PyTorch. +Use CPU-only torchvision in container since TRT inference runs on host. + +### 4. Base Image Selection + +```dockerfile +# WRONG - doesn't exist for r36.x +FROM dustynv/ros:humble-pytorch-l4t-r36.4.0 + +# CORRECT +FROM dustynv/ros:humble-desktop-l4t-r36.4.0 ``` +## Host-Container TensorRT Architecture + +Due to broken TensorRT Python bindings in containers, we use a split architecture: + +- **Host**: Runs `scripts/trt_inference_service.py` (TensorRT inference) +- **Container**: Runs ROS2 nodes (camera driver, depth publisher) +- **Communication**: File-based IPC via `/tmp/da3_shared/` + +| File | Direction | Format | +|--------------|--------------------|------------------------------------------| +| `input.npy` | Container -> Host | float32 [1,1,3,518,518] | +| `output.npy` | Host -> Container | float32 [1,518,518] | +| `request` | Container -> Host | Timestamp signal | +| `status` | Host -> Container | "ready", "complete:time", "error:msg" | + +### Current Performance Status (2026-02-04) + +| Metric | Current | Target | Notes | +|--------|---------|--------|-------| +| FPS | 5-12 | >30 | File IPC overhead limits throughput | +| TRT Inference | ~50ms | ~26ms | Host TRT working correctly | +| GPU Utilization | 99% | - | GPU fully utilized during inference | +| IPC Overhead | ~40ms | 0ms | File read/write bottleneck | + +**Optimization Path**: Native TensorRT in container would eliminate file IPC overhead and achieve 30+ FPS. Requires TensorRT Python bindings working in L4T r36.4.0 container. + ## Architecture This is a ROS2 Humble wrapper for ByteDance's Depth Anything 3 monocular depth estimation, targeting >30 FPS on NVIDIA Jetson Orin AGX. ### 3-Layer Design + - **Node Layer** (`depth_anything_3_node.py`, `*_optimized.py`): ROS2 interface, parameter handling, topic management - **Inference Layer** (`da3_inference.py`, `*_optimized.py`): Model loading via HuggingFace, CUDA/CPU inference - **Utility Layer** (`utils.py`, `gpu_utils.py`): Depth processing, colorization, GPU acceleration ### Dual Implementation Pattern + - Standard nodes: Baseline functionality - Optimized nodes (`*_optimized.py`): TensorRT, async processing, >30 FPS target - Both expose identical ROS2 interfaces - changes to one should be reflected in the other ### Inference Wrapper Return Format + ```python {'depth': np.ndarray, # (H, W) float32 'confidence': np.ndarray, # (H, W) float32, optional @@ -60,11 +250,13 @@ This is a ROS2 Humble wrapper for ByteDance's Depth Anything 3 monocular depth e ## Critical Design Principles ### Camera-Agnostic Design (Non-Negotiable) + - NEVER add camera-specific logic to core modules - Camera integration ONLY via topic remapping and example launch files in `launch/examples/` - All cameras work through standard `sensor_msgs/Image` interface ### ROS2 Patterns + - Use relative topic names with `~` prefix (e.g., `~/depth`, `~/image_raw`) - BEST_EFFORT QoS for image subscribers (allows frame drops) - Declare all parameters in node constructor @@ -79,6 +271,7 @@ This is a ROS2 Humble wrapper for ByteDance's Depth Anything 3 monocular depth e ## Testing Tests use mocked DA3 model (doesn't require GPU): + - `test/test_inference.py` - Unit tests for inference wrapper - `test/test_node.py` - Integration tests for ROS2 node - `test/test_generic_camera.py` - Camera-agnostic functionality @@ -90,20 +283,31 @@ Tests use mocked DA3 model (doesn't require GPU): - `config/params.yaml` - Default parameters - `.github/copilot-instructions.md` - Extended AI coding guidelines +## Troubleshooting + +See these resources for common issues: + +- **README.md > Troubleshooting** - Model download, CUDA OOM, encoding issues +- **docs/JETSON_DEPLOYMENT_GUIDE.md** - TensorRT setup, host-container architecture +- **OPTIMIZATION_GUIDE.md** - Performance tuning, TensorRT compatibility + ## Specialized Agents This repository includes specialized agents in `.claude/agents/`. Use them proactively for domain-specific tasks. ### Available Agents -| Agent | Domain | Use When | -|-------|--------|----------| -| `jetson-expert` | Hardware | Module selection, flashing, BSP, carrier boards, GPIO/CSI, thermal, boot issues | -| `nvidia-expert` | Software | CUDA, TensorRT, DeepStream, Isaac ROS, containers, profiling, PyTorch/TensorFlow | +Located in `.claude/agents/`: + +| Agent | File | Domain | Use When | +|-------|------|--------|----------| +| `jetson-expert` | `jetson-expert.md` | Hardware | Module selection, flashing, BSP, carrier boards, GPIO/CSI, thermal, boot issues | +| `nvidia-expert` | `nvidia-expert.md` | Software | CUDA, TensorRT, DeepStream, Isaac ROS, containers, profiling, PyTorch/TensorFlow | ### Agent Selection Guide **Hardware questions** -> `jetson-expert`: + - "Which Jetson module should I use?" - "How do I flash JetPack 6.x?" - "Camera not detected on CSI port" @@ -113,6 +317,7 @@ This repository includes specialized agents in `.claude/agents/`. Use them proac - "Device tree or pinmux setup" **Software questions** -> `nvidia-expert`: + - "How do I convert ONNX to TensorRT?" - "Optimize inference performance" - "DeepStream pipeline design" @@ -125,20 +330,21 @@ This repository includes specialized agents in `.claude/agents/`. Use them proac Some issues require both agents working together: -| Scenario | Primary Agent | Secondary Agent | Reason | -|----------|---------------|-----------------|--------| -| Slow inference on Orin NX | `nvidia-expert` | `jetson-expert` | Software first, then check thermal/power | -| Container can't access GPU | `nvidia-expert` | `jetson-expert` | Runtime config first, then driver/L4T check | -| CSI camera not detected | `jetson-expert` | - | Hardware/device tree issue | -| TensorRT build fails | `nvidia-expert` | - | Software/model issue | -| JetPack 6.x upgrade | `jetson-expert` | `nvidia-expert` | Flash first, then container compatibility | -| Performance varies wildly | `nvidia-expert` | `jetson-expert` | Profile first, then check thermal throttling | +| Scenario | Primary Agent | Secondary Agent | Reason | +|----------------------------|-----------------|------------------|------------------------------------------------| +| Slow inference on Orin NX | `nvidia-expert` | `jetson-expert` | Software first, then check thermal/power | +| Container can't access GPU | `nvidia-expert` | `jetson-expert` | Runtime config first, then driver/L4T check | +| CSI camera not detected | `jetson-expert` | - | Hardware/device tree issue | +| TensorRT build fails | `nvidia-expert` | - | Software/model issue | +| JetPack 6.x upgrade | `jetson-expert` | `nvidia-expert` | Flash first, then container compatibility | +| Performance varies wildly | `nvidia-expert` | `jetson-expert` | Profile first, then check thermal throttling | ### Proactive Agent Usage ALWAYS consider using specialized agents when: + 1. User mentions Jetson hardware or deployment -> Consider `jetson-expert` 2. User asks about AI/ML optimization -> Consider `nvidia-expert` 3. Troubleshooting involves both HW and SW -> Use both agents sequentially 4. Task is outside ROS2/Python expertise -> Use appropriate agent -5. Performance issues arise -> Start with `nvidia-expert`, escalate to `jetson-expert` if thermal +5. Performance issues arise -> Start with `nvidia-expert`, escalate to `jetson-expert` if thermal \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 78c7b0d..8d26886 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,14 +87,17 @@ RUN apt-get update && apt-get install -y \ # # IMPORTANT: OpenCV Conflict Resolution # -------------------------------------- -# The dustynv base image includes OpenCV 4.8.1 with CUDA support (opencv-dev, opencv-libs). +# The dustynv base images include OpenCV with CUDA support: +# - L4T r36.2.0: OpenCV 4.8.1 +# - L4T r36.4.0: OpenCV 4.10.0 # ROS Humble apt packages (ros-humble-cv-bridge, ros-humble-vision-opencv) depend on # Ubuntu's OpenCV 4.5.4, which conflicts with the pre-installed version. # -# Solution: Build cv_bridge and image_geometry from source against OpenCV 4.8.1. +# Solution: Build cv_bridge and image_geometry from source against the installed OpenCV. # This preserves CUDA acceleration while providing ROS2 compatibility. # ============================================================================== -FROM dustynv/ros:humble-pytorch-l4t-${L4T_VERSION} AS jetson-base +# NOTE: humble-pytorch variant doesn't exist for r36.x - use humble-desktop instead +FROM dustynv/ros:humble-desktop-l4t-${L4T_VERSION} AS jetson-base ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 @@ -104,13 +107,15 @@ RUN curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o && gpg --batch --yes --dearmor -o /usr/share/keyrings/ros-archive-keyring.gpg /tmp/ros.key \ && rm /tmp/ros.key -# Verify OpenCV 4.8.x is present (critical check - build will fail if missing) +# Verify OpenCV 4.x is present (critical check - build will fail if missing) +# Supported: 4.5.x (apt), 4.8.x (L4T r36.2), 4.10.x (L4T r36.4) RUN echo "=== Checking pre-installed OpenCV ===" && \ pkg-config --modversion opencv4 && \ OPENCV_VERSION=$(pkg-config --modversion opencv4) && \ echo "Found OpenCV version: $OPENCV_VERSION" && \ case "$OPENCV_VERSION" in \ - 4.8.*) echo "OpenCV 4.8.x detected - proceeding with source build of cv_bridge" ;; \ + 4.10.*) echo "OpenCV 4.10.x detected (L4T r36.4) - proceeding with source build of cv_bridge" ;; \ + 4.8.*) echo "OpenCV 4.8.x detected (L4T r36.2) - proceeding with source build of cv_bridge" ;; \ 4.5.*) echo "WARNING: OpenCV 4.5.x detected - apt packages should work, but we'll build from source anyway" ;; \ *) echo "ERROR: Unexpected OpenCV version: $OPENCV_VERSION" && exit 1 ;; \ esac @@ -209,13 +214,18 @@ FROM ${BUILD_TYPE} AS builder ARG BUILD_TYPE +# IMPORTANT: Override dustynv's pip configuration that points to unreliable jetson.webredirect.org +# The base image sets PIP_INDEX_URL env var, so we must override it +ENV PIP_INDEX_URL=https://pypi.org/simple/ +RUN rm -f /etc/pip.conf /root/.config/pip/pip.conf 2>/dev/null || true + # Set working directory WORKDIR /tmp/build # Copy requirements COPY requirements.txt . -# Upgrade pip +# Upgrade pip (uses PyPI now that pip.conf is removed) RUN pip3 install --upgrade pip setuptools wheel # Install system libraries required by PyTorch on Jetson @@ -235,6 +245,8 @@ RUN if [ "$BUILD_TYPE" = "cuda-base" ]; then \ # L4T r36.4.0 ships with CUDA 12.6 and cuDNN 9.x # Download PyTorch wheel from NVIDIA (JetPack 6.2 / L4T r36.4.0) # Note: PyTorch 2.3.0 wheel is compatible with both r36.2.0 and r36.4.0 + # pip.conf removed at builder stage start, so this uses PyPI + pip3 install --no-cache-dir filelock typing-extensions sympy networkx jinja2 fsspec && \ wget -q -O /tmp/torch-2.3.0-cp310-cp310-linux_aarch64.whl \ "https://nvidia.box.com/shared/static/mp164asf3sceb570wvjsrezk1p4ftj8t.whl" && \ pip3 install --no-cache-dir /tmp/torch-2.3.0-cp310-cp310-linux_aarch64.whl && \ @@ -245,18 +257,12 @@ RUN if [ "$BUILD_TYPE" = "cuda-base" ]; then \ --ignore-installed sympy; \ fi -# For Jetson: Build torchvision from source to match NVIDIA PyTorch ABI -# This is required because PyPI torchvision wheels don't match NVIDIA PyTorch's C++ ABI +# For Jetson: Skip torchvision source build - not needed for host-container TRT architecture +# TensorRT inference runs on HOST (not in container), so container only needs basic vision ops +# Install CPU-only torchvision from PyPI (sufficient for ROS2 image handling) RUN if [ "$BUILD_TYPE" = "jetson-base" ]; then \ - apt-get update && apt-get install -y --no-install-recommends \ - libjpeg-dev zlib1g-dev libpython3-dev libopenblas-dev \ - libavcodec-dev libavformat-dev libswscale-dev && \ - rm -rf /var/lib/apt/lists/* && \ - git clone --depth 1 --branch v0.18.0 https://github.com/pytorch/vision.git /tmp/torchvision && \ - cd /tmp/torchvision && \ - TORCH_CUDA_ARCH_LIST="8.7" FORCE_CUDA=1 python3 setup.py bdist_wheel && \ - pip3 install --no-cache-dir dist/*.whl && \ - cd / && rm -rf /tmp/torchvision; \ + pip3 install --no-cache-dir torchvision --no-deps && \ + pip3 install --no-cache-dir pillow; \ fi # NOTE: PyTorch import verification deferred to runtime @@ -264,7 +270,8 @@ RUN if [ "$BUILD_TYPE" = "jetson-base" ]; then \ # torch.cuda.is_available() only works at runtime with GPU access # Install other Python dependencies -RUN pip3 install --no-cache-dir \ +# --ignore-installed needed for sympy on Jetson (distutils-installed, cannot uninstall) +RUN pip3 install --no-cache-dir --ignore-installed \ "transformers>=4.35.0" \ "huggingface-hub>=0.19.0" \ "opencv-python>=4.8.0" \ @@ -348,12 +355,17 @@ ENV PYTHONPATH=/ros2_ws/install/depth_anything_3_ros2/lib/python3.10/site-packag # Source ROS2 workspace in bashrc (both user and system-wide for docker exec) # NOTE: dustynv Jetson containers use /opt/ros/humble/install/setup.bash # Standard OSRF containers use /opt/ros/humble/setup.bash -# We source both paths with fallback to support all base images -RUN echo "source /opt/ros/humble/setup.bash 2>/dev/null || source /opt/ros/humble/install/setup.bash" >> ~/.bashrc && \ - echo "source /ros2_ws/install/setup.bash 2>/dev/null || true" >> ~/.bashrc && \ - echo "source /opt/ros/humble/setup.bash 2>/dev/null || source /opt/ros/humble/install/setup.bash" >> /etc/bash.bashrc && \ +# We add sourcing BEFORE the "[ -z "$PS1" ] && return" line in .bashrc +# to ensure it runs for non-interactive shells (docker exec bash -c "...") +RUN if grep -q 'PS1.*return' ~/.bashrc; then \ + sed -i '/\[ -z "\$PS1" \] && return/i source /opt/ros/humble/install/setup.bash 2>/dev/null || source /opt/ros/humble/setup.bash\nsource /ros2_ws/install/setup.bash 2>/dev/null || true' ~/.bashrc; \ + else \ + echo "source /opt/ros/humble/install/setup.bash 2>/dev/null || source /opt/ros/humble/setup.bash" >> ~/.bashrc; \ + echo "source /ros2_ws/install/setup.bash 2>/dev/null || true" >> ~/.bashrc; \ + fi && \ + echo "source /opt/ros/humble/install/setup.bash 2>/dev/null || source /opt/ros/humble/setup.bash" >> /etc/bash.bashrc && \ echo "source /ros2_ws/install/setup.bash 2>/dev/null || true" >> /etc/bash.bashrc && \ - echo "source /opt/ros/humble/setup.bash 2>/dev/null || source /opt/ros/humble/install/setup.bash" >> /etc/profile.d/ros2.sh && \ + echo "source /opt/ros/humble/install/setup.bash 2>/dev/null || source /opt/ros/humble/setup.bash" >> /etc/profile.d/ros2.sh && \ echo "source /ros2_ws/install/setup.bash 2>/dev/null || true" >> /etc/profile.d/ros2.sh && \ chmod +x /etc/profile.d/ros2.sh 2>/dev/null || true diff --git a/OPTIMIZATION_GUIDE.md b/OPTIMIZATION_GUIDE.md index bd8d340..133dcb7 100644 --- a/OPTIMIZATION_GUIDE.md +++ b/OPTIMIZATION_GUIDE.md @@ -35,6 +35,21 @@ DA3_TENSORRT_AUTO=true docker compose up depth-anything-3-jetson --- +## Current Architecture Limitation (2026-02-04) + +**Host-Container File IPC** limits throughput to 10-15 FPS due to numpy file read/write overhead. + +| Architecture | TRT Inference | IPC Overhead | Total | FPS | +|--------------|---------------|--------------|-------|-----| +| Native (target) | ~26ms | 0ms | ~26ms | ~38 | +| Host-Container File IPC (current) | ~50ms | ~40ms | ~90ms | ~11 | + +**Current Bottleneck:** TensorRT runs on host, ROS2 in container. Communication via `/tmp/da3_shared/` files (input.npy, output.npy) adds ~40ms per frame. + +**To achieve 30+ FPS:** Run TensorRT natively inside container (requires working TensorRT Python bindings in L4T r36.4.0 containers). + +--- + ## Validated Performance on Jetson Orin NX 16GB ### PyTorch Baseline diff --git a/README.md b/README.md index 5d8d0da..1ba7759 100644 --- a/README.md +++ b/README.md @@ -504,34 +504,34 @@ cat /tmp/da3_demo_logs/node_*.log ### TensorRT Demo (Jetson) -For Jetson users, we provide a single-command demo script that handles everything automatically: +For Jetson users, we provide a single-command demo script at the repo root that handles everything automatically: ```bash -# After SCP'ing to Jetson -ssh gerdsenai@10.69.7.112 +# Clone directly on Jetson +git clone https://github.com/GerdsenAI/Depth-Anything-3-ROS2-Wrapper.git ~/depth_anything_3_ros2 cd ~/depth_anything_3_ros2 -bash scripts/demo.sh + +# Run the demo (first run takes ~15-20 min for Docker build + TRT engine) +./run.sh ``` -The demo script will: -1. **Auto-detect cameras** (USB and CSI) and let you select if multiple are found -2. **Build TensorRT engine** on first run (~2 minutes) -3. **Start TRT inference service** for 40+ FPS performance (93+ FPS at 308x308) -4. **Launch Docker container** with ROS2 depth estimation node -5. **Open performance monitor** showing live FPS, latency, and GPU stats -6. **Optionally launch RViz2** for visualization +The `run.sh` script will: +1. **Build Docker image** if not already built (~15-20 minutes first run) +2. **Download ONNX model** from HuggingFace and build TensorRT engine (~2 minutes) +3. **Auto-detect cameras** (USB and CSI) +4. **Start TRT inference service** on host for 40+ FPS performance +5. **Launch Docker container** with ROS2 depth estimation node + +Subsequent runs start in ~10 seconds. ### Demo Script Options ```bash -bash scripts/demo.sh --help +./run.sh --help Options: --camera DEVICE Specify camera device (e.g., /dev/video0) - --topic TOPIC Specify ROS2 image topic directly - --no-rviz Skip RViz2 launch - --no-monitor Skip performance monitor - --no-trt Use PyTorch instead of TensorRT + --no-display Run in headless mode (for SSH) --rebuild Force rebuild Docker image ``` @@ -1179,11 +1179,11 @@ Thermal stability validated: 10-minute sustained load at 40.79 FPS with no throt ```bash cd ~/depth_anything_3_ros2 -bash scripts/deploy_jetson.sh --host-trt +./run.sh ``` This script: -1. Verifies TensorRT 10.3 on host +1. Builds Docker image if needed 2. Downloads ONNX model if missing 3. Builds TensorRT FP16 engine (~2 min) 4. Starts host inference service @@ -1193,8 +1193,8 @@ This script: | File | Purpose | |------|--------| +| `run.sh` | One-click demo launcher (repo root) | | `scripts/trt_inference_service.py` | Host-side TRT inference service | -| `scripts/deploy_jetson.sh` | Automated deployment | | `depth_anything_3_ros2/da3_inference.py` | Inference wrapper (shared memory) | See [docs/JETSON_DEPLOYMENT_GUIDE.md](docs/JETSON_DEPLOYMENT_GUIDE.md) for complete documentation. @@ -1368,6 +1368,24 @@ ros2 topic info /camera/image_raw --param log_inference_time:=true ``` +#### 6. Jetson Docker Build Failures + +**Error**: `dustynv/ros:humble-pytorch-l4t-r36.x.x` not found + +**Solution**: The humble-pytorch variant doesn't exist for L4T r36.x. Use `humble-desktop` instead: +```dockerfile +# In docker-compose.yml, set: +L4T_VERSION: r36.4.0 # Uses humble-desktop variant +``` + +**Error**: `pip install` fails with connection errors to `jetson.webredirect.org` + +**Solution**: The dustynv base images configure pip to use an unreliable custom index. The Dockerfile includes `--index-url https://pypi.org/simple/` to override this. + +**Error**: `ImportError: libcudnn.so.8: cannot open shared object file` + +**Solution**: L4T r36.4.0 ships with cuDNN 9.x, but some PyTorch wheels expect cuDNN 8. For the host-container TRT architecture, the container doesn't need CUDA-accelerated PyTorch since TensorRT inference runs on the host. The Dockerfile uses CPU-only torchvision in the container. + --- ## Development diff --git a/TODO.md b/TODO.md index 9976d54..ff280a9 100644 --- a/TODO.md +++ b/TODO.md @@ -120,4 +120,33 @@ See `docs/JETSON_BENCHMARKS.md` for full benchmark documentation. --- -**Last Updated:** 2026-02-02 +## Phase 5: Live Demo System [IN PROGRESS] + +### Components Added +- [x] `scripts/demo_depth_viewer.py` - ROS2 viewer with side-by-side camera + depth display +- [x] `scripts/run_demo.sh` - Demo runner (starts TRT service, camera, depth node, viewer) +- [x] `scripts/jetson_demo.sh` - Jetson-specific entrypoint for systems with local display +- [x] Atomic IO for numpy files (prevents partial reads) +- [x] Dockerfile ROS2 sourcing fix for non-interactive shells + +### Demo Features +- Side-by-side camera feed and colorized TensorRT depth +- FPS toggle display +- Frame capture to `demo_captures/` +- X11 support with SSH fallback (shows TRT stats instead of GUI) + +### Usage +```bash +# On Jetson with display +bash scripts/jetson_demo.sh + +# General demo (container) +bash scripts/run_demo.sh +``` + +### Pending +- [ ] Merge TensorRT-Testing branch to main (PR pending) + +--- + +**Last Updated:** 2026-02-03 diff --git a/depth_anything_3_ros2/da3_inference.py b/depth_anything_3_ros2/da3_inference.py index e8d3ec2..0f2bd5d 100644 --- a/depth_anything_3_ros2/da3_inference.py +++ b/depth_anything_3_ros2/da3_inference.py @@ -134,7 +134,7 @@ def _inference_via_shared_memory(self, image: np.ndarray) -> Dict[str, np.ndarra Protocol: 1. Preprocess image to tensor format expected by TRT engine - 2. Write tensor to INPUT_PATH + 2. Write tensor to INPUT_PATH (atomic via temp file + rename) 3. Write timestamp to REQUEST_PATH to signal new request 4. Wait for STATUS_PATH to show "complete" 5. Read depth from OUTPUT_PATH @@ -143,11 +143,18 @@ def _inference_via_shared_memory(self, image: np.ndarray) -> Dict[str, np.ndarra # DA3 expects: (1, 1, 3, H, W) normalized float32 input_tensor = self._preprocess_image(image) - # Write input tensor - np.save(INPUT_PATH, input_tensor) - - # Signal new request with timestamp - REQUEST_PATH.write_text(str(time.time())) + # Write input tensor atomically (temp file + fsync + rename) + temp_path = INPUT_PATH.parent / "input_tmp.npy" + with open(temp_path, 'wb') as f: + np.save(f, input_tensor, allow_pickle=False) + f.flush() + os.fsync(f.fileno()) + temp_path.replace(INPUT_PATH) # Atomic rename + + # Signal new request with timestamp (atomic write to prevent race condition) + temp_request = REQUEST_PATH.parent / "request_tmp" + temp_request.write_text(str(time.time())) + temp_request.replace(REQUEST_PATH) # Atomic rename # Wait for completion start_time = time.time() diff --git a/docker-compose.yml b/docker-compose.yml index 110487e..0df1eb5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,7 +68,7 @@ services: dockerfile: Dockerfile args: BUILD_TYPE: jetson-base - L4T_VERSION: r36.2.0 # Base image - host TRT 10.3 mounted at runtime + L4T_VERSION: r36.4.0 # Base image - matches host L4T R36.4.x image: depth_anything_3_ros2:jetson container_name: da3_ros2_jetson stdin_open: true diff --git a/docs/JETSON_DEPLOYMENT_GUIDE.md b/docs/JETSON_DEPLOYMENT_GUIDE.md index f5999e4..475cc7a 100644 --- a/docs/JETSON_DEPLOYMENT_GUIDE.md +++ b/docs/JETSON_DEPLOYMENT_GUIDE.md @@ -44,7 +44,7 @@ Due to broken TensorRT Python bindings in available Jetson containers ([Issue #7 **Why this approach:** - `dustynv/l4t-pytorch:r36.4.0` has broken TensorRT Python bindings -- `dustynv/ros:humble-pytorch-l4t-r36.4.0` does not exist +- Using `dustynv/ros:humble-desktop-l4t-r36.4.0` (humble-pytorch variant doesn't exist) - Container TRT 8.6.2 cannot build DA3 engines (DINOv2 incompatibility) - Host TRT 10.3 works perfectly (validated at 25ms latency) diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..3b9d66b --- /dev/null +++ b/run.sh @@ -0,0 +1,413 @@ +#!/bin/bash +# +# Depth Anything 3 ROS2 - One-Click Demo +# +# This script handles everything needed to run the depth estimation demo: +# 1. Builds Docker image (if not already built) +# 2. Downloads ONNX model and builds TensorRT engine (first run only) +# 3. Starts TensorRT inference service on host (10-15 FPS with file IPC) +# 4. Auto-detects camera (USB or CSI) +# 5. Starts ROS2 container with camera and depth nodes +# 6. Opens depth visualization window +# +# Usage: +# ./run.sh # Auto-detect everything +# ./run.sh --camera /dev/video0 # Specify camera +# ./run.sh --no-display # Headless mode (SSH) +# ./run.sh --rebuild # Force rebuild Docker +# ./run.sh --help # Show all options +# +# Requirements: +# - Jetson with JetPack 6.x (TensorRT 10.3+) +# - Docker with nvidia runtime +# - USB or CSI camera (auto-detected) +# - ~20GB disk space for Docker image +# +# First run takes ~15-20 minutes (Docker build + TRT engine). +# Subsequent runs start in ~10 seconds. +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +# Configuration +CONTAINER_NAME="da3_demo" +IMAGE_NAME="depth_anything_3_ros2:jetson" +SHARED_DIR="/tmp/da3_shared" +ONNX_DIR="models/onnx" +TRT_DIR="models/tensorrt" +ONNX_MODEL="$ONNX_DIR/da3-small-embedded.onnx" +TRT_ENGINE="$TRT_DIR/da3-small-fp16.engine" +TRTEXEC="/usr/src/tensorrt/bin/trtexec" + +# Default options +CAMERA_DEVICE="" +NO_DISPLAY=false +FORCE_REBUILD=false +SHOW_HELP=false + +# Process IDs for cleanup +TRT_SERVICE_PID="" +CONTAINER_PID="" + +# Banner +print_banner() { + echo "" + echo -e "${BOLD}============================================${NC}" + echo -e "${BOLD} Depth Anything 3 ROS2 - Demo${NC}" + echo -e "${BOLD}============================================${NC}" + echo "" +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --camera) + CAMERA_DEVICE="$2" + shift 2 + ;; + --no-display|--headless) + NO_DISPLAY=true + shift + ;; + --rebuild) + FORCE_REBUILD=true + shift + ;; + -h|--help) + SHOW_HELP=true + shift + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" + echo "Use --help for usage" + exit 1 + ;; + esac +done + +if [ "$SHOW_HELP" = true ]; then + print_banner + echo "Usage: ./run.sh [options]" + echo "" + echo "Options:" + echo " --camera DEVICE Specify camera device (e.g., /dev/video0)" + echo " --no-display Run in headless mode (for SSH)" + echo " --rebuild Force rebuild Docker image" + echo " -h, --help Show this help message" + echo "" + echo "Examples:" + echo " ./run.sh # Auto-detect camera, show display" + echo " ./run.sh --camera /dev/video0 # Use specific camera" + echo " ./run.sh --no-display # SSH mode (no viewer window)" + echo "" + echo "First run: ~15-20 minutes (Docker build + TensorRT engine)" + echo "Subsequent: ~10 seconds" + exit 0 +fi + +print_banner + +# Cleanup function +cleanup() { + echo "" + echo -e "${YELLOW}Shutting down...${NC}" + + # Stop container + if docker ps -q -f name="$CONTAINER_NAME" 2>/dev/null | grep -q .; then + docker stop "$CONTAINER_NAME" 2>/dev/null || true + fi + docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + + # Stop TRT service + if [ -n "$TRT_SERVICE_PID" ] && kill -0 "$TRT_SERVICE_PID" 2>/dev/null; then + kill "$TRT_SERVICE_PID" 2>/dev/null || true + wait "$TRT_SERVICE_PID" 2>/dev/null || true + fi + + # Cleanup shared memory + rm -rf "$SHARED_DIR"/* 2>/dev/null || true + + echo -e "${GREEN}Done.${NC}" +} +trap cleanup EXIT INT TERM + +# Step 1: Pre-flight checks +echo -e "${CYAN}[1/6] Checking requirements...${NC}" + +# Check Docker +if ! command -v docker &> /dev/null; then + echo -e "${RED}ERROR: Docker not installed${NC}" + echo "Install: sudo apt install docker.io nvidia-container-toolkit" + exit 1 +fi + +if ! docker info &> /dev/null; then + echo -e "${RED}ERROR: Cannot connect to Docker daemon${NC}" + echo "Fix: sudo usermod -aG docker \$USER && newgrp docker" + exit 1 +fi + +# Check TensorRT +USE_TRT=true +if [ ! -f "$TRTEXEC" ]; then + echo -e "${YELLOW}WARNING: TensorRT trtexec not found${NC}" + echo " Will use PyTorch backend (~5 FPS instead of ~40 FPS)" + USE_TRT=false +fi + +# Check Python TensorRT bindings +if [ "$USE_TRT" = true ]; then + if ! python3 -c "import tensorrt" 2>/dev/null; then + echo -e "${YELLOW}WARNING: TensorRT Python bindings not installed${NC}" + echo " Will use PyTorch backend (~5 FPS instead of ~40 FPS)" + USE_TRT=false + fi +fi + +echo -e " ${GREEN}Requirements OK${NC}" + +# Step 2: Camera detection +echo -e "${CYAN}[2/6] Detecting camera...${NC}" + +if [ -z "$CAMERA_DEVICE" ]; then + # Auto-detect camera + if [ -e "/dev/video0" ]; then + CAMERA_DEVICE="/dev/video0" + # Try to get camera name + CAMERA_NAME=$(v4l2-ctl --device="$CAMERA_DEVICE" --info 2>/dev/null | grep "Card type" | cut -d: -f2 | xargs || echo "USB Camera") + echo -e " Found: ${GREEN}$CAMERA_NAME${NC} ($CAMERA_DEVICE)" + else + echo -e "${RED}ERROR: No camera found at /dev/video0${NC}" + echo "" + echo "Options:" + echo " 1. Connect a USB camera" + echo " 2. Specify camera: ./run.sh --camera /dev/video1" + exit 1 + fi +else + if [ ! -e "$CAMERA_DEVICE" ]; then + echo -e "${RED}ERROR: Camera not found: $CAMERA_DEVICE${NC}" + exit 1 + fi + echo -e " Using: ${GREEN}$CAMERA_DEVICE${NC}" +fi + +# Step 3: Docker image +echo -e "${CYAN}[3/6] Checking Docker image...${NC}" + +if [ "$FORCE_REBUILD" = true ] || ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -q "^$IMAGE_NAME$"; then + echo " Building Docker image (this takes 15-20 minutes)..." + echo "" + docker compose build depth-anything-3-jetson 2>&1 | tail -20 + echo "" + echo -e " ${GREEN}Docker image built${NC}" +else + IMAGE_SIZE=$(docker images --format '{{.Size}}' "$IMAGE_NAME" 2>/dev/null || echo "unknown") + echo -e " Image ready: ${GREEN}$IMAGE_NAME${NC} ($IMAGE_SIZE)" +fi + +# Step 4: TensorRT engine +if [ "$USE_TRT" = true ]; then + echo -e "${CYAN}[4/6] Checking TensorRT engine...${NC}" + + mkdir -p "$ONNX_DIR" "$TRT_DIR" + + # Download ONNX if needed + if [ ! -f "$ONNX_MODEL" ]; then + echo " Downloading ONNX model..." + + # Install dependencies if needed + python3 -c "import huggingface_hub" 2>/dev/null || pip3 install -q huggingface_hub + python3 -c "import onnx" 2>/dev/null || pip3 install -q onnx + + python3 << 'PYEOF' +import os +from huggingface_hub import snapshot_download +import onnx + +onnx_dir = "models/onnx" +hf_dir = os.path.join(onnx_dir, "hf-download") +output = os.path.join(onnx_dir, "da3-small-embedded.onnx") + +print(" Downloading from HuggingFace...") +snapshot_download( + repo_id="onnx-community/depth-anything-v3-small", + local_dir=hf_dir, + allow_patterns=["*.onnx", "*.onnx_data"] +) + +print(" Creating embedded ONNX file...") +model = onnx.load(os.path.join(hf_dir, "onnx", "model.onnx")) +onnx.save(model, output, save_as_external_data=False) +print(f" Saved: {output}") +PYEOF + fi + + # Build TensorRT engine if needed + if [ ! -f "$TRT_ENGINE" ]; then + echo " Building TensorRT engine (takes ~2 minutes)..." + echo "" + + $TRTEXEC \ + --onnx="$ONNX_MODEL" \ + --saveEngine="$TRT_ENGINE" \ + --fp16 \ + --memPoolSize=workspace:2048MiB \ + --optShapes=pixel_values:1x1x3x518x518 \ + 2>&1 | grep -E "(Building|Serializing|SUCCESS|Throughput)" || true + + echo "" + fi + + if [ -f "$TRT_ENGINE" ]; then + ENGINE_SIZE=$(du -h "$TRT_ENGINE" | cut -f1) + echo -e " Engine ready: ${GREEN}$TRT_ENGINE${NC} ($ENGINE_SIZE)" + else + echo -e "${YELLOW} Engine build failed, using PyTorch backend${NC}" + USE_TRT=false + fi +else + echo -e "${CYAN}[4/6] Skipping TensorRT (PyTorch mode)${NC}" +fi + +# Step 5: Start TensorRT service +if [ "$USE_TRT" = true ]; then + echo -e "${CYAN}[5/6] Starting TensorRT service...${NC}" + + mkdir -p "$SHARED_DIR" + chmod 777 "$SHARED_DIR" + rm -f "$SHARED_DIR"/* 2>/dev/null || true + + # Install pycuda if needed + python3 -c "import pycuda.driver" 2>/dev/null || pip3 install -q pycuda + + python3 scripts/trt_inference_service.py \ + --engine "$TRT_ENGINE" \ + --poll-interval 0.001 \ + > /tmp/trt_service.log 2>&1 & + TRT_SERVICE_PID=$! + + # Wait for ready + for i in {1..50}; do + if [ -f "$SHARED_DIR/status" ]; then + STATUS=$(cat "$SHARED_DIR/status" 2>/dev/null || echo "") + if [[ "$STATUS" == ready* ]] || [[ "$STATUS" == complete* ]]; then + echo -e " ${GREEN}TRT service running${NC} (PID: $TRT_SERVICE_PID)" + break + fi + fi + sleep 0.1 + done + + if ! kill -0 "$TRT_SERVICE_PID" 2>/dev/null; then + echo -e "${YELLOW} TRT service failed, using PyTorch${NC}" + USE_TRT=false + fi +else + echo -e "${CYAN}[5/6] Skipping TRT service (PyTorch mode)${NC}" +fi + +# Step 6: Start container +echo -e "${CYAN}[6/6] Starting demo container...${NC}" + +# Stop any existing container +docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + +# Build Docker run args +DOCKER_ARGS=( + "--rm" + "--name" "$CONTAINER_NAME" + "--runtime" "nvidia" + "--network" "host" + "--ipc" "host" + "-e" "ROS_DOMAIN_ID=${ROS_DOMAIN_ID:-0}" +) + +# Add camera device with full v4l2 access +# v4l2 cameras need video group and proper device permissions for memory mapping +DOCKER_ARGS+=( + "--device" "$CAMERA_DEVICE:$CAMERA_DEVICE" + "--group-add" "video" + "-v" "/dev:/dev:rw" + "--device-cgroup-rule" "c 81:* rmw" +) + +# Add display if available +if [ "$NO_DISPLAY" = false ] && [ -n "$DISPLAY" ]; then + xhost +local:docker 2>/dev/null || true + DOCKER_ARGS+=( + "-e" "DISPLAY=$DISPLAY" + "-e" "QT_X11_NO_MITSHM=1" + "-v" "/tmp/.X11-unix:/tmp/.X11-unix:rw" + ) +fi + +# Add TRT shared memory +if [ "$USE_TRT" = true ]; then + DOCKER_ARGS+=( + "-v" "$SHARED_DIR:$SHARED_DIR:rw" + "-e" "DA3_USE_SHARED_MEMORY=true" + ) +fi + +# Build launch command +LAUNCH_CMD="ros2 run v4l2_camera v4l2_camera_node --ros-args" +LAUNCH_CMD+=" -p video_device:=$CAMERA_DEVICE" +LAUNCH_CMD+=" -r /image_raw:=/camera/image_raw" +LAUNCH_CMD+=" & sleep 2" +LAUNCH_CMD+=" && ros2 launch depth_anything_3_ros2 depth_anything_3.launch.py" +LAUNCH_CMD+=" image_topic:=/camera/image_raw" +LAUNCH_CMD+=" publish_colored:=true" +if [ "$USE_TRT" = true ]; then + LAUNCH_CMD+=" use_shared_memory:=true" +fi + +echo "" +echo -e "${BOLD}Demo Configuration:${NC}" +echo " Camera: $CAMERA_DEVICE" +echo " Backend: $([ "$USE_TRT" = true ] && echo "TensorRT FP16 (10-15 FPS via file IPC)" || echo "PyTorch (~5 FPS)")" +echo " Display: $([ "$NO_DISPLAY" = false ] && [ -n "$DISPLAY" ] && echo "Yes" || echo "Headless")" +echo "" + +# Start container +echo -e "${GREEN}Starting depth estimation...${NC}" +echo "" + +docker run "${DOCKER_ARGS[@]}" "$IMAGE_NAME" \ + bash -c "source /opt/ros/humble/install/setup.bash 2>/dev/null || source /opt/ros/humble/setup.bash && source /ros2_ws/install/setup.bash && $LAUNCH_CMD" & +CONTAINER_PID=$! + +sleep 5 + +echo -e "${BOLD}============================================${NC}" +echo -e "${GREEN}Demo is running!${NC}" +echo -e "${BOLD}============================================${NC}" +echo "" +echo "ROS2 Topics:" +echo " Input: /camera/image_raw" +echo " Depth: /depth_anything_3/depth" +echo " Colored: /depth_anything_3/depth_colored" +echo "" + +if [ "$NO_DISPLAY" = false ] && [ -n "$DISPLAY" ]; then + echo "View depth output:" + echo " rqt_image_view /depth_anything_3/depth_colored" + echo "" +fi + +echo "Press Ctrl+C to stop" +echo "" + +# Wait for container +wait $CONTAINER_PID 2>/dev/null || true diff --git a/scripts/demo.sh b/scripts/demo.sh index 097b9ec..664d7dc 100644 --- a/scripts/demo.sh +++ b/scripts/demo.sh @@ -1,5 +1,15 @@ #!/bin/bash -# Depth Anything V3 - Demo Script +# DEPRECATED: Use ./run.sh at repo root instead +# +# This script is kept for backwards compatibility but may be removed in future versions. +# The new ./run.sh script provides all functionality with a simpler interface: +# ./run.sh # Auto-detect everything +# ./run.sh --camera /dev/video0 # Specific camera +# ./run.sh --no-display # Headless mode +# ./run.sh --rebuild # Force rebuild +# +# ------------------------------------------------------------------ +# Depth Anything V3 - Demo Script (Legacy) # Single-command demo launcher for Jetson deployment # # Usage: bash scripts/demo.sh [options] diff --git a/scripts/demo_depth_viewer.py b/scripts/demo_depth_viewer.py new file mode 100755 index 0000000..36c6258 --- /dev/null +++ b/scripts/demo_depth_viewer.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +""" +Depth Anything 3 - Live Depth Viewer Demo + +This script displays a live side-by-side view of: +- Left: Original camera feed +- Right: Colorized depth estimation + +Requirements: +- TRT inference service running (started automatically) +- Camera connected (/dev/video0) +- Display connected to Jetson + +Usage: + python3 scripts/demo_depth_viewer.py + +Controls: + q - Quit + s - Save current frame + f - Toggle FPS display +""" + +import cv2 +import numpy as np +import subprocess +import signal +import sys +import time +import os +from pathlib import Path + +# ROS2 imports +try: + import rclpy + from rclpy.node import Node + from rclpy.qos import QoSProfile, ReliabilityPolicy, HistoryPolicy + from sensor_msgs.msg import Image + from cv_bridge import CvBridge + ROS2_AVAILABLE = True +except ImportError: + ROS2_AVAILABLE = False + print("ROS2 not available - running in standalone mode") + + +class DepthViewer: + """Live depth visualization viewer.""" + + def __init__(self): + self.bridge = CvBridge() if ROS2_AVAILABLE else None + self.latest_rgb = None + self.latest_depth = None + self.fps_display = True + self.frame_count = 0 + self.start_time = time.time() + self.last_fps = 0 + self.running = True + + # Window setup + self.window_name = "Depth Anything 3 - Live Demo" + cv2.namedWindow(self.window_name, cv2.WINDOW_NORMAL) + cv2.resizeWindow(self.window_name, 1280, 480) + + def rgb_callback(self, msg): + """Handle incoming RGB image.""" + try: + self.latest_rgb = self.bridge.imgmsg_to_cv2(msg, "bgr8") + except Exception as e: + print(f"RGB conversion error: {e}") + + def depth_callback(self, msg): + """Handle incoming depth image.""" + try: + self.latest_depth = self.bridge.imgmsg_to_cv2(msg, "bgr8") + self.frame_count += 1 + except Exception as e: + print(f"Depth conversion error: {e}") + + def calculate_fps(self): + """Calculate current FPS.""" + elapsed = time.time() - self.start_time + if elapsed > 1.0: + self.last_fps = self.frame_count / elapsed + self.frame_count = 0 + self.start_time = time.time() + return self.last_fps + + def create_display(self): + """Create side-by-side display image.""" + if self.latest_rgb is None and self.latest_depth is None: + # Show waiting message + display = np.zeros((480, 1280, 3), dtype=np.uint8) + cv2.putText(display, "Waiting for camera and depth data...", + (400, 240), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) + return display + + # Target size for each panel + panel_width, panel_height = 640, 480 + + # Process RGB + if self.latest_rgb is not None: + rgb = cv2.resize(self.latest_rgb, (panel_width, panel_height)) + else: + rgb = np.zeros((panel_height, panel_width, 3), dtype=np.uint8) + cv2.putText(rgb, "No RGB", (250, 240), + cv2.FONT_HERSHEY_SIMPLEX, 1, (128, 128, 128), 2) + + # Process Depth + if self.latest_depth is not None: + depth = cv2.resize(self.latest_depth, (panel_width, panel_height)) + else: + depth = np.zeros((panel_height, panel_width, 3), dtype=np.uint8) + cv2.putText(depth, "No Depth", (230, 240), + cv2.FONT_HERSHEY_SIMPLEX, 1, (128, 128, 128), 2) + + # Add labels + cv2.putText(rgb, "Camera", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + cv2.putText(depth, "Depth (TensorRT)", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 2) + + # Combine side by side + display = np.hstack([rgb, depth]) + + # Add FPS if enabled + if self.fps_display: + fps = self.calculate_fps() + cv2.putText(display, f"FPS: {fps:.1f}", (1100, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 0), 2) + + # Add instructions + cv2.putText(display, "Q: Quit | S: Save | F: Toggle FPS", (10, 465), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (200, 200, 200), 1) + + return display + + def save_frame(self): + """Save current frame to disk.""" + timestamp = time.strftime("%Y%m%d_%H%M%S") + save_dir = Path("demo_captures") + save_dir.mkdir(exist_ok=True) + + if self.latest_rgb is not None: + cv2.imwrite(str(save_dir / f"rgb_{timestamp}.jpg"), self.latest_rgb) + if self.latest_depth is not None: + cv2.imwrite(str(save_dir / f"depth_{timestamp}.jpg"), self.latest_depth) + + print(f"Saved frames to {save_dir}/") + + def run(self): + """Main display loop.""" + print("\n" + "="*50) + print("Depth Anything 3 - Live Demo") + print("="*50) + print("Controls:") + print(" Q - Quit") + print(" S - Save current frame") + print(" F - Toggle FPS display") + print("="*50 + "\n") + + while self.running: + display = self.create_display() + cv2.imshow(self.window_name, display) + + key = cv2.waitKey(30) & 0xFF + if key == ord('q'): + self.running = False + elif key == ord('s'): + self.save_frame() + elif key == ord('f'): + self.fps_display = not self.fps_display + + cv2.destroyAllWindows() + + +def check_trt_service(): + """Check if the TRT inference service is running (started by run_demo.sh on host).""" + shared_dir = Path("/tmp/da3_shared") + status_file = shared_dir / "status" + + # Check if service is available via status file + if status_file.exists(): + status = status_file.read_text().strip() + if status.startswith("ready") or status.startswith("complete"): + print("[OK] TRT inference service is running") + return True + + print("[WARN] TRT inference service not detected") + print(" Make sure to run: bash scripts/run_demo.sh") + print(" Or start manually: python3 scripts/trt_inference_service.py --engine ") + return False + + +def main(): + """Main entry point.""" + def signal_handler(signum, frame): + print("\nShutting down...") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Check TRT service (should be started by run_demo.sh on host) + check_trt_service() + + if not ROS2_AVAILABLE: + print("\n[ERROR] ROS2 is required for this demo.") + print("Run inside the Docker container:") + print(" docker exec -it da3_ros2_jetson bash") + print(" python3 /ros2_ws/src/depth_anything_3_ros2/scripts/demo_depth_viewer.py") + sys.exit(1) + + # Initialize ROS2 + rclpy.init() + + # Create viewer + viewer = DepthViewer() + + # Create ROS2 node + node = rclpy.create_node('depth_viewer') + + qos = QoSProfile( + reliability=ReliabilityPolicy.BEST_EFFORT, + history=HistoryPolicy.KEEP_LAST, + depth=1 + ) + + # Subscribe to topics + node.create_subscription( + Image, '/camera/image_raw', viewer.rgb_callback, qos) + node.create_subscription( + Image, '/depth_anything_3/depth_colored', viewer.depth_callback, qos) + + print("[OK] Subscribed to camera and depth topics") + print("[...] Waiting for data (make sure camera and depth nodes are running)...\n") + + # Spin in background thread + import threading + spin_thread = threading.Thread(target=rclpy.spin, args=(node,), daemon=True) + spin_thread.start() + + # Run viewer + try: + viewer.run() + finally: + node.destroy_node() + rclpy.shutdown() + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy_jetson.sh b/scripts/deploy_jetson.sh deleted file mode 100644 index 3428723..0000000 --- a/scripts/deploy_jetson.sh +++ /dev/null @@ -1,322 +0,0 @@ -#!/bin/bash -# Depth Anything V3 - Jetson Deployment Script -# Requires: JetPack 6.2+ (L4T R36.4.x) with TensorRT 10.3 -# -# Usage: bash scripts/deploy_jetson.sh [options] -# -# Options: -# --build-only Build engine only, don't start container -# --run-only Skip engine build, just start container -# --host-trt Use host-container split architecture (recommended) -# Runs TRT inference on host, ROS2 in container -# -# Architecture (--host-trt): -# [Container: ROS2 Node] <-- /tmp/da3_shared --> [Host: TRT Service] -# | | -# v v -# /image_raw (sub) TRT 10.3 engine -# /depth (pub) ~30 FPS inference -# -# Performance: 40 FPS @ 518x518 (7.7x speedup over PyTorch) - -set -e - -# Ensure ~/.local/bin is in PATH (pip installs CLI tools there) -export PATH="$HOME/.local/bin:$PATH" - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_DIR="$(dirname "$SCRIPT_DIR")" -cd "$REPO_DIR" - -# Paths -ONNX_DIR="models/onnx" -TRT_DIR="models/tensorrt" -ONNX_MODEL="$ONNX_DIR/da3-small-embedded.onnx" -TRT_ENGINE="$TRT_DIR/da3-small-fp16.engine" -TRTEXEC="/usr/src/tensorrt/bin/trtexec" -SHARED_DIR="/tmp/da3_shared" -TRT_SERVICE_PID="" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -CYAN='\033[0;36m' -NC='\033[0m' - -echo "========================================" -echo "Depth Anything V3 - Jetson Deployment" -echo "========================================" -echo "" - -# Parse arguments -BUILD_ONLY=false -RUN_ONLY=false -HOST_TRT=false -while [[ $# -gt 0 ]]; do - case $1 in - --build-only) BUILD_ONLY=true; shift ;; - --run-only) RUN_ONLY=true; shift ;; - --host-trt) HOST_TRT=true; shift ;; - -h|--help) - echo "Usage: $0 [--build-only|--run-only|--host-trt]" - echo "" - echo "Options:" - echo " --build-only Build TRT engine only" - echo " --run-only Skip engine build, start container" - echo " --host-trt Use host-container split (recommended)" - exit 0 - ;; - *) echo "Unknown option: $1"; exit 1 ;; - esac -done - -# Cleanup function -cleanup() { - if [ -n "$TRT_SERVICE_PID" ] && kill -0 "$TRT_SERVICE_PID" 2>/dev/null; then - echo "" - echo -e "${YELLOW}Stopping TRT inference service...${NC}" - kill "$TRT_SERVICE_PID" 2>/dev/null || true - wait "$TRT_SERVICE_PID" 2>/dev/null || true - fi -} -trap cleanup EXIT - -# Step 1: Verify TensorRT 10.3 -echo -e "${YELLOW}[1/5] Checking TensorRT version...${NC}" -if [ ! -f "$TRTEXEC" ]; then - echo -e "${RED}ERROR: trtexec not found at $TRTEXEC${NC}" - echo " JetPack 6.2+ required. Current system missing TensorRT." - exit 1 -fi - -TRT_VERSION=$($TRTEXEC --help 2>&1 | grep -oP 'TensorRT v\K[0-9]+' | head -1) -if [ "$TRT_VERSION" -lt 10 ]; then - echo -e "${RED}ERROR: TensorRT $TRT_VERSION found, but 10.3+ required${NC}" - echo " DA3 uses DINOv2 backbone which requires TRT 10.x" - exit 1 -fi -echo -e "${GREEN} TensorRT 10.x detected${NC}" - -# Step 2: Check/Download ONNX model -echo -e "${YELLOW}[2/5] Checking ONNX model...${NC}" -mkdir -p "$ONNX_DIR" "$TRT_DIR" - -if [ ! -f "$ONNX_MODEL" ]; then - echo " Downloading from HuggingFace..." - - # Auto-install dependencies if not available - python3 -c "import huggingface_hub" 2>/dev/null || { - echo " Installing huggingface_hub..." - pip3 install huggingface_hub 2>&1 | tail -1 - } - - python3 -c "import onnx" 2>/dev/null || { - echo " Installing onnx..." - pip3 install onnx 2>&1 | tail -1 - } - - # Download and embed model using Python API (more reliable than CLI) - python3 << 'PYEOF' -import os -import sys - -try: - from huggingface_hub import snapshot_download - import onnx - - onnx_dir = "models/onnx" - hf_download_dir = os.path.join(onnx_dir, "hf-download") - output_model = os.path.join(onnx_dir, "da3-small-embedded.onnx") - - print(" Downloading ONNX model...") - snapshot_download( - repo_id="onnx-community/depth-anything-v3-small", - local_dir=hf_download_dir, - allow_patterns=["*.onnx", "*.onnx_data"] - ) - - print(" Embedding weights into single ONNX file...") - model_path = os.path.join(hf_download_dir, "onnx", "model.onnx") - model = onnx.load(model_path) - onnx.save(model, output_model, save_as_external_data=False) - print(f" Created: {output_model}") - -except Exception as e: - print(f"ERROR: {e}", file=sys.stderr) - sys.exit(1) -PYEOF - - if [ $? -ne 0 ]; then - echo -e "${RED}ERROR: Failed to download ONNX model${NC}" - exit 1 - fi -fi - -ONNX_SIZE=$(du -h "$ONNX_MODEL" | cut -f1) -echo -e "${GREEN} ONNX model ready: $ONNX_MODEL ($ONNX_SIZE)${NC}" - -# Step 3: Build TensorRT engine -if [ "$RUN_ONLY" = false ]; then - echo -e "${YELLOW}[3/5] Building TensorRT FP16 engine...${NC}" - - if [ -f "$TRT_ENGINE" ]; then - ENGINE_SIZE=$(du -h "$TRT_ENGINE" | cut -f1) - echo " Engine exists: $TRT_ENGINE ($ENGINE_SIZE)" - read -p " Rebuild? [y/N] " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo " Skipping rebuild" - else - rm -f "$TRT_ENGINE" - fi - fi - - if [ ! -f "$TRT_ENGINE" ]; then - echo " This takes ~2 minutes..." - echo "" - - $TRTEXEC \ - --onnx="$ONNX_MODEL" \ - --saveEngine="$TRT_ENGINE" \ - --fp16 \ - --memPoolSize=workspace:2048MiB \ - --optShapes=pixel_values:1x1x3x518x518 \ - 2>&1 | tee /tmp/trtexec_build.log | grep -E "(Building|Serializing|SUCCESS|ERROR|Throughput)" - - if [ ! -f "$TRT_ENGINE" ]; then - echo -e "${RED}ERROR: Engine build failed${NC}" - echo " Check /tmp/trtexec_build.log for details" - exit 1 - fi - - ENGINE_SIZE=$(du -h "$TRT_ENGINE" | cut -f1) - echo -e "${GREEN} Engine built: $TRT_ENGINE ($ENGINE_SIZE)${NC}" - fi -else - echo -e "${YELLOW}[3/5] Skipping engine build (--run-only)${NC}" -fi - -if [ "$BUILD_ONLY" = true ]; then - echo "" - echo -e "${GREEN}Build complete. Run with: bash scripts/deploy_jetson.sh --run-only${NC}" - exit 0 -fi - -# Step 4: Setup shared memory directory and start host TRT service -if [ "$HOST_TRT" = true ]; then - echo -e "${YELLOW}[4/5] Starting host TRT inference service...${NC}" - - # Create shared directory with proper permissions - mkdir -p "$SHARED_DIR" - chmod 777 "$SHARED_DIR" - - # Check for required Python packages - echo " Checking Python dependencies..." - - # Check TensorRT (should be available via JetPack) - if ! python3 -c "import tensorrt" 2>/dev/null; then - echo -e "${RED}ERROR: TensorRT Python bindings not found${NC}" - echo " TensorRT should be available via JetPack 6.2+" - echo " Try: sudo apt install python3-libnvinfer python3-libnvinfer-dev" - exit 1 - fi - - # Auto-install numpy if missing - if ! python3 -c "import numpy" 2>/dev/null; then - echo " Installing numpy..." - if pip3 install numpy 2>&1 | tail -2; then - echo -e "${GREEN} numpy installed successfully${NC}" - else - echo -e "${RED}ERROR: Failed to install numpy${NC}" - exit 1 - fi - fi - - # Auto-install pycuda if missing - if ! python3 -c "import pycuda.driver" 2>/dev/null; then - echo " Installing pycuda (required for TRT inference)..." - if pip3 install pycuda 2>&1 | tail -3; then - echo -e "${GREEN} pycuda installed successfully${NC}" - else - echo -e "${RED}ERROR: Failed to install pycuda${NC}" - echo " Try manually: pip3 install pycuda" - exit 1 - fi - - # Verify installation - if ! python3 -c "import pycuda.driver" 2>/dev/null; then - echo -e "${RED}ERROR: pycuda installed but import failed${NC}" - echo " This may indicate a CUDA configuration issue" - exit 1 - fi - fi - - echo -e "${GREEN} TensorRT, numpy, and pycuda ready${NC}" - - # Start TRT inference service in background - echo " Starting inference service..." - python3 "$SCRIPT_DIR/trt_inference_service.py" \ - --engine "$TRT_ENGINE" \ - --poll-interval 0.001 \ - > /tmp/trt_service.log 2>&1 & - TRT_SERVICE_PID=$! - - # Wait for service to initialize - sleep 2 - if ! kill -0 "$TRT_SERVICE_PID" 2>/dev/null; then - echo -e "${RED}ERROR: TRT service failed to start${NC}" - echo " Check /tmp/trt_service.log for details" - cat /tmp/trt_service.log - exit 1 - fi - - # Verify service is ready - if [ -f "$SHARED_DIR/status" ]; then - STATUS=$(cat "$SHARED_DIR/status") - echo -e "${GREEN} TRT service running (PID: $TRT_SERVICE_PID, status: $STATUS)${NC}" - else - echo -e "${YELLOW} TRT service starting...${NC}" - fi -else - echo -e "${YELLOW}[4/5] Skipping host TRT service (use --host-trt to enable)${NC}" -fi - -# Step 5: Start container -echo -e "${YELLOW}[5/5] Starting Docker container...${NC}" - -# Build image if needed -if ! docker images | grep -q "depth_anything_3_ros2.*jetson"; then - echo " Building Docker image (first time only, ~20 min)..." - docker compose build depth-anything-3-jetson -fi - -echo "" -echo "========================================" -echo "Starting Depth Anything V3 container" -echo "========================================" -echo "" -echo "Engine: $TRT_ENGINE" -echo "Resolution: 518x518 FP16" - -if [ "$HOST_TRT" = true ]; then - echo -e "Mode: ${CYAN}Host-Container Split (TRT on host)${NC}" - echo "Expected: ~40 FPS @ 518x518 (93 FPS @ 308x308)" - echo "" - echo "TRT Service: PID $TRT_SERVICE_PID (log: /tmp/trt_service.log)" - echo "Shared Dir: $SHARED_DIR" -else - echo "Mode: Container-only (PyTorch fallback)" - echo "Expected: ~5 FPS (TRT unavailable in container)" -fi -echo "" -echo "Inside container, run:" -echo " ros2 launch depth_anything_3_ros2 depth_anything_3.launch.py \\" -echo " image_topic:=/camera/image_raw" -echo "" - -# Export environment for docker-compose -export DA3_ENGINE_PATH="$TRT_ENGINE" -export DA3_HOST_TRT="$HOST_TRT" - -docker compose up depth-anything-3-jetson diff --git a/scripts/trt_inference_service.py b/scripts/trt_inference_service.py index 4f70182..be2e997 100644 --- a/scripts/trt_inference_service.py +++ b/scripts/trt_inference_service.py @@ -261,21 +261,38 @@ def process_request(self) -> bool: return False try: - # Read request timestamp - request_time = float(REQUEST_PATH.read_text().strip()) + # Read request timestamp with race condition handling + # The file may exist but be empty if caught during write + request_text = REQUEST_PATH.read_text().strip() + if not request_text: + # File exists but empty - caught during write, skip this iteration + return False + try: + request_time = float(request_text) + except ValueError: + # Invalid content, skip this iteration + return False - # Load input tensor + # Load input tensor with validation if not INPUT_PATH.exists(): return False input_tensor = np.load(INPUT_PATH) + # Validate input shape matches expected + expected_size = int(np.prod(self.engine.get_input_shape())) + if input_tensor.size != expected_size: + raise ValueError( + f"Input size mismatch: got {input_tensor.size}, " + f"expected {expected_size} for shape {self.engine.get_input_shape()}" + ) + # Run inference start = time.perf_counter() outputs = self.engine.infer(input_tensor) inference_time = time.perf_counter() - start - # Save output (primary depth output) + # Save output (primary depth output) - atomic write # Find the depth output tensor depth_key = None for key in outputs: @@ -285,7 +302,13 @@ def process_request(self) -> bool: if depth_key is None: depth_key = list(outputs.keys())[0] - np.save(OUTPUT_PATH, outputs[depth_key]) + # Atomic write: temp file + fsync + rename + temp_output = OUTPUT_PATH.parent / "output_tmp.npy" + with open(temp_output, 'wb') as f: + np.save(f, outputs[depth_key], allow_pickle=False) + f.flush() + os.fsync(f.fileno()) + temp_output.replace(OUTPUT_PATH) # Update stats self.stats["frames"] += 1