YOLO Rust Deployment Guide

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# Install Rust (recommended 1.85+, Edition 2024)
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh

# Switch to stable edition and enable 2024 Edition
rustup default stable
rustup update

# Verify
rustc --version  # rustc 1.85.0+
cargo --version

1
2
3
4
5
6
7

[dependencies]
ort = { version = "2.0.0-rc.12", features = [
    "download-binaries",  # Auto-download ONNX Runtime binaries
    "ndarray",            # ndarray integration
    "cuda",               # CUDA support (optional)
    "tensorrt",           # TensorRT support (optional)
] }

1
2
3
4
5
6
7
8

# Ubuntu/Debian
sudo apt install libopencv-dev clang libclang-dev

# macOS
brew install opencv

# Windows
# Use vcpkg or precompiled packages

1
2

[dependencies]
opencv = { version = "0.94.4", features = ["opencv-480"] }

1
2
3
4
5
6
7
8
9

from ultralytics import YOLO

# ✅ YOLO26 recommended, simplest post-processing in Rust
model = YOLO("yolo26n.pt")
model.export(format="onnx", simplify=True, opset=17)

# YOLO11/v8 also supported
model = YOLO("yolo11n.pt")
model.export(format="onnx", simplify=True)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

use ort::{session::Session, GraphOptimizationLevel, Environment};
use std::path::Path;

const MODEL_PATH: &str = "yolo26n.onnx";
const INPUT_SIZE: usize = 640;
const NUM_CLASSES: usize = 80;

fn main() -> ort::Result<()> {
    // Initialize global environment (only once)
    let environment = Environment::builder()
        .with_name("yolo-inference")
        .build()?;

    // ========== Create inference session (core configuration) ==========
    let session = Session::builder()?
        // Optimization level: Level3 = maximum optimization
        .with_optimization_level(GraphOptimizationLevel::Level3)?
        // CPU thread configuration
        .with_intra_threads(8)?
        .with_inter_threads(2)?
        // ========== GPU acceleration (optional) ==========
        // .with_execution_providers([
        //     ExecutionProvider::CUDA(Default::default()),
        //     ExecutionProvider::TensorRT(Default::default()),
        // ])?
        // Load model
        .commit_from_file(MODEL_PATH)?;

    println!("✅ Model loaded successfully!");
    println!("   Input: {:?}", session.inputs[0]);
    println!("   Output: {:?}", session.outputs[0]);

    Ok(())
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196

use ort::{session::Session, GraphOptimizationLevel, value::Value, Tensor};
use image::{imageops::FilterType, GenericImageView, Pixel};
use ndarray::{s, Array, Array4, Axis};
use std::path::Path;
use std::time::Instant;

// ========== Configuration ==========
const MODEL_PATH: &str = "yolo26n.onnx";
const INPUT_SIZE: usize = 640;
const CONF_THRESH: f32 = 0.25;
const IOU_THRESH: f32 = 0.45;

#[derive(Debug, Clone)]
struct Detection {
    x1: f32,
    y1: f32,
    x2: f32,
    y2: f32,
    confidence: f32,
    class_id: usize,
    class_name: &'static str,
}

// COCO 80 class names
const CLASS_NAMES: [&str; 80] = [
    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat",
    "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat",
    "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack",
    "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball",
    "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket",
    "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
    "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake",
    "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop",
    "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
    "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier",
    "toothbrush",
];

fn main() -> anyhow::Result<()> {
    // 1. Create session
    let session = Session::builder()?
        .with_optimization_level(GraphOptimizationLevel::Level3)?
        .with_intra_threads(8)?
        .commit_from_file(MODEL_PATH)?;

    println!("✅ YOLO26 Rust inference ready");

    // 2. Load and preprocess image
    let img_path = "test.jpg";
    let img = image::open(img_path)?;
    let (orig_w, orig_h) = (img.width() as f32, img.height() as f32);

    let start = Instant::now();

    // 3. Preprocessing
    let input = preprocess(&img);

    // 4. Inference
    let input_value = Tensor::from_array(input)?;
    let outputs = session.run(ort::inputs!["images" => input_value]?)?;

    // 5. Post-processing
    let output = outputs["output0"].try_extract_tensor::<f32>()?;
    let detections = postprocess(output.view(), orig_w, orig_h);

    let elapsed = start.elapsed();

    // 6. Output results
    println!("\n📊 Detection completed, elapsed: {:?}", elapsed);
    println!("Total detections: {}\n", detections.len());

    for (i, det) in detections.iter().enumerate() {
        println!(
            "{:2}. {:<15} Confidence: {:.3}  Location: [{:.0}, {:.0}, {:.0}, {:.0}]",
            i + 1, det.class_name, det.confidence, det.x1, det.y1, det.x2, det.y2
        );
    }

    Ok(())
}

/// Image preprocessing: Resize + normalization + NCHW format
fn preprocess(img: &image::DynamicImage) -> Array4<f32> {
    // Resize to 640x640
    let resized = img.resize_exact(
        INPUT_SIZE as u32,
        INPUT_SIZE as u32,
        FilterType::CatmullRom,
    );

    // Create NCHW format array [1, 3, H, W]
    let mut input = Array::zeros((1, 3, INPUT_SIZE, INPUT_SIZE));

    for y in 0..INPUT_SIZE {
        for x in 0..INPUT_SIZE {
            let pixel = resized.get_pixel(x as u32, y as u32).to_rgb();
            input[[0, 0, y, x]] = pixel[0] as f32 / 255.0;
            input[[0, 1, y, x]] = pixel[1] as f32 / 255.0;
            input[[0, 2, y, x]] = pixel[2] as f32 / 255.0;
        }
    }

    input
}

/// Post-processing: Parse output + NMS
fn postprocess(
    output: ndarray::ArrayView3<'_, f32>,
    orig_w: f32,
    orig_h: f32,
) -> Vec<Detection> {
    let scale_x = orig_w / INPUT_SIZE as f32;
    let scale_y = orig_h / INPUT_SIZE as f32;

    let mut detections = Vec::new();

    // Output shape [1, 84, 8400] -> transpose to [8400, 84]
    let output = output.permuted_axes((1, 2, 0)).remove_axis(Axis(2));

    for i in 0..8400 {
        let row = output.slice(s![i, ..]);

        // Find maximum confidence
        let (class_id, confidence) = (4..84)
            .map(|c| (c - 4, row[c]))
            .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap())
            .unwrap();

        if confidence < CONF_THRESH {
            continue;
        }

        // Parse coordinates cx, cy, w, h
        let cx = row[0] * scale_x;
        let cy = row[1] * scale_y;
        let w = row[2] * scale_x;
        let h = row[3] * scale_y;

        detections.push(Detection {
            x1: cx - w / 2.0,
            y1: cy - h / 2.0,
            x2: cx + w / 2.0,
            y2: cy + h / 2.0,
            confidence,
            class_id,
            class_name: CLASS_NAMES[class_id],
        });
    }

    // NMS non-maximum suppression
    nms(&mut detections, IOU_THRESH)
}

/// Non-maximum suppression (efficient Rust implementation)
fn nms(detections: &mut Vec<Detection>, iou_thresh: f32) -> Vec<Detection> {
    // Sort by confidence descending
    detections.sort_by(|a, b| b.confidence.partial_cmp(&a.confidence).unwrap());

    let mut keep = Vec::new();
    let mut suppressed = vec![false; detections.len()];

    for i in 0..detections.len() {
        if suppressed[i] {
            continue;
        }
        keep.push(detections[i].clone());

        for j in (i + 1)..detections.len() {
            if suppressed[j] {
                continue;
            }
            if calculate_iou(&detections[i], &detections[j]) > iou_thresh {
                suppressed[j] = true;
            }
        }
    }

    keep
}

fn calculate_iou(a: &Detection, b: &Detection) -> f32 {
    let x1 = a.x1.max(b.x1);
    let y1 = a.y1.max(b.y1);
    let x2 = a.x2.min(b.x2);
    let y2 = a.y2.min(b.y2);

    if x2 <= x1 || y2 <= y1 {
        return 0.0;
    }

    let intersection = (x2 - x1) * (y2 - y1);
    let area_a = (a.x2 - a.x1) * (a.y2 - a.y1);
    let area_b = (b.x2 - b.x1) * (b.y2 - b.y1);

    intersection / (area_a + area_b - intersection)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

fn process_video_frames(session: &Session, frames: &[image::DynamicImage]) -> anyhow::Result<Vec<Vec<Detection>>> {
    let mut all_detections = Vec::new();
    let start = Instant::now();
    for (i, frame) in frames.iter().enumerate() {
        let input = preprocess(frame);
        let outputs = session.run(ort::inputs!["images" => Tensor::from_array(input)?]?)?;
        let output = outputs["output0"].try_extract_tensor::<f32>()?;
        let detections = postprocess(output.view(), frame.width() as f32, frame.height() as f32);
        let fps = (i + 1) as f64 / start.elapsed().as_secs_f64();
        println!("Frame {}: {} detections, FPS: {:.1}", i, detections.len(), fps);
        all_detections.push(detections);
    }
    Ok(all_detections)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

fn draw_detections(frame: &image::DynamicImage, detections: &[Detection]) -> image::RgbaImage {
    let mut canvas = frame.to_rgba8();
    for det in detections {
        for x in (det.x1 as u32)..=(det.x2 as u32) {
            canvas.put_pixel(x, det.y1 as u32, image::Rgba([255, 0, 0, 255]));
            canvas.put_pixel(x, det.y2 as u32, image::Rgba([255, 0, 0, 255]));
        }
        for y in (det.y1 as u32)..=(det.y2 as u32) {
            canvas.put_pixel(det.x1 as u32, y, image::Rgba([255, 0, 0, 255]));
            canvas.put_pixel(det.x2 as u32, y, image::Rgba([255, 0, 0, 255]));
        }
    }
    canvas
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

#[cfg(test)]
mod tests {
    use super::*;
    const TEST_IMAGE: &str = "tests/test_car.jpg";

    #[test]
    fn test_preprocess_shape() {
        let img = image::open(TEST_IMAGE).unwrap();
        let input = preprocess(&img);
        assert_eq!(input.shape(), &[1, 3, 640, 640]);
    }

    #[test]
    fn test_nms_removes_overlaps() {
        let det = |x1, y1, x2, y2, conf| Detection { x1, y1, x2, y2, confidence: conf, class_id: 0, class_name: "person" };
        let dets = vec![det(10.,10.,100.,100.,0.9), det(15.,15.,95.,95.,0.8)];
        let result = nms(&mut dets.clone(), 0.5);
        assert_eq!(result.len(), 1);
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

fn benchmark(session: &Session, input: &Tensor, n: usize) {
    for _ in 0..5 { session.run(ort::inputs!["images" => input].unwrap()).unwrap(); }
    let mut times: Vec<Duration> = (0..n).map(|_| {
        let s = Instant::now();
        session.run(ort::inputs!["images" => input].unwrap()).unwrap();
        s.elapsed()
    }).collect();
    times.sort();
    let sum: Duration = times.iter().sum();
    println!("Avg: {:?} | P50: {:?} | P95: {:?} | P99: {:?}",
        sum / n as u32, times[n * 50 / 100], times[n * 95 / 100], times[n * 99 / 100]);
}

1
2
3
4
5

[profile.release]
opt-level = 3        # Highest optimization level
lto = "fat"          # Link-time optimization
codegen-units = 1    # Single code unit (slow compile, fast run)
panic = "abort"      # Remove panic unwinding

1
2
3
4
5
6

let session = Session::builder()?
    .with_optimization_level(GraphOptimizationLevel::Level3)?
    .with_intra_threads(num_cpus::get())?  // = CPU core count
    .with_memory_pattern(true)?             // Memory pattern optimization
    .with_cpu_mem_arena(true)?              // CPU memory arena
    .commit_from_file(MODEL_PATH)?;

1
2
3

// Use Rust SIMD to accelerate normalization, 2-3x speedup
#![feature(portable_simd)]
use std::simd::f32x8;

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

use axum::{routing::post, Json, Router};
use std::sync::Arc;
use tokio::sync::Semaphore;

// Session pool: one independent session per worker
struct AppState {
    sessions: Vec<Arc<Session>>,
    semaphore: Semaphore,
}

#[tokio::main]
async fn main() {
    let num_workers = num_cpus::get();
    let mut sessions = Vec::new();
    
    // Pre-create multiple inference sessions
    for _ in 0..num_workers {
        let session = create_session().unwrap();
        sessions.push(Arc::new(session));
    }

    let state = Arc::new(AppState {
        sessions,
        semaphore: Semaphore::new(num_workers),
    });

    let app = Router::new()
        .route("/detect", post(detect_handler))
        .with_state(state);

    axum::Server::bind(&"0.0.0.0:8080".parse().unwrap())
        .serve(app.into_make_service())
        .await
        .unwrap();
}

async fn detect_handler(
    State(state): State<Arc<AppState>>,
    Json(req): Json<DetectRequest>,
) -> Json<DetectResponse> {
    // Get permit, control concurrency
    let _permit = state.semaphore.acquire().await.unwrap();
    let worker_id = _permit.id() % state.sessions.len();
    
    // Use corresponding worker's session for inference
    let result = infer(&state.sessions[worker_id], &req.image).await;
    
    Json(result)
}

1
2
3
4
5
6
7

# Install cross-compilation toolchain
rustup target add aarch64-unknown-linux-gnu

# Compile
cargo build --release --target aarch64-unknown-linux-gnu

# Binary size: ~5MB (statically linked)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

FROM rust:1.85 as builder
WORKDIR /app
COPY . .
RUN cargo build --release

FROM debian:bookworm-slim
COPY --from=builder /app/target/release/yolo-rust /usr/local/bin/
RUN apt update && apt install -y libgcc1 && rm -rf /var/lib/apt/lists/*
EXPOSE 8080
CMD ["yolo-rust"]
# Image size: ~80MB

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

[package]
name = "yolo-rust"
version = "0.1.0"
edition = "2024"
rust-version = "1.85"

[dependencies]
# Core inference
ort = { version = "2.0.0-rc.12", features = [
    "download-binaries",
    "ndarray",
    "fetch-models",
] }

# Image processing
image = { version = "0.25", features = ["jpeg", "png"] }
ndarray = "0.16"

# Web services (optional)
axum = "0.7"
tokio = { version = "1.0", features = ["full"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

# Tools
anyhow = "1.0"
num_cpus = "1.0"

[profile.release]
opt-level = 3
lto = "fat"
codegen-units = 1
panic = "abort"
strip = "debuginfo"

1
2
3
4
5
6
7
8

# Development run
cargo run

# Release run (best performance)
cargo run --release

# Performance profiling
cargo flamegraph --bin yolo-rust