From 735e0cecdf0ccd58aaa03418c7ad3ba6d87f732e Mon Sep 17 00:00:00 2001 From: Jamjamjon <51357717+jamjamjon@users.noreply.github.com> Date: Tue, 10 Jun 2025 16:07:10 +0800 Subject: [PATCH] Add RTMPose, RTMW, DWPose models (#110) --- README.md | 7 +- examples/dwpose/README.md | 9 + examples/dwpose/main.rs | 97 +++++++++ examples/rtmo/main.rs | 4 +- examples/rtmpose/README.md | 11 ++ examples/rtmpose/main.rs | 122 ++++++++++++ examples/rtmw/README.md | 9 + examples/rtmw/main.rs | 97 +++++++++ src/core/names.rs | 238 ++++++++++++++++++++++ src/core/processor.rs | 1 - src/models/dwpose/README.md | 9 + src/models/dwpose/config.rs | 22 +++ src/models/dwpose/mod.rs | 1 + src/models/mod.rs | 4 + src/models/rtmo/config.rs | 1 + src/models/rtmo/impl.rs | 18 +- src/models/rtmpose/README.md | 9 + src/models/rtmpose/config.rs | 79 ++++++++ src/models/rtmpose/impl.rs | 369 +++++++++++++++++++++++++++++++++++ src/models/rtmpose/mod.rs | 4 + src/models/rtmw/README.md | 9 + src/models/rtmw/config.rs | 24 +++ src/models/rtmw/mod.rs | 1 + src/results/keypoint.rs | 6 + src/results/skeleton.rs | 294 ++++++++++++++++++++++++++++ 25 files changed, 1436 insertions(+), 9 deletions(-) create mode 100644 examples/dwpose/README.md create mode 100644 examples/dwpose/main.rs create mode 100644 examples/rtmpose/README.md create mode 100644 examples/rtmpose/main.rs create mode 100644 examples/rtmw/README.md create mode 100644 examples/rtmw/main.rs create mode 100644 src/models/dwpose/README.md create mode 100644 src/models/dwpose/config.rs create mode 100644 src/models/dwpose/mod.rs create mode 100644 src/models/rtmpose/README.md create mode 100644 src/models/rtmpose/config.rs create mode 100644 src/models/rtmpose/impl.rs create mode 100644 src/models/rtmpose/mod.rs create mode 100644 src/models/rtmw/README.md create mode 100644 src/models/rtmw/config.rs create mode 100644 src/models/rtmw/mod.rs diff --git a/README.md b/README.md index 5888792..6d165c6 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,10 @@ ## 🚀 Quick Start ```bash # CPU -cargo run -r --example yolo # YOLOv8-n detect by default +cargo run -r --example yolo -- --task detect --ver 8 --scale n --dtype fp16 # q8, q4, q4f16 # NVIDIA CUDA -cargo run -r -F cuda --example yolo -- --device cuda:0 +cargo run -r -F cuda --example yolo -- --device cuda:0 # YOLOv8-n detect by default # NVIDIA TensorRT cargo run -r -F tensorrt --example yolo -- --device tensorrt:0 @@ -79,6 +79,9 @@ usls = "latest-version" | [DocLayout-YOLO](https://github.com/opendatalab/DocLayout-YOLO) | Object Detection | [demo](examples/picodet-layout) | | [D-FINE](https://github.com/manhbd-22022602/D-FINE) | Object Detection | [demo](examples/d-fine) | | [DEIM](https://github.com/ShihuaHuang95/DEIM) | Object Detection | [demo](examples/deim) | +| [RTMPose](https://github.com/open-mmlab/mmpose/tree/dev-1.x/projects/rtmpose) | Keypoint Detection | [demo](examples/rtmpose) | +| [DWPose](https://github.com/IDEA-Research/DWPose) | Keypoint Detection | [demo](examples/dwpose) | +| [RTMW](https://arxiv.org/abs/2407.08634) | Keypoint Detection | [demo](examples/rtmw) | | [RTMO](https://github.com/open-mmlab/mmpose/tree/main/projects/rtmo) | Keypoint Detection | [demo](examples/rtmo) | | [SAM](https://github.com/facebookresearch/segment-anything) | Segment Anything | [demo](examples/sam) | | [SAM2](https://github.com/facebookresearch/segment-anything-2) | Segment Anything | [demo](examples/sam) | diff --git a/examples/dwpose/README.md b/examples/dwpose/README.md new file mode 100644 index 0000000..8a99462 --- /dev/null +++ b/examples/dwpose/README.md @@ -0,0 +1,9 @@ +## Quick Start + +```shell +cargo run -r --example dwpose +``` + +## Results + +![](https://github.com/jamjamjon/assets/releases/download/rtmpose/demo-dwpose.jpg) diff --git a/examples/dwpose/main.rs b/examples/dwpose/main.rs new file mode 100644 index 0000000..60fc7b8 --- /dev/null +++ b/examples/dwpose/main.rs @@ -0,0 +1,97 @@ +use anyhow::Result; +use usls::{ + models::{RTMPose, YOLO}, + Annotator, Config, DataLoader, Scale, Style, SKELETON_COCO_65, SKELETON_COLOR_COCO_65, +}; + +#[derive(argh::FromArgs)] +/// Example +struct Args { + /// source: image, image folder, video stream + #[argh(option, default = "String::from(\"./assets/bus.jpg\")")] + source: String, + + /// device: cuda:0, cpu:0, ... + #[argh(option, default = "String::from(\"cpu:0\")")] + device: String, + + /// scale: t, s, m, l + #[argh(option, default = "String::from(\"m\")")] + scale: String, + + /// dtype: fp16, q8, q4, q4f16, ... + #[argh(option, default = "String::from(\"auto\")")] + dtype: String, +} + +fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_timer(tracing_subscriber::fmt::time::ChronoLocal::rfc_3339()) + .init(); + let args: Args = argh::from_env(); + + // build YOLOv8 + let yolo_config = Config::yolo_detect() + .with_scale(Scale::N) + .with_version(8.into()) + .with_model_device(args.device.parse()?) + .retain_classes(&[0]) // keep person class only + .with_class_confs(&[0.5]) + .commit()?; + let mut yolo = YOLO::new(yolo_config)?; + + // build RTMPose + let config = match args.scale.as_str() { + "t" => Config::dwpose_133_t(), + "s" => Config::dwpose_133_s(), + "m" => Config::dwpose_133_m(), + "l" => Config::dwpose_133_l(), + _ => todo!(), + } + .with_model_dtype(args.dtype.parse()?) + .with_model_device(args.device.parse()?) + .commit()?; + let mut rtmpose = RTMPose::new(config)?; + + // build annotator + let annotator = Annotator::default() + .with_hbb_style(Style::hbb().with_draw_fill(true)) + .with_keypoint_style( + Style::keypoint() + .with_radius(2) + .with_skeleton((SKELETON_COCO_65, SKELETON_COLOR_COCO_65).into()) + .show_id(false) + .show_confidence(false) + .show_name(false), + ); + + // build dataloader + let dl = DataLoader::new(&args.source)?.with_batch(1).build()?; + + // iterate + for xs in &dl { + // YOLO infer + let ys_det = yolo.forward(&xs)?; + + // RTMPose infer + for (x, y_det) in xs.iter().zip(ys_det.iter()) { + let y = rtmpose.forward(x, y_det.hbbs())?; + + // Annotate + annotator.annotate(x, &y)?.save(format!( + "{}.jpg", + usls::Dir::Current + .base_dir_with_subs(&["runs", rtmpose.spec()])? + .join(usls::timestamp(None)) + .display(), + ))? + } + } + + // summary + yolo.summary(); + rtmpose.summary(); + + Ok(()) +} diff --git a/examples/rtmo/main.rs b/examples/rtmo/main.rs index 69ce542..4d73e9f 100644 --- a/examples/rtmo/main.rs +++ b/examples/rtmo/main.rs @@ -12,7 +12,7 @@ struct Args { #[argh(option, default = "String::from(\"cpu:0\")")] device: String, - /// scale: s, m, l + /// scale: t, s, m, l #[argh(option, default = "String::from(\"t\")")] scale: String, } @@ -52,7 +52,7 @@ fn main() -> Result<()> { .with_skeleton(SKELETON_COCO_19.into()) .show_confidence(false) .show_id(true) - .show_name(false), + .show_name(true), ); for (x, y) in xs.iter().zip(ys.iter()) { annotator.annotate(x, y)?.save(format!( diff --git a/examples/rtmpose/README.md b/examples/rtmpose/README.md new file mode 100644 index 0000000..173bbfb --- /dev/null +++ b/examples/rtmpose/README.md @@ -0,0 +1,11 @@ +## Quick Start + +```shell +cargo run -r --example rtmpose -- --is-coco true +``` + +## Results + +| rtmpose-17 | rtmpose-26 | +| :---------------: | :------------------------: | +| ![](https://github.com/jamjamjon/assets/releases/download/rtmpose/demo-rtmpose-17.jpg) | ![](https://github.com/jamjamjon/assets/releases/download/rtmpose/demo-rtmpose-26.jpg)| \ No newline at end of file diff --git a/examples/rtmpose/main.rs b/examples/rtmpose/main.rs new file mode 100644 index 0000000..2894bbf --- /dev/null +++ b/examples/rtmpose/main.rs @@ -0,0 +1,122 @@ +use anyhow::Result; +use usls::{ + models::{RTMPose, YOLO}, + Annotator, Config, DataLoader, Scale, Style, SKELETON_COCO_19, SKELETON_COLOR_COCO_19, + SKELETON_COLOR_HALPE_27, SKELETON_HALPE_27, +}; + +#[derive(argh::FromArgs)] +/// Example +struct Args { + /// source: image, image folder, video stream + #[argh(option, default = "String::from(\"./assets/bus.jpg\")")] + source: String, + + /// device: cuda:0, cpu:0, ... + #[argh(option, default = "String::from(\"cpu:0\")")] + device: String, + + /// scale: t, s, m, l, x + #[argh(option, default = "String::from(\"t\")")] + scale: String, + + /// dtype: fp16, q8, q4, q4f16, ... + #[argh(option, default = "String::from(\"auto\")")] + dtype: String, + + /// is coco 17 keypoints or halpe 26 keypoints + #[argh(option, default = "true")] + is_coco: bool, +} + +fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_timer(tracing_subscriber::fmt::time::ChronoLocal::rfc_3339()) + .init(); + let args: Args = argh::from_env(); + + // build YOLOv8 + let yolo_config = Config::yolo_detect() + .with_scale(Scale::N) + .with_version(8.into()) + .with_model_device(args.device.parse()?) + .retain_classes(&[0]) // keep person class only + .with_class_confs(&[0.5]) + .commit()?; + let mut yolo = YOLO::new(yolo_config)?; + + // build RTMPose + let config = match args.scale.as_str() { + "t" => match args.is_coco { + true => Config::rtmpose_17_t(), + false => Config::rtmpose_26_t(), + }, + "s" => match args.is_coco { + true => Config::rtmpose_17_s(), + false => Config::rtmpose_26_s(), + }, + "m" => match args.is_coco { + true => Config::rtmpose_17_m(), + false => Config::rtmpose_26_m(), + }, + "l" => match args.is_coco { + true => Config::rtmpose_17_l(), + false => Config::rtmpose_26_l(), + }, + "x" => match args.is_coco { + true => Config::rtmpose_17_x(), + false => Config::rtmpose_26_x(), + }, + _ => todo!(), + } + .with_model_dtype(args.dtype.parse()?) + .with_model_device(args.device.parse()?) + .commit()?; + let mut rtmpose = RTMPose::new(config)?; + + // build annotator + let annotator = Annotator::default() + .with_hbb_style(Style::hbb().with_draw_fill(true)) + .with_keypoint_style( + Style::keypoint() + .with_radius(4) + .with_skeleton(if args.is_coco { + (SKELETON_COCO_19, SKELETON_COLOR_COCO_19).into() + } else { + (SKELETON_HALPE_27, SKELETON_COLOR_HALPE_27).into() + }) + .show_id(false) + .show_confidence(false) + .show_name(false), + ); + + // build dataloader + let dl = DataLoader::new(&args.source)?.with_batch(1).build()?; + + // iterate + for xs in &dl { + // YOLO infer + let ys_det = yolo.forward(&xs)?; + + // RTMPose infer + for (x, y_det) in xs.iter().zip(ys_det.iter()) { + let y = rtmpose.forward(x, y_det.hbbs())?; + + // Annotate + annotator.annotate(x, &y)?.save(format!( + "{}.jpg", + usls::Dir::Current + .base_dir_with_subs(&["runs", rtmpose.spec()])? + .join(usls::timestamp(None)) + .display(), + ))? + } + } + + // summary + yolo.summary(); + rtmpose.summary(); + + Ok(()) +} diff --git a/examples/rtmw/README.md b/examples/rtmw/README.md new file mode 100644 index 0000000..42f8249 --- /dev/null +++ b/examples/rtmw/README.md @@ -0,0 +1,9 @@ +## Quick Start + +```shell +cargo run -r --example rtmw +``` + +## Result + +![](https://github.com/jamjamjon/assets/releases/download/rtmpose/demo-rtmw.jpg) diff --git a/examples/rtmw/main.rs b/examples/rtmw/main.rs new file mode 100644 index 0000000..1d3a4b7 --- /dev/null +++ b/examples/rtmw/main.rs @@ -0,0 +1,97 @@ +use anyhow::Result; +use usls::{ + models::{RTMPose, YOLO}, + Annotator, Config, DataLoader, Scale, Style, SKELETON_COCO_65, SKELETON_COLOR_COCO_65, +}; + +#[derive(argh::FromArgs)] +/// Example +struct Args { + /// source: image, image folder, video stream + #[argh(option, default = "String::from(\"./assets/bus.jpg\")")] + source: String, + + /// device: cuda:0, cpu:0, ... + #[argh(option, default = "String::from(\"cpu:0\")")] + device: String, + + /// scale: m, l, x + #[argh(option, default = "String::from(\"m\")")] + scale: String, + + /// dtype: fp16, q8, q4, q4f16, ... + #[argh(option, default = "String::from(\"auto\")")] + dtype: String, +} + +fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_timer(tracing_subscriber::fmt::time::ChronoLocal::rfc_3339()) + .init(); + let args: Args = argh::from_env(); + + // build YOLOv8 + let yolo_config = Config::yolo_detect() + .with_scale(Scale::N) + .with_version(8.into()) + .with_model_device(args.device.parse()?) + .retain_classes(&[0]) // keep person class only + .with_class_confs(&[0.5]) + .commit()?; + let mut yolo = YOLO::new(yolo_config)?; + + // build RTMPose + let config = match args.scale.as_str() { + "m" => Config::rtmw_133_m(), + "l" => Config::rtmw_133_l(), + "x" => Config::rtmw_133_x(), + _ => todo!(), + } + .with_model_dtype(args.dtype.parse()?) + .with_model_device(args.device.parse()?) + .with_model_file("/HDD/zhangj/PROJECTS/e/mmml/rtmw-133-m-q4.onnx") + .commit()?; + let mut rtmpose = RTMPose::new(config)?; + + // build annotator + let annotator = Annotator::default() + .with_hbb_style(Style::hbb().with_draw_fill(true)) + .with_keypoint_style( + Style::keypoint() + .with_radius(2) + .with_skeleton((SKELETON_COCO_65, SKELETON_COLOR_COCO_65).into()) + .show_id(false) + .show_confidence(false) + .show_name(false), + ); + + // build dataloader + let dl = DataLoader::new(&args.source)?.with_batch(1).build()?; + + // iterate + for xs in &dl { + // YOLO infer + let ys_det = yolo.forward(&xs)?; + + // RTMPose infer + for (x, y_det) in xs.iter().zip(ys_det.iter()) { + let y = rtmpose.forward(x, y_det.hbbs())?; + + // Annotate + annotator.annotate(x, &y)?.save(format!( + "{}.jpg", + usls::Dir::Current + .base_dir_with_subs(&["runs", rtmpose.spec()])? + .join(usls::timestamp(None)) + .display(), + ))? + } + } + + // summary + yolo.summary(); + rtmpose.summary(); + + Ok(()) +} diff --git a/src/core/names.rs b/src/core/names.rs index cf25bbc..4d312dd 100644 --- a/src/core/names.rs +++ b/src/core/names.rs @@ -1,3 +1,188 @@ +/// A comprehensive list of keypoints used in the COCO-133 person pose estimation model. +/// The keypoints are organized into several groups: +/// - Body keypoints (23 points): +/// - Face features (nose, eyes, ears) +/// - Body joints (shoulders, elbows, wrists, hips, knees, ankles) +/// - Foot details (toes and heels) +/// - Face landmarks (68 points): +/// - Detailed facial features numbered from face-0 to face-67 +/// - Hand keypoints (21 points per hand): +/// - Left hand joints (wrist and fingers) +/// - Right hand joints (wrist and fingers) +pub const NAMES_COCO_133: [&str; 133] = [ + // Body keypoints (0-22) + "nose", + "left_eye", + "right_eye", + "left_ear", + "right_ear", + "left_shoulder", + "right_shoulder", + "left_elbow", + "right_elbow", + "left_wrist", + "right_wrist", + "left_hip", + "right_hip", + "left_knee", + "right_knee", + "left_ankle", + "right_ankle", + "left_big_toe", + "left_small_toe", + "left_heel", + "right_big_toe", + "right_small_toe", + "right_heel", + // Face landmarks (23-90) + "face-0", + "face-1", + "face-2", + "face-3", + "face-4", + "face-5", + "face-6", + "face-7", + "face-8", + "face-9", + "face-10", + "face-11", + "face-12", + "face-13", + "face-14", + "face-15", + "face-16", + "face-17", + "face-18", + "face-19", + "face-20", + "face-21", + "face-22", + "face-23", + "face-24", + "face-25", + "face-26", + "face-27", + "face-28", + "face-29", + "face-30", + "face-31", + "face-32", + "face-33", + "face-34", + "face-35", + "face-36", + "face-37", + "face-38", + "face-39", + "face-40", + "face-41", + "face-42", + "face-43", + "face-44", + "face-45", + "face-46", + "face-47", + "face-48", + "face-49", + "face-50", + "face-51", + "face-52", + "face-53", + "face-54", + "face-55", + "face-56", + "face-57", + "face-58", + "face-59", + "face-60", + "face-61", + "face-62", + "face-63", + "face-64", + "face-65", + "face-66", + "face-67", + // Left hand keypoints (91-111) + "left_hand_root", + "left_thumb1", + "left_thumb2", + "left_thumb3", + "left_thumb4", + "left_forefinger1", + "left_forefinger2", + "left_forefinger3", + "left_forefinger4", + "left_middle_finger1", + "left_middle_finger2", + "left_middle_finger3", + "left_middle_finger4", + "left_ring_finger1", + "left_ring_finger2", + "left_ring_finger3", + "left_ring_finger4", + "left_pinky_finger1", + "left_pinky_finger2", + "left_pinky_finger3", + "left_pinky_finger4", + // Right hand keypoints (112-132) + "right_hand_root", + "right_thumb1", + "right_thumb2", + "right_thumb3", + "right_thumb4", + "right_forefinger1", + "right_forefinger2", + "right_forefinger3", + "right_forefinger4", + "right_middle_finger1", + "right_middle_finger2", + "right_middle_finger3", + "right_middle_finger4", + "right_ring_finger1", + "right_ring_finger2", + "right_ring_finger3", + "right_ring_finger4", + "right_pinky_finger1", + "right_pinky_finger2", + "right_pinky_finger3", + "right_pinky_finger4", +]; + +/// Hand keypoint labels used in hand pose estimation with 21 points. +/// The keypoints are organized by fingers: +/// - Wrist (1 point) +/// - Thumb (4 points) +/// - Forefinger/Index (4 points) +/// - Middle finger (4 points) +/// - Ring finger (4 points) +/// - Pinky finger (4 points) +/// +/// Each finger (except wrist) has 4 keypoints representing joints from base to tip +pub const NAMES_HAND_21: [&str; 21] = [ + "wrist", // Central point + "thumb1", // Thumb base + "thumb2", // Thumb joint 1 + "thumb3", // Thumb joint 2 + "thumb4", // Thumb tip + "forefinger1", // Index finger base + "forefinger2", // Index finger joint 1 + "forefinger3", // Index finger joint 2 + "forefinger4", // Index finger tip + "middle_finger1", // Middle finger base + "middle_finger2", // Middle finger joint 1 + "middle_finger3", // Middle finger joint 2 + "middle_finger4", // Middle finger tip + "ring_finger1", // Ring finger base + "ring_finger2", // Ring finger joint 1 + "ring_finger3", // Ring finger joint 2 + "ring_finger4", // Ring finger tip + "pinky_finger1", // Pinky finger base + "pinky_finger2", // Pinky finger joint 1 + "pinky_finger3", // Pinky finger joint 2 + "pinky_finger4", // Pinky finger tip +]; + /// A comprehensive list of 4585 object categories used in the YOLOE model. /// A comprehensive list of 4585 object categories used in the YOLOE object detection model. pub static NAMES_YOLOE_4585: [&str; 4585] = [ @@ -4724,6 +4909,59 @@ pub const NAMES_COCO_KEYPOINTS_17: [&str; 17] = [ "right_ankle", ]; +/// A constant array containing the 26 keypoint labels used in the HALPE human pose estimation model. +/// +/// These keypoints are organized into several anatomical groups: +/// - **Head Region** +/// - nose, eyes (left/right), ears (left/right), head +/// - **Upper Body** +/// - shoulders (left/right), elbows (left/right), wrists (left/right), neck +/// - **Core Body** +/// - hips (left/right + central hip) +/// - **Lower Body** +/// - knees (left/right), ankles (left/right) +/// - **Feet** +/// - big toes (left/right), small toes (left/right), heels (left/right) +/// +/// This model extends the standard COCO 17-keypoint format by adding: +/// - Additional head keypoint (separate from nose) +/// - Neck keypoint +/// - Central hip point +/// - Detailed foot keypoints (toes and heels) +/// +/// The keypoints follow a consistent naming convention: +/// - Paired body parts are prefixed with "left_" or "right_" +/// - All labels use lowercase and underscores +/// - Generic point names for central points (e.g., "head", "neck", "hip") +pub const NAMES_HALPE_KEYPOINTS_26: [&str; 26] = [ + "nose", + "left_eye", + "right_eye", + "left_ear", + "right_ear", + "left_shoulder", + "right_shoulder", + "left_elbow", + "right_elbow", + "left_wrist", + "right_wrist", + "left_hip", + "right_hip", + "left_knee", + "right_knee", + "left_ankle", + "right_ankle", + "head", + "neck", + "hip", + "left_big_toe", + "right_big_toe", + "left_small_toe", + "right_small_toe", + "left_heel", + "right_heel", +]; + /// COCO dataset object detection labels with 80 categories. /// /// A widely-used set of object categories for general object detection, including: diff --git a/src/core/processor.rs b/src/core/processor.rs index ad3a3d3..0571843 100644 --- a/src/core/processor.rs +++ b/src/core/processor.rs @@ -97,7 +97,6 @@ impl Processor { pub fn process_images(&mut self, xs: &[Image]) -> Result { let (mut x, images_transform_info) = self.par_resize(xs)?; self.images_transform_info = images_transform_info; - if self.do_normalize { x = x.normalize(0., 255.)?; } diff --git a/src/models/dwpose/README.md b/src/models/dwpose/README.md new file mode 100644 index 0000000..11e975d --- /dev/null +++ b/src/models/dwpose/README.md @@ -0,0 +1,9 @@ +# DWPose: Effective Whole-body Pose Estimation with Two-stages Distillation + +## Official Repository + +The official repository can be found on: [GitHub](https://github.com/IDEA-Research/DWPose) + +## Example + +Refer to the [example](../../../examples/dwpose) diff --git a/src/models/dwpose/config.rs b/src/models/dwpose/config.rs new file mode 100644 index 0000000..3f95d5c --- /dev/null +++ b/src/models/dwpose/config.rs @@ -0,0 +1,22 @@ +/// Model configuration for `DWPose` +impl crate::Config { + pub fn dwpose_133_t() -> Self { + Self::rtmpose_133().with_model_file("dwpose-133-t.onnx") + } + + pub fn dwpose_133_s() -> Self { + Self::rtmpose_133().with_model_file("dwpose-133-s.onnx") + } + + pub fn dwpose_133_m() -> Self { + Self::rtmpose_133().with_model_file("dwpose-133-m.onnx") + } + + pub fn dwpose_133_l() -> Self { + Self::rtmpose_133().with_model_file("dwpose-133-l.onnx") + } + + pub fn dwpose_133_l_384() -> Self { + Self::rtmpose_133().with_model_file("dwpose-133-l-384.onnx") + } +} diff --git a/src/models/dwpose/mod.rs b/src/models/dwpose/mod.rs new file mode 100644 index 0000000..1bf79df --- /dev/null +++ b/src/models/dwpose/mod.rs @@ -0,0 +1 @@ +mod config; diff --git a/src/models/mod.rs b/src/models/mod.rs index 1391c99..2ff91ee 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -10,6 +10,7 @@ mod deit; mod depth_anything; mod depth_pro; mod dinov2; +mod dwpose; mod fast; mod fastvit; mod florence2; @@ -26,6 +27,8 @@ mod rfdetr; mod rmbg; mod rtdetr; mod rtmo; +mod rtmpose; +mod rtmw; mod sam; mod sam2; mod sapiens; @@ -54,6 +57,7 @@ pub use rfdetr::*; pub use rmbg::*; pub use rtdetr::*; pub use rtmo::*; +pub use rtmpose::*; pub use sam::*; pub use sam2::*; pub use sapiens::*; diff --git a/src/models/rtmo/config.rs b/src/models/rtmo/config.rs index 768d97e..76a6f40 100644 --- a/src/models/rtmo/config.rs +++ b/src/models/rtmo/config.rs @@ -13,6 +13,7 @@ impl crate::Config { .with_nk(17) .with_class_confs(&[0.3]) .with_keypoint_confs(&[0.5]) + .with_keypoint_names(&crate::NAMES_COCO_KEYPOINTS_17) } pub fn rtmo_t() -> Self { diff --git a/src/models/rtmo/impl.rs b/src/models/rtmo/impl.rs index 5ec5c9d..2376282 100644 --- a/src/models/rtmo/impl.rs +++ b/src/models/rtmo/impl.rs @@ -16,6 +16,7 @@ pub struct RTMO { processor: Processor, confs: DynConf, kconfs: DynConf, + names: Vec, } impl RTMO { @@ -28,6 +29,7 @@ impl RTMO { engine.try_width().unwrap_or(&512.into()).opt(), engine.ts().clone(), ); + let names: Vec = config.keypoint_names().to_vec(); let nk = config.nk().unwrap_or(17); let confs = DynConf::new(config.class_confs(), 1); let kconfs = DynConf::new(config.keypoint_confs(), nk); @@ -45,6 +47,7 @@ impl RTMO { processor, confs, kconfs, + names, }) } @@ -119,10 +122,17 @@ impl RTMO { if c < self.kconfs[i] { kpts_.push(Keypoint::default()); } else { - kpts_.push(Keypoint::default().with_id(i).with_confidence(c).with_xy( - x.max(0.0f32).min(width_original as _), - y.max(0.0f32).min(height_original as _), - )); + let mut kpt_ = + Keypoint::default().with_id(i).with_confidence(c).with_xy( + x.max(0.0f32).min(width_original as _), + y.max(0.0f32).min(height_original as _), + ); + + if !self.names.is_empty() { + kpt_ = kpt_.with_name(&self.names[i]); + } + + kpts_.push(kpt_); } } y_kpts.push(kpts_); diff --git a/src/models/rtmpose/README.md b/src/models/rtmpose/README.md new file mode 100644 index 0000000..5e9e777 --- /dev/null +++ b/src/models/rtmpose/README.md @@ -0,0 +1,9 @@ +# RTMPose: Real-Time Multi-Person Pose Estimation toolkit based on MMPose + +## Official Repository + +The official repository can be found on: [GitHub](https://github.com/open-mmlab/mmpose/tree/dev-1.x/projects/rtmpose) + +## Example + +Refer to the [example](../../../examples/rtmpose) diff --git a/src/models/rtmpose/config.rs b/src/models/rtmpose/config.rs new file mode 100644 index 0000000..ca50959 --- /dev/null +++ b/src/models/rtmpose/config.rs @@ -0,0 +1,79 @@ +/// Model configuration for `RTMPose` +impl crate::Config { + pub fn rtmpose() -> Self { + Self::default() + .with_name("rtmpose") + .with_model_ixx(0, 0, 1.into()) + .with_model_ixx(0, 1, 3.into()) + .with_model_ixx(0, 2, 256.into()) + .with_model_ixx(0, 3, 192.into()) + .with_image_mean(&[123.675, 116.28, 103.53]) + .with_image_std(&[58.395, 57.12, 57.375]) + .with_normalize(false) + .with_keypoint_confs(&[0.35]) + } + + pub fn rtmpose_17() -> Self { + Self::rtmpose() + .with_nk(17) + .with_keypoint_names(&crate::NAMES_COCO_KEYPOINTS_17) + } + + pub fn rtmpose_17_t() -> Self { + Self::rtmpose_17().with_model_file("rtmpose-17-t.onnx") + } + + pub fn rtmpose_17_s() -> Self { + Self::rtmpose_17().with_model_file("rtmpose-17-s.onnx") + } + + pub fn rtmpose_17_m() -> Self { + Self::rtmpose_17().with_model_file("rtmpose-17-m.onnx") + } + + pub fn rtmpose_17_l() -> Self { + Self::rtmpose_17().with_model_file("rtmpose-17-l.onnx") + } + + pub fn rtmpose_17_l_384() -> Self { + Self::rtmpose_17().with_model_file("rtmpose-17-l-384.onnx") + } + + pub fn rtmpose_17_x() -> Self { + Self::rtmpose_17().with_model_file("rtmpose-17-x.onnx") + } + + pub fn rtmpose_26() -> Self { + Self::rtmpose() + .with_nk(26) + .with_keypoint_names(&crate::NAMES_HALPE_KEYPOINTS_26) + } + + pub fn rtmpose_26_t() -> Self { + Self::rtmpose_26().with_model_file("rtmpose-26-t.onnx") + } + + pub fn rtmpose_26_s() -> Self { + Self::rtmpose_26().with_model_file("rtmpose-26-s.onnx") + } + + pub fn rtmpose_26_m() -> Self { + Self::rtmpose_26().with_model_file("rtmpose-26-m.onnx") + } + + pub fn rtmpose_26_m_384() -> Self { + Self::rtmpose_26().with_model_file("rtmpose-26-m-384.onnx") + } + + pub fn rtmpose_26_l() -> Self { + Self::rtmpose_26().with_model_file("rtmpose-26-l.onnx") + } + + pub fn rtmpose_26_l_384() -> Self { + Self::rtmpose_26().with_model_file("rtmpose-26-l-384.onnx") + } + + pub fn rtmpose_26_x() -> Self { + Self::rtmpose_26().with_model_file("rtmpose-26-x.onnx") + } +} diff --git a/src/models/rtmpose/impl.rs b/src/models/rtmpose/impl.rs new file mode 100644 index 0000000..1af5d67 --- /dev/null +++ b/src/models/rtmpose/impl.rs @@ -0,0 +1,369 @@ +use aksr::Builder; +use anyhow::Result; +use ndarray::s; +use rayon::prelude::*; +use std::f32::consts::PI; + +use crate::{elapsed, Config, DynConf, Engine, Hbb, Image, Keypoint, Processor, Ts, Xs, Y}; + +struct CentersAndScales { + pub centers: Vec, + pub scales: Vec, +} + +#[derive(Builder, Debug)] +pub struct RTMPose { + engine: Engine, + height: usize, + width: usize, + batch: usize, + ts: Ts, + spec: String, + processor: Processor, + nk: usize, + kconfs: DynConf, + names: Vec, + simcc_split_ratio: f32, +} + +impl RTMPose { + pub fn new(config: Config) -> Result { + let engine = Engine::try_from_config(&config.model)?; + let spec = engine.spec().to_string(); + let (batch, height, width, ts) = ( + engine.batch().opt(), + engine.try_height().unwrap_or(&256.into()).opt(), + engine.try_width().unwrap_or(&192.into()).opt(), + engine.ts().clone(), + ); + let nk = config.nk().unwrap_or(17); + let kconfs = DynConf::new(config.keypoint_confs(), nk); + let names = config.keypoint_names().to_vec(); + let simcc_split_ratio = 2.0; + let processor = Processor::try_from_config(&config.processor)? + .with_image_width(width as _) + .with_image_height(height as _); + + Ok(Self { + engine, + height, + width, + batch, + ts, + spec, + processor, + nk, + kconfs, + names, + simcc_split_ratio, + }) + } + + fn preprocess_one( + img: &Image, + hbb: Option<&Hbb>, + model_input_size: (usize, usize), + ) -> Result<(Image, CentersAndScales)> { + let hbb = if let Some(hbb) = hbb { + hbb + } else { + &Hbb::from_xyxy(0.0, 0.0, img.width() as f32, img.height() as f32) + }; + let (center, scale) = Self::hbb2cs(hbb, 1.25); + let (resized_img, scale) = Self::top_down_affine( + (model_input_size.0 as i32, model_input_size.1 as i32), + &scale, + ¢er, + img, + )?; + + Ok(( + resized_img, + CentersAndScales { + centers: vec![center], + scales: vec![scale], + }, + )) + } + + fn preprocess(&mut self, x: &Image, hbbs: Option<&[Hbb]>) -> Result<(Xs, CentersAndScales)> { + let results: Result> = match hbbs { + None | Some(&[]) => vec![Self::preprocess_one(x, None, (self.width, self.height))] + .into_iter() + .collect(), + Some(hbbs) => hbbs + .par_iter() + .map(|hbb| Self::preprocess_one(x, Some(hbb), (self.width, self.height))) + .collect(), + }; + let (processed_images, centers_and_scales): (Vec<_>, Vec<_>) = results?.into_iter().unzip(); + let mut centerss = Vec::new(); + let mut scaless = Vec::new(); + for cs in centers_and_scales { + centerss.extend(cs.centers); + scaless.extend(cs.scales); + } + // TODO + let x = self.processor.process_images(&processed_images)?; + + // Update batch size + self.batch = match hbbs { + None | Some(&[]) => 1, + Some(hbbs) => hbbs.len(), + }; + + Ok(( + x.into(), + CentersAndScales { + centers: centerss, + scales: scaless, + }, + )) + } + + fn inference(&mut self, xs: Xs) -> Result { + self.engine.run(xs) + } + + pub fn forward(&mut self, x: &Image, hbbs: Option<&[Hbb]>) -> Result { + let (ys, centers_and_scales) = + elapsed!("preprocess", self.ts, { self.preprocess(x, hbbs)? }); + let ys = elapsed!("inference", self.ts, { self.inference(ys)? }); + let y = elapsed!("postprocess", self.ts, { + self.postprocess(ys, centers_and_scales)? + }); + + Ok(y) + } + + fn postprocess(&mut self, xs: Xs, centers_and_scales: CentersAndScales) -> Result { + // Update nk + self.nk = xs[0].shape()[1]; + + let simcc_x_array = &xs[0]; + let simcc_y_array = &xs[1]; + let y_kpts: Vec> = (0..self.batch) + .into_par_iter() + .map(|batch_idx| { + let mut keypoints = Vec::new(); + let center = ¢ers_and_scales.centers[batch_idx]; + let scale = ¢ers_and_scales.scales[batch_idx]; + for kpt_idx in 0..self.nk { + let simcc_x_slice = simcc_x_array.slice(s![batch_idx, kpt_idx, ..]); + let simcc_y_slice = simcc_y_array.slice(s![batch_idx, kpt_idx, ..]); + let (x_loc, max_val_x) = Self::argmax_and_max(&simcc_x_slice); + let (y_loc, max_val_y) = Self::argmax_and_max(&simcc_y_slice); + let confidence = 0.5 * (max_val_x + max_val_y); + let mut x = x_loc as f32 / self.simcc_split_ratio; + let mut y = y_loc as f32 / self.simcc_split_ratio; + + // keypoints = keypoints / model_input_size * scale + center - scale / 2 + let keypoint = if confidence > self.kconfs[kpt_idx] { + x = x / self.width as f32 * scale.x() + center.x() - scale.x() / 2.0; + y = y / self.height as f32 * scale.y() + center.y() - scale.y() / 2.0; + + let mut kpt = Keypoint::from((x, y)) + .with_confidence(confidence) + .with_id(kpt_idx); + + if !self.names.is_empty() { + kpt = kpt.with_name(&self.names[kpt_idx]); + } + kpt + } else { + Keypoint::default() + }; + + keypoints.push(keypoint); + } + + keypoints + }) + .collect(); + + Ok(Y::default().with_keypointss(&y_kpts)) + } + + pub fn summary(&mut self) { + self.ts.summary(); + } + + fn argmax_and_max(arr: &ndarray::ArrayView1) -> (usize, f32) { + let mut max_idx = 0; + let mut max_val = arr[0]; + + for (idx, &val) in arr.iter().enumerate() { + if val > max_val { + max_val = val; + max_idx = idx; + } + } + + (max_idx, max_val) + } + + fn hbb2cs(hbb: &Hbb, padding: f32) -> (Keypoint, Keypoint) { + let (x1, y1, x2, y2) = hbb.xyxy(); + ( + ((x1 + x2) * 0.5, (y1 + y2) * 0.5).into(), + ((x2 - x1) * padding, (y2 - y1) * padding).into(), + ) + } + + fn get_warp_matrix( + center: &Keypoint, + scale: &Keypoint, + rot: f32, + output_size: (i32, i32), + shift: (f32, f32), + inv: bool, + ) -> Vec { + fn get_3rd_point(a: &Keypoint, b: &Keypoint) -> Keypoint { + let direction = Keypoint::new(a.x() - b.x(), a.y() - b.y()); + (b.x() - direction.y(), b.y() + direction.x()).into() + } + + let shift = Keypoint::new(shift.0, shift.1); + let src_w = scale.x(); + let dst_w = output_size.0 as f32; + let dst_h = output_size.1 as f32; + let rot_rad = rot * PI / 180.0; + let src_dir = Keypoint::new(0.0, src_w * -0.5).rotate(rot_rad); + let dst_dir = Keypoint::new(0.0, dst_w * -0.5); + let src_0 = Keypoint::new( + center.x() + scale.x() * shift.x(), + center.y() + scale.y() * shift.y(), + ); + let src_1 = Keypoint::new( + center.x() + src_dir.x() + scale.x() * shift.x(), + center.y() + src_dir.y() + scale.y() * shift.y(), + ); + let src_2 = get_3rd_point(&src_0, &src_1); + let dst_0 = Keypoint::new(dst_w * 0.5, dst_h * 0.5); + let dst_1 = Keypoint::new(dst_w * 0.5 + dst_dir.x(), dst_h * 0.5 + dst_dir.y()); + let dst_2 = get_3rd_point(&dst_0, &dst_1); + let (src_points, dst_points) = if inv { + (vec![&dst_0, &dst_1, &dst_2], vec![&src_0, &src_1, &src_2]) + } else { + (vec![&src_0, &src_1, &src_2], vec![&dst_0, &dst_1, &dst_2]) + }; + + let x1 = src_points[0].x(); + let y1 = src_points[0].y(); + let x2 = src_points[1].x(); + let y2 = src_points[1].y(); + let x3 = src_points[2].x(); + let y3 = src_points[2].y(); + + let u1 = dst_points[0].x(); + let v1 = dst_points[0].y(); + let u2 = dst_points[1].x(); + let v2 = dst_points[1].y(); + let u3 = dst_points[2].x(); + let v3 = dst_points[2].y(); + + let det = x1 * (y2 - y3) + x2 * (y3 - y1) + x3 * (y1 - y2); + + if det.abs() < 1e-6 { + return vec![1.0, 0.0, 0.0, 0.0, 1.0, 0.0]; + } + + let m00 = (u1 * (y2 - y3) + u2 * (y3 - y1) + u3 * (y1 - y2)) / det; + let m01 = (x1 * (u3 - u2) + x2 * (u1 - u3) + x3 * (u2 - u1)) / det; + let m02 = + (x1 * (y2 * u3 - y3 * u2) + x2 * (y3 * u1 - y1 * u3) + x3 * (y1 * u2 - y2 * u1)) / det; + let m10 = (v1 * (y2 - y3) + v2 * (y3 - y1) + v3 * (y1 - y2)) / det; + let m11 = (x1 * (v2 - v3) + x2 * (v3 - v1) + x3 * (v1 - v2)) / det; + let m12 = + (x1 * (y2 * v3 - y3 * v2) + x2 * (y3 * v1 - y1 * v3) + x3 * (y1 * v2 - y2 * v1)) / det; + + vec![m00, m01, m02, m10, m11, m12] + } + + fn warp_affine(img: &Image, warp_mat: &[f32], output_size: (i32, i32)) -> Result { + let (width, height) = output_size; + let mut result_data = vec![0u8; (height * width * 3) as usize]; + let img_rgb = img.to_rgb8(); + let (img_w, img_h) = img_rgb.dimensions(); + + let m00 = warp_mat[0]; + let m01 = warp_mat[1]; + let m02 = warp_mat[2]; + let m10 = warp_mat[3]; + let m11 = warp_mat[4]; + let m12 = warp_mat[5]; + + let det = m00 * m11 - m01 * m10; + if det.abs() < 1e-6 { + return Image::from_u8s(&result_data, width as u32, height as u32); + } + + let inv_det = 1.0 / det; + let inv_m00 = m11 * inv_det; + let inv_m01 = -m01 * inv_det; + let inv_m10 = -m10 * inv_det; + let inv_m11 = m00 * inv_det; + let inv_m02 = (m01 * m12 - m11 * m02) * inv_det; + let inv_m12 = (m10 * m02 - m00 * m12) * inv_det; + let img_data = img_rgb.as_raw(); + + for y in 0..height { + for x in 0..width { + let dst_x = x as f32; + let dst_y = y as f32; + let src_x = inv_m00 * dst_x + inv_m01 * dst_y + inv_m02; + let src_y = inv_m10 * dst_x + inv_m11 * dst_y + inv_m12; + let x0 = src_x.floor() as i32; + let y0 = src_y.floor() as i32; + let x1 = x0 + 1; + let y1 = y0 + 1; + if x0 >= 0 && x1 < img_w as i32 && y0 >= 0 && y1 < img_h as i32 { + let dx = src_x - x0 as f32; + let dy = src_y - y0 as f32; + let dst_idx = ((y * width + x) * 3) as usize; + + for c in 0..3 { + let src_idx_00 = ((y0 as u32 * img_w + x0 as u32) * 3 + c as u32) as usize; + let src_idx_01 = ((y0 as u32 * img_w + x1 as u32) * 3 + c as u32) as usize; + let src_idx_10 = ((y1 as u32 * img_w + x0 as u32) * 3 + c as u32) as usize; + let src_idx_11 = ((y1 as u32 * img_w + x1 as u32) * 3 + c as u32) as usize; + let p00 = img_data[src_idx_00] as f32; + let p01 = img_data[src_idx_01] as f32; + let p10 = img_data[src_idx_10] as f32; + let p11 = img_data[src_idx_11] as f32; + let interpolated = p00 * (1.0 - dx) * (1.0 - dy) + + p01 * dx * (1.0 - dy) + + p10 * (1.0 - dx) * dy + + p11 * dx * dy; + + result_data[dst_idx + c] = interpolated.round() as u8; + } + } + } + } + + Image::from_u8s(&result_data, width as u32, height as u32) + } + + fn top_down_affine( + input_size: (i32, i32), + scale: &Keypoint, + center: &Keypoint, + img: &Image, + ) -> Result<(Image, Keypoint)> { + let (w, h) = input_size; + let aspect_ratio = w as f32 / h as f32; + let b_w = scale.x(); + let b_h = scale.y(); + let scale = if b_w > b_h * aspect_ratio { + Keypoint::new(b_w, b_w / aspect_ratio) + } else { + Keypoint::new(b_h * aspect_ratio, b_h) + }; + let rot = 0.0; + let warp_mat = Self::get_warp_matrix(center, &scale, rot, (w, h), (0.0, 0.0), false); + let img = Self::warp_affine(img, &warp_mat, (w, h))?; + + Ok((img, scale)) + } +} diff --git a/src/models/rtmpose/mod.rs b/src/models/rtmpose/mod.rs new file mode 100644 index 0000000..fbd2b75 --- /dev/null +++ b/src/models/rtmpose/mod.rs @@ -0,0 +1,4 @@ +mod config; +mod r#impl; + +pub use r#impl::*; diff --git a/src/models/rtmw/README.md b/src/models/rtmw/README.md new file mode 100644 index 0000000..fc3e7e4 --- /dev/null +++ b/src/models/rtmw/README.md @@ -0,0 +1,9 @@ +# RTMW: Real-Time Multi-Person 2D and 3D Whole-body Pose Estimation + +## Official Repository + +The official paper can be found on: [arxiv](https://arxiv.org/abs/2407.08634) + +## Example + +Refer to the [example](../../../examples/rtmw) diff --git a/src/models/rtmw/config.rs b/src/models/rtmw/config.rs new file mode 100644 index 0000000..7a3f524 --- /dev/null +++ b/src/models/rtmw/config.rs @@ -0,0 +1,24 @@ +/// Model configuration for `RTMW` +impl crate::Config { + pub fn rtmpose_133() -> Self { + Self::rtmpose() + .with_nk(133) + .with_keypoint_names(&crate::NAMES_COCO_133) + } + + pub fn rtmw_133_m() -> Self { + Self::rtmpose_133().with_model_file("rtmw-133-m.onnx") + } + + pub fn rtmw_133_m_384() -> Self { + Self::rtmpose_133().with_model_file("rtmw-133-m-384.onnx") + } + + pub fn rtmw_133_l() -> Self { + Self::rtmpose_133().with_model_file("rtmw-133-l.onnx") + } + + pub fn rtmw_133_x() -> Self { + Self::rtmpose_133().with_model_file("rtmw-133-x.onnx") + } +} diff --git a/src/models/rtmw/mod.rs b/src/models/rtmw/mod.rs new file mode 100644 index 0000000..1bf79df --- /dev/null +++ b/src/models/rtmw/mod.rs @@ -0,0 +1 @@ +mod config; diff --git a/src/results/keypoint.rs b/src/results/keypoint.rs index b486535..f3e0304 100644 --- a/src/results/keypoint.rs +++ b/src/results/keypoint.rs @@ -104,6 +104,12 @@ impl Keypoint { self.x + self.y } + pub fn rotate(&self, angle_rad: f32) -> Self { + let sn = angle_rad.sin(); + let cs = angle_rad.cos(); + (cs * self.x - sn * self.y, sn * self.x + cs * self.y).into() + } + /// Calculates the perpendicular distance from the current keypoint (`self`) /// to a line segment defined by two other keypoints (`start` and `end`). pub fn perpendicular_distance(&self, start: &Self, end: &Self) -> f32 { diff --git a/src/results/skeleton.rs b/src/results/skeleton.rs index 0d5663e..eb859f5 100644 --- a/src/results/skeleton.rs +++ b/src/results/skeleton.rs @@ -142,3 +142,297 @@ pub const SKELETON_COLOR_COCO_19: [Color; 19] = [ Color(0x00ff00ff), Color(0x00ff00ff), ]; + +/// Defines the keypoint connections for the HALPE person skeleton with 27 connections. +/// Each tuple (a, b) represents a connection between keypoint indices a and b. +/// The connections define the following body parts: +/// - Upper body: +/// - Head and neck connections +/// - Shoulders to elbows to wrists +/// - Torso: +/// - Neck to hip +/// - Shoulders to neck +/// - Lower body: +/// - Hip to knees to ankles +/// - Ankles to toes and heels +pub const SKELETON_HALPE_27: [(usize, usize); 27] = [ + (15, 13), // left_ankle -> left_knee + (13, 11), // left_knee -> left_hip + (11, 19), // left_hip -> hip + (16, 14), // right_ankle -> right_knee + (14, 12), // right_knee -> right_hip + (12, 19), // right_hip -> hip + (17, 18), // head -> neck + (18, 19), // neck -> hip + (18, 5), // neck -> left_shoulder + (5, 7), // left_shoulder -> left_elbow + (7, 9), // left_elbow -> left_wrist + (18, 6), // neck -> right_shoulder + (6, 8), // right_shoulder -> right_elbow + (8, 10), // right_elbow -> right_wrist + (1, 2), // left_eye -> right_eye + (0, 1), // nose -> left_eye + (0, 2), // nose -> right_eye + (1, 3), // left_eye -> left_ear + (2, 4), // right_eye -> right_ear + (3, 5), // left_ear -> left_shoulder + (4, 6), // right_ear -> right_shoulder + (15, 20), // left_ankle -> left_big_toe + (15, 22), // left_ankle -> left_small_toe + (15, 24), // left_ankle -> left_heel + (16, 21), // right_ankle -> right_big_toe + (16, 23), // right_ankle -> right_small_toe + (16, 25), // right_ankle -> right_heel +]; + +/// Defines colors for visualizing each connection in the HALPE person skeleton. +/// Colors are grouped by body parts and sides: +/// - Blue (0x3399ff): Face and central connections (eyes, neck to hip) +/// - Green (0x00ff00): Left side limbs and extremities +/// - Orange (0xff8000): Right side limbs and extremities +pub const SKELETON_COLOR_HALPE_27: [Color; 27] = [ + Color(0x00ff00ff), // left_ankle -> left_knee + Color(0x00ff00ff), // left_knee -> left_hip + Color(0x00ff00ff), // left_hip -> hip + Color(0xff8000ff), // right_ankle -> right_knee + Color(0xff8000ff), // right_knee -> right_hip + Color(0xff8000ff), // right_hip -> hip + Color(0x3399ffff), // head -> neck + Color(0x3399ffff), // neck -> hip + Color(0x00ff00ff), // neck -> left_shoulder + Color(0x00ff00ff), // left_shoulder -> left_elbow + Color(0x00ff00ff), // left_elbow -> left_wrist + Color(0xff8000ff), // neck -> right_shoulder + Color(0xff8000ff), // right_shoulder -> right_elbow + Color(0xff8000ff), // right_elbow -> right_wrist + Color(0x3399ffff), // left_eye -> right_eye + Color(0x3399ffff), // nose -> left_eye + Color(0x3399ffff), // nose -> right_eye + Color(0x3399ffff), // left_eye -> left_ear + Color(0x3399ffff), // right_eye -> right_ear + Color(0x3399ffff), // left_ear -> left_shoulder + Color(0x3399ffff), // right_ear -> right_shoulder + Color(0x00ff00ff), // left_ankle -> left_big_toe + Color(0x00ff00ff), // left_ankle -> left_small_toe + Color(0x00ff00ff), // left_ankle -> left_heel + Color(0xff8000ff), // right_ankle -> right_big_toe + Color(0xff8000ff), // right_ankle -> right_small_toe + Color(0xff8000ff), // right_ankle -> right_heel +]; + +/// Defines the keypoint connections for the hand skeleton with 20 connections. +/// Each tuple (a, b) represents a connection between keypoint indices a and b. +/// The connections define the following parts: +/// - Thumb: wrist -> thumb1 -> thumb2 -> thumb3 -> thumb4 +/// - Index: wrist -> forefinger1 -> forefinger2 -> forefinger3 -> forefinger4 +/// - Middle: wrist -> middle_finger1 -> middle_finger2 -> middle_finger3 -> middle_finger4 +/// - Ring: wrist -> ring_finger1 -> ring_finger2 -> ring_finger3 -> ring_finger4 +/// - Pinky: wrist -> pinky_finger1 -> pinky_finger2 -> pinky_finger3 -> pinky_finger4 +pub const SKELETON_HAND_21: [(usize, usize); 20] = [ + (0, 1), // wrist -> thumb1 + (1, 2), // thumb1 -> thumb2 + (2, 3), // thumb2 -> thumb3 + (3, 4), // thumb3 -> thumb4 + (0, 5), // wrist -> forefinger1 + (5, 6), // forefinger1 -> forefinger2 + (6, 7), // forefinger2 -> forefinger3 + (7, 8), // forefinger3 -> forefinger4 + (0, 9), // wrist -> middle_finger1 + (9, 10), // middle_finger1 -> middle_finger2 + (10, 11), // middle_finger2 -> middle_finger3 + (11, 12), // middle_finger3 -> middle_finger4 + (0, 13), // wrist -> ring_finger1 + (13, 14), // ring_finger1 -> ring_finger2 + (14, 15), // ring_finger2 -> ring_finger3 + (15, 16), // ring_finger3 -> ring_finger4 + (0, 17), // wrist -> pinky_finger1 + (17, 18), // pinky_finger1 -> pinky_finger2 + (18, 19), // pinky_finger2 -> pinky_finger3 + (19, 20), // pinky_finger3 -> pinky_finger4 +]; + +/// Defines colors for visualizing each connection in the hand skeleton. +/// Colors are grouped by fingers: +/// - Thumb: Orange (0xff8000) +/// - Index: Pink (0xff99ff) +/// - Middle: Light Blue (0x66b2ff) +/// - Ring: Red (0xff3333) +/// - Pinky: Green (0x00ff00) +pub const SKELETON_COLOR_HAND_21: [Color; 20] = [ + Color(0xff8000ff), // wrist -> thumb1 + Color(0xff8000ff), // thumb1 -> thumb2 + Color(0xff8000ff), // thumb2 -> thumb3 + Color(0xff8000ff), // thumb3 -> thumb4 + Color(0xff99ffff), // wrist -> forefinger1 + Color(0xff99ffff), // forefinger1 -> forefinger2 + Color(0xff99ffff), // forefinger2 -> forefinger3 + Color(0xff99ffff), // forefinger3 -> forefinger4 + Color(0x66b2ffff), // wrist -> middle_finger1 + Color(0x66b2ffff), // middle_finger1 -> middle_finger2 + Color(0x66b2ffff), // middle_finger2 -> middle_finger3 + Color(0x66b2ffff), // middle_finger3 -> middle_finger4 + Color(0xff3333ff), // wrist -> ring_finger1 + Color(0xff3333ff), // ring_finger1 -> ring_finger2 + Color(0xff3333ff), // ring_finger2 -> ring_finger3 + Color(0xff3333ff), // ring_finger3 -> ring_finger4 + Color(0x00ff00ff), // wrist -> pinky_finger1 + Color(0x00ff00ff), // pinky_finger1 -> pinky_finger2 + Color(0x00ff00ff), // pinky_finger2 -> pinky_finger3 + Color(0x00ff00ff), // pinky_finger3 -> pinky_finger4 +]; + +/// Defines the keypoint connections for the COCO-133 person skeleton with 65 connections. +/// Each tuple (a, b) represents a connection between keypoint indices a and b. +/// The connections define the following parts: +/// - Body parts: shoulders, elbows, wrists, hips, knees, ankles, toes +/// - Face: detailed face landmarks +/// - Hands: detailed finger joints for both hands +pub const SKELETON_COCO_65: [(usize, usize); 65] = [ + (15, 13), // left_ankle -> left_knee + (13, 11), // left_knee -> left_hip + (16, 14), // right_ankle -> right_knee + (14, 12), // right_knee -> right_hip + (11, 12), // left_hip -> right_hip + (5, 11), // left_shoulder -> left_hip + (6, 12), // right_shoulder -> right_hip + (5, 6), // left_shoulder -> right_shoulder + (5, 7), // left_shoulder -> left_elbow + (6, 8), // right_shoulder -> right_elbow + (7, 9), // left_elbow -> left_wrist + (8, 10), // right_elbow -> right_wrist + (1, 2), // left_eye -> right_eye + (0, 1), // nose -> left_eye + (0, 2), // nose -> right_eye + (1, 3), // left_eye -> left_ear + (2, 4), // right_eye -> right_ear + (3, 5), // left_ear -> left_shoulder + (4, 6), // right_ear -> right_shoulder + (15, 17), // left_ankle -> left_big_toe + (15, 18), // left_ankle -> left_small_toe + (15, 19), // left_ankle -> left_heel + (16, 20), // right_ankle -> right_big_toe + (16, 21), // right_ankle -> right_small_toe + (16, 22), // right_ankle -> right_heel + (91, 92), // left_hand_root -> left_thumb1 + (92, 93), // left_thumb1 -> left_thumb2 + (93, 94), // left_thumb2 -> left_thumb3 + (94, 95), // left_thumb3 -> left_thumb4 + (91, 96), // left_hand_root -> left_forefinger1 + (96, 97), // left_forefinger1 -> left_forefinger2 + (97, 98), // left_forefinger2 -> left_forefinger3 + (98, 99), // left_forefinger3 -> left_forefinger4 + (91, 100), // left_hand_root -> left_middle_finger1 + (100, 101), // left_middle_finger1 -> left_middle_finger2 + (101, 102), // left_middle_finger2 -> left_middle_finger3 + (102, 103), // left_middle_finger3 -> left_middle_finger4 + (91, 104), // left_hand_root -> left_ring_finger1 + (104, 105), // left_ring_finger1 -> left_ring_finger2 + (105, 106), // left_ring_finger2 -> left_ring_finger3 + (106, 107), // left_ring_finger3 -> left_ring_finger4 + (91, 108), // left_hand_root -> left_pinky_finger1 + (108, 109), // left_pinky_finger1 -> left_pinky_finger2 + (109, 110), // left_pinky_finger2 -> left_pinky_finger3 + (110, 111), // left_pinky_finger3 -> left_pinky_finger4 + (112, 113), // right_hand_root -> right_thumb1 + (113, 114), // right_thumb1 -> right_thumb2 + (114, 115), // right_thumb2 -> right_thumb3 + (115, 116), // right_thumb3 -> right_thumb4 + (112, 117), // right_hand_root -> right_forefinger1 + (117, 118), // right_forefinger1 -> right_forefinger2 + (118, 119), // right_forefinger2 -> right_forefinger3 + (119, 120), // right_forefinger3 -> right_forefinger4 + (112, 121), // right_hand_root -> right_middle_finger1 + (121, 122), // right_middle_finger1 -> right_middle_finger2 + (122, 123), // right_middle_finger2 -> right_middle_finger3 + (123, 124), // right_middle_finger3 -> right_middle_finger4 + (112, 125), // right_hand_root -> right_ring_finger1 + (125, 126), // right_ring_finger1 -> right_ring_finger2 + (126, 127), // right_ring_finger2 -> right_ring_finger3 + (127, 128), // right_ring_finger3 -> right_ring_finger4 + (112, 129), // right_hand_root -> right_pinky_finger1 + (129, 130), // right_pinky_finger1 -> right_pinky_finger2 + (130, 131), // right_pinky_finger2 -> right_pinky_finger3 + (131, 132), // right_pinky_finger3 -> right_pinky_finger4 +]; + +/// Defines colors for visualizing each connection in the COCO-133 person skeleton. +/// Colors are grouped by body parts: +/// - Blue (0x3399ff): Face and central body connections +/// - Green (0x00ff00): Left side body parts +/// - Orange (0xff8000): Right side body parts +/// +/// For hands: +/// - Orange (0xff8000): Thumb +/// - Pink (0xff99ff): Index finger +/// - Light Blue (0x66b2ff): Middle finger +/// - Red (0xff3333): Ring finger +/// - Green (0x00ff00): Pinky finger +pub const SKELETON_COLOR_COCO_65: [Color; 65] = [ + Color(0x00ff00ff), // left_ankle -> left_knee + Color(0x00ff00ff), // left_knee -> left_hip + Color(0xff8000ff), // right_ankle -> right_knee + Color(0xff8000ff), // right_knee -> right_hip + Color(0x3399ffff), // left_hip -> right_hip + Color(0x3399ffff), // left_shoulder -> left_hip + Color(0x3399ffff), // right_shoulder -> right_hip + Color(0x3399ffff), // left_shoulder -> right_shoulder + Color(0x00ff00ff), // left_shoulder -> left_elbow + Color(0xff8000ff), // right_shoulder -> right_elbow + Color(0x00ff00ff), // left_elbow -> left_wrist + Color(0xff8000ff), // right_elbow -> right_wrist + Color(0x3399ffff), // left_eye -> right_eye + Color(0x3399ffff), // nose -> left_eye + Color(0x3399ffff), // nose -> right_eye + Color(0x3399ffff), // left_eye -> left_ear + Color(0x3399ffff), // right_eye -> right_ear + Color(0x3399ffff), // left_ear -> left_shoulder + Color(0x3399ffff), // right_ear -> right_shoulder + Color(0x00ff00ff), // left_ankle -> left_big_toe + Color(0x00ff00ff), // left_ankle -> left_small_toe + Color(0x00ff00ff), // left_ankle -> left_heel + Color(0xff8000ff), // right_ankle -> right_big_toe + Color(0xff8000ff), // right_ankle -> right_small_toe + Color(0xff8000ff), // right_ankle -> right_heel + // Left hand + Color(0xff8000ff), // left_thumb connections + Color(0xff8000ff), + Color(0xff8000ff), + Color(0xff8000ff), + Color(0xff99ffff), // left_forefinger connections + Color(0xff99ffff), + Color(0xff99ffff), + Color(0xff99ffff), + Color(0x66b2ffff), // left_middle_finger connections + Color(0x66b2ffff), + Color(0x66b2ffff), + Color(0x66b2ffff), + Color(0xff3333ff), // left_ring_finger connections + Color(0xff3333ff), + Color(0xff3333ff), + Color(0xff3333ff), + Color(0x00ff00ff), // left_pinky_finger connections + Color(0x00ff00ff), + Color(0x00ff00ff), + Color(0x00ff00ff), + // Right hand + Color(0xff8000ff), // right_thumb connections + Color(0xff8000ff), + Color(0xff8000ff), + Color(0xff8000ff), + Color(0xff99ffff), // right_forefinger connections + Color(0xff99ffff), + Color(0xff99ffff), + Color(0xff99ffff), + Color(0x66b2ffff), // right_middle_finger connections + Color(0x66b2ffff), + Color(0x66b2ffff), + Color(0x66b2ffff), + Color(0xff3333ff), // right_ring_finger connections + Color(0xff3333ff), + Color(0xff3333ff), + Color(0xff3333ff), + Color(0x00ff00ff), // right_pinky_finger connections + Color(0x00ff00ff), + Color(0x00ff00ff), + Color(0x00ff00ff), +];