Add GroundingDINO (#30)

examples/blip/main.rs

@@ -10,7 +10,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // textual
     let options_textual = Options::default()
         .with_model("blip-textual-base.onnx")?
-        .with_tokenizer("tokenizer-blip.json")?
+        // .with_tokenizer("tokenizer-blip.json")?
         .with_i00((1, 1, 4).into()) // input_id: batch
         .with_i01((1, 1, 4).into()) // input_id: seq_len
         .with_i10((1, 1, 4).into()) // attention_mask: batch
@@ -23,9 +23,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut model = Blip::new(options_visual, options_textual)?;
 
     // image caption (this demo use batch_size=1)
-    let x = vec![DataLoader::try_read("./assets/bus.jpg")?];
-    let _y = model.caption(&x, None, true)?; // unconditional
-    let y = model.caption(&x, Some("three man"), true)?; // conditional
+    let xs = vec![DataLoader::try_read("./assets/bus.jpg")?];
+    let image_embeddings = model.encode_images(&xs)?;
+    let _y = model.caption(&image_embeddings, None, true)?; // unconditional
+    let y = model.caption(&image_embeddings, Some("three man"), true)?; // conditional
     println!("{:?}", y[0].texts());
 
     Ok(())

examples/clip/main.rs

@@ -10,7 +10,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // textual
     let options_textual = Options::default()
         .with_model("clip-b32-textual-dyn.onnx")?
-        .with_tokenizer("tokenizer-clip.json")?
+        // .with_tokenizer("tokenizer-clip.json")?
         .with_i00((1, 1, 4).into())
         .with_profile(false);
 

examples/grounding-dino/main.rs

@@ -0,0 +1,40 @@
+use usls::{models::GroundingDINO, Annotator, DataLoader, Options};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let opts = Options::default()
+        .with_i00((1, 1, 4).into())
+        .with_i02((640, 800, 1200).into())
+        .with_i03((640, 1200, 1200).into())
+        .with_i10((1, 1, 4).into())
+        .with_i11((256, 256, 512).into())
+        .with_i20((1, 1, 4).into())
+        .with_i21((256, 256, 512).into())
+        .with_i30((1, 1, 4).into())
+        .with_i31((256, 256, 512).into())
+        .with_i40((1, 1, 4).into())
+        .with_i41((256, 256, 512).into())
+        .with_i50((1, 1, 4).into())
+        .with_i51((256, 256, 512).into())
+        .with_i52((256, 256, 512).into())
+        .with_model("groundingdino-swint-ogc-dyn-u8.onnx")? // TODO: current onnx model does not support bs > 1
+        // .with_model("groundingdino-swint-ogc-dyn-f32.onnx")?
+        .with_confs(&[0.2])
+        .with_profile(false);
+    let mut model = GroundingDINO::new(opts)?;
+
+    // Load images and set class names
+    let x = [DataLoader::try_read("./assets/bus.jpg")?];
+    let texts = [
+        "person", "hand", "shoes", "bus", "dog", "cat", "sign", "tie", "monitor", "window",
+        "glasses", "tree", "head",
+    ];
+
+    // Run and annotate
+    let y = model.run(&x, &texts)?;
+    let annotator = Annotator::default()
+        .with_bboxes_thickness(4)
+        .with_saveout("GroundingDINO");
+    annotator.annotate(&x, &y);
+
+    Ok(())
+}

examples/sam/main.rs

@@ -99,7 +99,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut model = SAM::new(options_encoder, options_decoder)?;
 
     // Load image
-    let xs = vec![DataLoader::try_read("./assets/truck.jpg")?];
+    let xs = [
+        DataLoader::try_read("./assets/truck.jpg")?,
+        // DataLoader::try_read("./assets/dog.jpg")?,
+    ];
 
     // Build annotator
     let annotator = Annotator::default().with_saveout(saveout);