Add tracked-target for DNN pipeline

Now shows up in the UI!

build.gradle

@@ -4,7 +4,7 @@ plugins {
     id "com.github.node-gradle.node" version "3.1.1" apply false
     id "edu.wpi.first.GradleJni" version "1.1.0"
     id "edu.wpi.first.GradleVsCode" version "1.3.0"
-    id "edu.wpi.first.NativeUtils" version "2023.11.1" apply false
+    id "edu.wpi.first.NativeUtils" version "2024.2.0" apply false
     id "edu.wpi.first.wpilib.repositories.WPILibRepositoriesPlugin" version "2020.2"
     id "org.hidetake.ssh" version "2.10.1"
     id 'edu.wpi.first.WpilibTools' version '1.1.0'

photon-client/src/types/PipelineTypes.ts

@@ -6,7 +6,7 @@ export enum PipelineType {
   ColoredShape = 3,
   AprilTag = 4,
   Aruco = 5,
-  Dnn = 6,
+  Dnn = 6
 }
 
 export enum AprilTagFamily {
@@ -250,7 +250,6 @@ export const DefaultDnnPipelineSettings: DnnPipelineSettings = {
   pipelineType: PipelineType.Dnn
 };
 
-
 export interface ArucoPipelineSettings extends PipelineSettings {
   pipelineType: PipelineType.Aruco;
   decimate: number;

photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java

@@ -38,6 +38,7 @@
 import org.photonvision.vision.pipe.impl.*;
 import org.photonvision.vision.pipeline.result.CVPipelineResult;
 import org.photonvision.vision.target.TrackedTarget;
+import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters;
 
 public class DnnPipeline extends CVPipeline<CVPipelineResult, DnnPipelineSettings> {
     private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();
@@ -67,7 +68,7 @@ public DnnPipeline(DnnPipelineSettings settings) {
                     Dnn.readNetFromDarknet(
                             "/home/matt/Downloads/yolov4-csp-swish.cfg",
                             "/home/matt/Downloads/yolov4-csp-swish.weights");
-            Core.setNumThreads(4);
+            Core.setNumThreads(8);
         } catch (Exception e) {
             System.out.println(e);
         }
@@ -161,9 +162,6 @@ public DnnPipeline(DnnPipelineSettings settings) {
     protected void setPipeParamsImpl() {}
 
     private static List<String> getOutputNames(Net net) {
-        // return new
-        // ArrayList<>(List.of("person","bicycle","car","motorcycle","airplane","bus","train","truck","boat","traffic light","fire hydrant","stop sign","parking meter","bench","bird","cat","dog","horse","sheep","cow","elephant","bear","zebra","giraffe","backpack","umbrella","handbag","tie","suitcase","frisbee","skis","snowboard","sports ball","kite","baseball bat","baseball glove","skateboard","surfboard","tennis racket","bottle","wine glass","cup","fork","knife","spoon","bowl","banana","apple","sandwich","orange","broccoli","carrot","hot dog","pizza","donut","cake","chair","couch","potted plant","bed","dining table","toilet","tv","laptop","mouse","remote","keyboard","cell phone","microwave","oven","toaster","sink","refrigerator","book","clock","vase","scissors","teddy bear","hair drier","toothbrush"));
-
         List<String> names = new ArrayList<>();
 
         List<Integer> outLayers = net.getUnconnectedOutLayers().toList();
@@ -179,14 +177,12 @@ private static List<String> getOutputNames(Net net) {
     protected CVPipelineResult process(Frame input_frame, DnnPipelineSettings settings) {
         long sumPipeNanosElapsed = 0L;
 
-        List<TrackedTarget> targetList = List.of();
-
         // ======================
 
         var frame = input_frame.colorImage.getMat();
 
         if (frame.empty()) {
-            return new CVPipelineResult(sumPipeNanosElapsed, 0, targetList, input_frame);
+            return new CVPipelineResult(sumPipeNanosElapsed, 0, List.of(), input_frame);
         }
 
         var blob = Dnn.blobFromImage(frame, 1.0 / 255.0, new Size(640, 640));
@@ -239,21 +235,32 @@ protected CVPipelineResult process(Frame input_frame, DnnPipelineSettings settin
                 indices); // We draw the bounding boxes for objects
         // here//
 
+        List<TrackedTarget> targetList = new ArrayList<>();
+
         int[] ind = indices.toArray();
         for (int i = 0; i < ind.length; ++i) {
             int idx = ind[i];
             var box = boxesArray[idx];
             Imgproc.rectangle(frame, box.tl(), box.br(), new Scalar(0, 0, 255), 2);
+
+            var name = String.format("%s (%f)", coco_names.get(clsIds.get(idx)), confs.get(idx));
+
             Imgproc.putText(
                     frame,
-                    coco_names.get(clsIds.get(idx)),
-                    box.br(),
+                    name,
+                    new Point(box.x + box.width / 2.0, box.y + box.height / 2.0),
                     0,
                     0.6,
                     ColorHelper.colorToScalar(java.awt.Color.white),
                     2);
 
-            // System.out.println(idx);
+            targetList.add(
+                    new TrackedTarget(
+                            box,
+                            clsIds.get(idx),
+                            confs.get(idx),
+                            new TargetCalculationParameters(
+                                    false, null, null, null, null, frameStaticProperties)));
         }
 
         // ======================

photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java

@@ -314,6 +314,12 @@ private CVPipelineSettings createSettingsForType(PipelineType type, String nickn
                     added.pipelineNickname = nickname;
                     return added;
                 }
+            case Dnn:
+                {
+                    var added = new DnnPipelineSettings();
+                    added.pipelineNickname = nickname;
+                    return added;
+                }
             default:
                 {
                     logger.error("Got invalid pipeline type: " + type.toString());

photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java

@@ -29,6 +29,7 @@
 import org.opencv.core.MatOfPoint;
 import org.opencv.core.MatOfPoint2f;
 import org.opencv.core.Point;
+import org.opencv.core.Rect2d;
 import org.opencv.core.RotatedRect;
 import org.photonvision.common.util.math.MathUtils;
 import org.photonvision.vision.aruco.ArucoDetectionResult;
@@ -192,6 +193,37 @@ public TrackedTarget(ArucoDetectionResult result, TargetCalculationParameters pa
         }
     }
 
+    public TrackedTarget(
+            Rect2d box, int class_id, double confidence, TargetCalculationParameters params) {
+        m_targetOffsetPoint = new Point(box.x, box.y);
+        m_robotOffsetPoint = new Point();
+
+        m_pitch =
+                TargetCalculations.calculatePitch(
+                        box.y + box.height / 2.0, params.cameraCenterPoint.y, params.verticalFocalLength);
+        m_yaw =
+                TargetCalculations.calculateYaw(
+                        box.x + box.width / 2.0, params.cameraCenterPoint.x, params.horizontalFocalLength);
+
+        Point[] cornerPoints =
+                new Point[] {
+                    // Box.x/y is the top-left corner, not the center
+                    new Point(box.x, box.y), // tl
+                    new Point(box.x + box.width, box.y), // tr
+                    new Point(box.x + box.width, box.y + box.height), // br
+                    new Point(box.x, box.y + box.height), // bl
+                };
+
+        m_targetCorners = List.of(cornerPoints);
+        MatOfPoint contourMat = new MatOfPoint(cornerPoints);
+        m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints);
+
+        m_mainContour = new Contour(contourMat);
+        m_area = m_mainContour.getArea() / params.imageArea * 100;
+
+        m_fiducialId = class_id;
+    }
+
     public void setFiducialId(int id) {
         m_fiducialId = id;
     }

shared/config.gradle

@@ -9,8 +9,9 @@ nativeUtils.wpi.configureDependencies {
     wpiVersion = wpilibVersion
     wpimathVersion = wpilibVersion
     niLibVersion = "2023.3.0"
+    opencvYear = 'frc2024'
     opencvVersion = "4.8.0-1"
-    googleTestVersion = "1.12.1-1"
+    googleTestVersion = "1.12.1-2"
     imguiVersion = "1.86-1"
 }