diff --git a/build.gradle b/build.gradle index 2714e145ea..0225b7003e 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ plugins { id "com.github.node-gradle.node" version "3.1.1" apply false id "edu.wpi.first.GradleJni" version "1.1.0" id "edu.wpi.first.GradleVsCode" version "1.3.0" - id "edu.wpi.first.NativeUtils" version "2023.11.1" apply false + id "edu.wpi.first.NativeUtils" version "2024.2.0" apply false id "edu.wpi.first.wpilib.repositories.WPILibRepositoriesPlugin" version "2020.2" id "org.hidetake.ssh" version "2.10.1" id 'edu.wpi.first.WpilibTools' version '1.1.0' diff --git a/photon-client/src/types/PipelineTypes.ts b/photon-client/src/types/PipelineTypes.ts index ee09af6e5b..4b58528169 100644 --- a/photon-client/src/types/PipelineTypes.ts +++ b/photon-client/src/types/PipelineTypes.ts @@ -6,7 +6,7 @@ export enum PipelineType { ColoredShape = 3, AprilTag = 4, Aruco = 5, - Dnn = 6, + Dnn = 6 } export enum AprilTagFamily { @@ -250,7 +250,6 @@ export const DefaultDnnPipelineSettings: DnnPipelineSettings = { pipelineType: PipelineType.Dnn }; - export interface ArucoPipelineSettings extends PipelineSettings { pipelineType: PipelineType.Aruco; decimate: number; diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java index 0a3ad28cbf..f6156c79a8 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java @@ -38,6 +38,7 @@ import org.photonvision.vision.pipe.impl.*; import org.photonvision.vision.pipeline.result.CVPipelineResult; import org.photonvision.vision.target.TrackedTarget; +import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters; public class DnnPipeline extends CVPipeline { private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe(); @@ -67,7 +68,7 @@ public DnnPipeline(DnnPipelineSettings settings) { Dnn.readNetFromDarknet( "/home/matt/Downloads/yolov4-csp-swish.cfg", "/home/matt/Downloads/yolov4-csp-swish.weights"); - Core.setNumThreads(4); + Core.setNumThreads(8); } catch (Exception e) { System.out.println(e); } @@ -161,9 +162,6 @@ public DnnPipeline(DnnPipelineSettings settings) { protected void setPipeParamsImpl() {} private static List getOutputNames(Net net) { - // return new - // ArrayList<>(List.of("person","bicycle","car","motorcycle","airplane","bus","train","truck","boat","traffic light","fire hydrant","stop sign","parking meter","bench","bird","cat","dog","horse","sheep","cow","elephant","bear","zebra","giraffe","backpack","umbrella","handbag","tie","suitcase","frisbee","skis","snowboard","sports ball","kite","baseball bat","baseball glove","skateboard","surfboard","tennis racket","bottle","wine glass","cup","fork","knife","spoon","bowl","banana","apple","sandwich","orange","broccoli","carrot","hot dog","pizza","donut","cake","chair","couch","potted plant","bed","dining table","toilet","tv","laptop","mouse","remote","keyboard","cell phone","microwave","oven","toaster","sink","refrigerator","book","clock","vase","scissors","teddy bear","hair drier","toothbrush")); - List names = new ArrayList<>(); List outLayers = net.getUnconnectedOutLayers().toList(); @@ -179,14 +177,12 @@ private static List getOutputNames(Net net) { protected CVPipelineResult process(Frame input_frame, DnnPipelineSettings settings) { long sumPipeNanosElapsed = 0L; - List targetList = List.of(); - // ====================== var frame = input_frame.colorImage.getMat(); if (frame.empty()) { - return new CVPipelineResult(sumPipeNanosElapsed, 0, targetList, input_frame); + return new CVPipelineResult(sumPipeNanosElapsed, 0, List.of(), input_frame); } var blob = Dnn.blobFromImage(frame, 1.0 / 255.0, new Size(640, 640)); @@ -239,21 +235,32 @@ protected CVPipelineResult process(Frame input_frame, DnnPipelineSettings settin indices); // We draw the bounding boxes for objects // here// + List targetList = new ArrayList<>(); + int[] ind = indices.toArray(); for (int i = 0; i < ind.length; ++i) { int idx = ind[i]; var box = boxesArray[idx]; Imgproc.rectangle(frame, box.tl(), box.br(), new Scalar(0, 0, 255), 2); + + var name = String.format("%s (%f)", coco_names.get(clsIds.get(idx)), confs.get(idx)); + Imgproc.putText( frame, - coco_names.get(clsIds.get(idx)), - box.br(), + name, + new Point(box.x + box.width / 2.0, box.y + box.height / 2.0), 0, 0.6, ColorHelper.colorToScalar(java.awt.Color.white), 2); - // System.out.println(idx); + targetList.add( + new TrackedTarget( + box, + clsIds.get(idx), + confs.get(idx), + new TargetCalculationParameters( + false, null, null, null, null, frameStaticProperties))); } // ====================== diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java b/photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java index f126a32892..16eb189155 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java @@ -314,6 +314,12 @@ private CVPipelineSettings createSettingsForType(PipelineType type, String nickn added.pipelineNickname = nickname; return added; } + case Dnn: + { + var added = new DnnPipelineSettings(); + added.pipelineNickname = nickname; + return added; + } default: { logger.error("Got invalid pipeline type: " + type.toString()); diff --git a/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java b/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java index 4bb7997047..7b5bf41131 100644 --- a/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java +++ b/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java @@ -29,6 +29,7 @@ import org.opencv.core.MatOfPoint; import org.opencv.core.MatOfPoint2f; import org.opencv.core.Point; +import org.opencv.core.Rect2d; import org.opencv.core.RotatedRect; import org.photonvision.common.util.math.MathUtils; import org.photonvision.vision.aruco.ArucoDetectionResult; @@ -192,6 +193,37 @@ public TrackedTarget(ArucoDetectionResult result, TargetCalculationParameters pa } } + public TrackedTarget( + Rect2d box, int class_id, double confidence, TargetCalculationParameters params) { + m_targetOffsetPoint = new Point(box.x, box.y); + m_robotOffsetPoint = new Point(); + + m_pitch = + TargetCalculations.calculatePitch( + box.y + box.height / 2.0, params.cameraCenterPoint.y, params.verticalFocalLength); + m_yaw = + TargetCalculations.calculateYaw( + box.x + box.width / 2.0, params.cameraCenterPoint.x, params.horizontalFocalLength); + + Point[] cornerPoints = + new Point[] { + // Box.x/y is the top-left corner, not the center + new Point(box.x, box.y), // tl + new Point(box.x + box.width, box.y), // tr + new Point(box.x + box.width, box.y + box.height), // br + new Point(box.x, box.y + box.height), // bl + }; + + m_targetCorners = List.of(cornerPoints); + MatOfPoint contourMat = new MatOfPoint(cornerPoints); + m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints); + + m_mainContour = new Contour(contourMat); + m_area = m_mainContour.getArea() / params.imageArea * 100; + + m_fiducialId = class_id; + } + public void setFiducialId(int id) { m_fiducialId = id; } diff --git a/shared/config.gradle b/shared/config.gradle index 7cf75d5193..401d2b7195 100644 --- a/shared/config.gradle +++ b/shared/config.gradle @@ -9,8 +9,9 @@ nativeUtils.wpi.configureDependencies { wpiVersion = wpilibVersion wpimathVersion = wpilibVersion niLibVersion = "2023.3.0" + opencvYear = 'frc2024' opencvVersion = "4.8.0-1" - googleTestVersion = "1.12.1-1" + googleTestVersion = "1.12.1-2" imguiVersion = "1.86-1" }