From 8faa3918c2faf31b2ec425525f264e986234f663 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 20 Dec 2023 09:06:08 -0500 Subject: [PATCH] Add dnn tab and start poking at backend --- .../components/dashboard/ConfigOptions.vue | 17 +- .../src/components/dashboard/tabs/DnnTab.vue | 48 +++ .../org/photonvision/vision/pipe/CVPipe.java | 3 + .../pipe/impl/NeuralNetworkPipeResult.java | 14 + .../vision/pipe/impl/OpencvDnnPipe.java | 265 +++++++++++++++++ .../vision/pipe/impl/RknnDetectionPipe.java | 22 ++ .../vision/pipeline/DnnPipeline.java | 273 ------------------ .../vision/pipeline/DnnPipelineSettings.java | 12 +- .../vision/pipeline/OpencvDnnPipeline.java | 109 +++++++ .../vision/pipeline/PipelineType.java | 2 +- .../vision/processes/PipelineManager.java | 2 +- .../vision/target/TrackedTarget.java | 22 +- 12 files changed, 502 insertions(+), 287 deletions(-) create mode 100644 photon-client/src/components/dashboard/tabs/DnnTab.vue create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipe/impl/OpencvDnnPipe.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java delete mode 100644 photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipeline/OpencvDnnPipeline.java diff --git a/photon-client/src/components/dashboard/ConfigOptions.vue b/photon-client/src/components/dashboard/ConfigOptions.vue index 6ed0156f9f..2ae604a859 100644 --- a/photon-client/src/components/dashboard/ConfigOptions.vue +++ b/photon-client/src/components/dashboard/ConfigOptions.vue @@ -12,6 +12,7 @@ import OutputTab from "@/components/dashboard/tabs/OutputTab.vue"; import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue"; import PnPTab from "@/components/dashboard/tabs/PnPTab.vue"; import Map3DTab from "@/components/dashboard/tabs/Map3DTab.vue"; +import DnnTab from "@/components/dashboard/tabs/DnnTab.vue"; import { WebsocketPipelineType } from "@/types/WebsocketDataTypes"; interface ConfigOption { @@ -55,7 +56,11 @@ const allTabs = Object.freeze({ map3dTab: { tabName: "3D", component: Map3DTab - } + }, + dnnTab: { + tabName: "DNN", + component: DnnTab + }, }); const selectedTabs = ref([0, 0, 0, 0]); @@ -75,6 +80,7 @@ const getTabGroups = (): ConfigOption[][] => { allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, + allTabs.dnnTab, allTabs.outputTab ], [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab] @@ -82,14 +88,14 @@ const getTabGroups = (): ConfigOption[][] => { } else if (lgAndDown) { return [ [allTabs.inputTab], - [allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab], + [allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.dnnTab, allTabs.outputTab], [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab] ]; } else if (xl) { return [ [allTabs.inputTab], [allTabs.thresholdTab], - [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab], + [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.dnnTab, allTabs.outputTab], [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab] ]; } @@ -103,6 +109,7 @@ const tabGroups = computed(() => { const allow3d = useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled; const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag; const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco; + const isDNN = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Dnn; return getTabGroups() .map((tabGroup) => @@ -110,8 +117,8 @@ const tabGroups = computed(() => { (tabConfig) => !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated !((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags - !((isAprilTag || isAruco) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags - !((isAprilTag || isAruco) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags + !((isAprilTag || isAruco || isDNN) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags + !((isAprilTag || isAruco || isDNN) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags !(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco ) diff --git a/photon-client/src/components/dashboard/tabs/DnnTab.vue b/photon-client/src/components/dashboard/tabs/DnnTab.vue new file mode 100644 index 0000000000..f13947dd41 --- /dev/null +++ b/photon-client/src/components/dashboard/tabs/DnnTab.vue @@ -0,0 +1,48 @@ + + + diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/CVPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/CVPipe.java index d941f51cd0..31faa94836 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipe/CVPipe.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/CVPipe.java @@ -32,6 +32,9 @@ public abstract class CVPipe { public void setParams(P params) { this.params = params; } + public P getParams() { + return this.params; + } /** * Runs the process for the pipe. diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java new file mode 100644 index 0000000000..aef5e2714c --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java @@ -0,0 +1,14 @@ +package org.photonvision.vision.pipe.impl; + +import org.opencv.core.Rect2d; + +public class NeuralNetworkPipeResult { + public NeuralNetworkPipeResult(Rect2d box2, Integer integer, Float float1) { + box = box2; + classIdx = integer; + confidence = float1; + } + public final int classIdx; + public final Rect2d box; + public final double confidence; +} diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/OpencvDnnPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/OpencvDnnPipe.java new file mode 100644 index 0000000000..ca81d6f2d0 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/OpencvDnnPipe.java @@ -0,0 +1,265 @@ +package org.photonvision.vision.pipe.impl; + +import java.util.ArrayList; +import java.util.List; + +import org.opencv.core.Core; +import org.opencv.core.Mat; +import org.opencv.core.MatOfFloat; +import org.opencv.core.MatOfInt; +import org.opencv.core.MatOfRect2d; +import org.opencv.core.Point; +import org.opencv.core.Rect2d; +import org.opencv.core.Size; +import org.opencv.dnn.Dnn; +import org.opencv.dnn.Net; +import org.opencv.utils.Converters; +import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.opencv.Releasable; +import org.photonvision.vision.pipe.CVPipe; + +public class OpencvDnnPipe extends CVPipe, OpencvDnnPipe.OpencvDnnPipeParams> { + + @Override + protected List process(CVMat in) { + if (params.net == null) { + // barf? + return List.of(); + } + + var frame = in.getMat(); + + if (frame.empty()) { + return List.of(); + } + + var blob = Dnn.blobFromImage(frame, 1.0 / 255.0, new Size(640, 640)); + params.net.setInput(blob); + + List result = new ArrayList<>(); + params.net.forward(result, params.outBlobNames); // outputlayer : output1 and output2 + + // From + // https://github.com/suddh123/YOLO-object-detection-in-java/blob/code/yolo.java + + float confThreshold = (float) params.confidence; // Insert thresholding beyond which the model will detect + // objects// + List clsIds = new ArrayList<>(); + List confs = new ArrayList<>(); + List rects = new ArrayList<>(); + for (int i = 0; i < result.size(); ++i) { + // each row is a candidate detection, the 1st 4 numbers are + // [center_x, center_y, width, height], followed by (N-4) class probabilities + Mat level = result.get(i); + for (int j = 0; j < level.rows(); ++j) { + Mat row = level.row(j); + Mat scores = row.colRange(5, level.cols()); + Core.MinMaxLocResult mm = Core.minMaxLoc(scores); + float confidence = (float) mm.maxVal; + Point classIdPoint = mm.maxLoc; + if (confidence > confThreshold) { + // scaling for drawing the bounding boxes// + int centerX = (int) (row.get(0, 0)[0] * frame.cols()); + int centerY = (int) (row.get(0, 1)[0] * frame.rows()); + int width = (int) (row.get(0, 2)[0] * frame.cols()); + int height = (int) (row.get(0, 3)[0] * frame.rows()); + int left = centerX - width / 2; + int top = centerY - height / 2; + + clsIds.add((int) classIdPoint.x); + confs.add((float) confidence); + rects.add(new Rect2d(left, top, width, height)); + } + } + } + float nmsThresh = 0.5f; + MatOfFloat confidences = new MatOfFloat(Converters.vector_float_to_Mat(confs)); + Rect2d[] boxesArray = rects.toArray(new Rect2d[0]); + MatOfRect2d boxes = new MatOfRect2d(boxesArray); + MatOfInt indices = new MatOfInt(); + Dnn.NMSBoxes( + boxes, + confidences, + confThreshold, + nmsThresh, + indices); + + List targetList = new ArrayList<>(); + + int[] ind = indices.toArray(); + for (int i = 0; i < ind.length; ++i) { + int idx = ind[i]; + Rect2d box = boxesArray[idx]; + + targetList.add(new NeuralNetworkPipeResult(box, clsIds.get(idx), confs.get(idx))); + } + + return targetList; + } + + @Override + public void setParams(OpencvDnnPipeParams newParams) { + // net/blob names/class names are all reset here if necessary + if (params.modelPath != newParams.modelPath) { + this.params.release(); + super.setParams(params); + } + + // Otherwise, update things over manually (ew) + params.confidence = newParams.confidence; + } + + public static class OpencvDnnPipeParams implements Releasable { + String modelPath; + private PhotonNet net; + private List outBlobNames = List.of(); + private List classNames; + + public float confidence; + + public OpencvDnnPipeParams(String modelPath) { + this.modelPath = modelPath; + // from https://dev.to/kojix2/yolov7-object-detection-in-ruby-in-10-minutes-5cjh + // https://s3.ap-northeast-2.wasabisys.com/pinto-model-zoo/307_YOLOv7/with-postprocess/resources_post.tar.gz + try { + // this.net = Dnn.readNetFromONNX("/home/matt/Downloads/best_1.onnx"); + // this.net = Dnn.readNet("/home/matt/Downloads/yolov7_post_640x640.onnx"); + var net = Dnn.readNetFromDarknet( + "/home/matt/Downloads/yolov4-csp-swish.cfg", + "/home/matt/Downloads/yolov4-csp-swish.weights"); + this.net = new PhotonNet(net.getNativeObjAddr()); + Core.setNumThreads(8); + } catch (Exception e) { + System.out.println(e); + } + this.outBlobNames = getOutputNames(net); + + this.classNames = List.of( + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush"); + } + + private static List getOutputNames(Net net) { + if (net == null) { + // barf? + return List.of(); + } + + List names = new ArrayList<>(); + + List outLayers = net.getUnconnectedOutLayers().toList(); + List layersNames = net.getLayerNames(); + + outLayers.forEach( + (item) -> names.add(layersNames.get(item - 1))); // unfold and create R-CNN layers from the + // loaded YOLO model// + return names; + } + + @Override + public void release() { + if (net != null) net.release(); + } + + public List getClassNames() { + return classNames; + } + } + + // Hack so we can see the delete function + private static class PhotonNet extends Net implements Releasable { + public PhotonNet(long nativeObjAddr) { + super(nativeObjAddr); + } + + @Override + public void release() { + // This relies on opencv calling their private delete from finalize + try { + finalize(); + } catch (Throwable e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java new file mode 100644 index 0000000000..08c8534847 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java @@ -0,0 +1,22 @@ +package org.photonvision.vision.pipe.impl; + +import java.util.List; + +import org.opencv.dnn.Net; +import org.photonvision.vision.opencv.CVMat; +import org.photonvision.vision.pipe.CVPipe; +import org.photonvision.vision.target.TrackedTarget; + +public class RknnDetectionPipe extends CVPipe, RknnDetectionPipe.RknnDetectionPipeParams> { + + @Override + protected List process(CVMat in) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'process'"); + } + + public static class RknnDetectionPipeParams { + + } + +} diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java deleted file mode 100644 index f6156c79a8..0000000000 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipeline.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright (C) Photon Vision. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.photonvision.vision.pipeline; - -import java.util.ArrayList; -import java.util.List; -import org.opencv.core.Core; -import org.opencv.core.Mat; -import org.opencv.core.MatOfFloat; -import org.opencv.core.MatOfInt; -import org.opencv.core.MatOfRect2d; -import org.opencv.core.Point; -import org.opencv.core.Rect2d; -import org.opencv.core.Scalar; -import org.opencv.core.Size; -import org.opencv.dnn.Dnn; -import org.opencv.dnn.Net; -import org.opencv.imgproc.Imgproc; -import org.opencv.utils.Converters; -import org.photonvision.common.util.ColorHelper; -import org.photonvision.vision.frame.Frame; -import org.photonvision.vision.frame.FrameThresholdType; -import org.photonvision.vision.pipe.impl.*; -import org.photonvision.vision.pipeline.result.CVPipelineResult; -import org.photonvision.vision.target.TrackedTarget; -import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters; - -public class DnnPipeline extends CVPipeline { - private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe(); - - private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE; - - public DnnPipeline() { - super(PROCESSING_TYPE); - settings = new DnnPipelineSettings(); - } - - Net net = null; - private List outBlobNames = List.of(); - - private List coco_names; - - public DnnPipeline(DnnPipelineSettings settings) { - super(PROCESSING_TYPE); - this.settings = settings; - - // Downloaded from https://dev.to/kojix2/yolov7-object-detection-in-ruby-in-10-minutes-5cjh - // https://s3.ap-northeast-2.wasabisys.com/pinto-model-zoo/307_YOLOv7/with-postprocess/resources_post.tar.gz - try { - // this.net = Dnn.readNetFromONNX("/home/matt/Downloads/best_1.onnx"); - // this.net = Dnn.readNet("/home/matt/Downloads/yolov7_post_640x640.onnx"); - this.net = - Dnn.readNetFromDarknet( - "/home/matt/Downloads/yolov4-csp-swish.cfg", - "/home/matt/Downloads/yolov4-csp-swish.weights"); - Core.setNumThreads(8); - } catch (Exception e) { - System.out.println(e); - } - this.outBlobNames = getOutputNames(net); - - this.coco_names = - List.of( - "person", - "bicycle", - "car", - "motorcycle", - "airplane", - "bus", - "train", - "truck", - "boat", - "traffic light", - "fire hydrant", - "stop sign", - "parking meter", - "bench", - "bird", - "cat", - "dog", - "horse", - "sheep", - "cow", - "elephant", - "bear", - "zebra", - "giraffe", - "backpack", - "umbrella", - "handbag", - "tie", - "suitcase", - "frisbee", - "skis", - "snowboard", - "sports ball", - "kite", - "baseball bat", - "baseball glove", - "skateboard", - "surfboard", - "tennis racket", - "bottle", - "wine glass", - "cup", - "fork", - "knife", - "spoon", - "bowl", - "banana", - "apple", - "sandwich", - "orange", - "broccoli", - "carrot", - "hot dog", - "pizza", - "donut", - "cake", - "chair", - "couch", - "potted plant", - "bed", - "dining table", - "toilet", - "tv", - "laptop", - "mouse", - "remote", - "keyboard", - "cell phone", - "microwave", - "oven", - "toaster", - "sink", - "refrigerator", - "book", - "clock", - "vase", - "scissors", - "teddy bear", - "hair drier", - "toothbrush"); - } - - @Override - protected void setPipeParamsImpl() {} - - private static List getOutputNames(Net net) { - List names = new ArrayList<>(); - - List outLayers = net.getUnconnectedOutLayers().toList(); - List layersNames = net.getLayerNames(); - - outLayers.forEach( - (item) -> names.add(layersNames.get(item - 1))); // unfold and create R-CNN layers from the - // loaded YOLO model// - return names; - } - - @Override - protected CVPipelineResult process(Frame input_frame, DnnPipelineSettings settings) { - long sumPipeNanosElapsed = 0L; - - // ====================== - - var frame = input_frame.colorImage.getMat(); - - if (frame.empty()) { - return new CVPipelineResult(sumPipeNanosElapsed, 0, List.of(), input_frame); - } - - var blob = Dnn.blobFromImage(frame, 1.0 / 255.0, new Size(640, 640)); - net.setInput(blob); - - List result = new ArrayList<>(); - net.forward(result, outBlobNames); // outputlayer : output1 and output2 - - // From https://github.com/suddh123/YOLO-object-detection-in-java/blob/code/yolo.java - - float confThreshold = 0.3f; // Insert thresholding beyond which the model will detect objects// - List clsIds = new ArrayList<>(); - List confs = new ArrayList<>(); - List rects = new ArrayList<>(); - for (int i = 0; i < result.size(); ++i) { - // each row is a candidate detection, the 1st 4 numbers are - // [center_x, center_y, width, height], followed by (N-4) class probabilities - Mat level = result.get(i); - for (int j = 0; j < level.rows(); ++j) { - Mat row = level.row(j); - Mat scores = row.colRange(5, level.cols()); - Core.MinMaxLocResult mm = Core.minMaxLoc(scores); - float confidence = (float) mm.maxVal; - Point classIdPoint = mm.maxLoc; - if (confidence > confThreshold) { - // scaling for drawing the bounding boxes// - int centerX = (int) (row.get(0, 0)[0] * frame.cols()); - int centerY = (int) (row.get(0, 1)[0] * frame.rows()); - int width = (int) (row.get(0, 2)[0] * frame.cols()); - int height = (int) (row.get(0, 3)[0] * frame.rows()); - int left = centerX - width / 2; - int top = centerY - height / 2; - - clsIds.add((int) classIdPoint.x); - confs.add((float) confidence); - rects.add(new Rect2d(left, top, width, height)); - } - } - } - float nmsThresh = 0.5f; - MatOfFloat confidences = new MatOfFloat(Converters.vector_float_to_Mat(confs)); - Rect2d[] boxesArray = rects.toArray(new Rect2d[0]); - MatOfRect2d boxes = new MatOfRect2d(boxesArray); - MatOfInt indices = new MatOfInt(); - Dnn.NMSBoxes( - boxes, - confidences, - confThreshold, - nmsThresh, - indices); // We draw the bounding boxes for objects - // here// - - List targetList = new ArrayList<>(); - - int[] ind = indices.toArray(); - for (int i = 0; i < ind.length; ++i) { - int idx = ind[i]; - var box = boxesArray[idx]; - Imgproc.rectangle(frame, box.tl(), box.br(), new Scalar(0, 0, 255), 2); - - var name = String.format("%s (%f)", coco_names.get(clsIds.get(idx)), confs.get(idx)); - - Imgproc.putText( - frame, - name, - new Point(box.x + box.width / 2.0, box.y + box.height / 2.0), - 0, - 0.6, - ColorHelper.colorToScalar(java.awt.Color.white), - 2); - - targetList.add( - new TrackedTarget( - box, - clsIds.get(idx), - confs.get(idx), - new TargetCalculationParameters( - false, null, null, null, null, frameStaticProperties))); - } - - // ====================== - - var fpsResult = calculateFPSPipe.run(null); - var fps = fpsResult.output; - - return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, input_frame); - } -} diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipelineSettings.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipelineSettings.java index e5ae95a7bc..2422bb9627 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipelineSettings.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/DnnPipelineSettings.java @@ -17,8 +17,18 @@ package org.photonvision.vision.pipeline; -public class DnnPipelineSettings extends CVPipelineSettings { +public class DnnPipelineSettings extends AdvancedPipelineSettings { + String modelPath; + double confidence; + public DnnPipelineSettings() { + super(); this.pipelineType = PipelineType.Dnn; + + // Sane defaults + this.outputShowMultipleTargets = true; + cameraExposure = 20; + cameraAutoExposure = false; + ledMode = false; } } diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/OpencvDnnPipeline.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/OpencvDnnPipeline.java new file mode 100644 index 0000000000..2c07fe66b1 --- /dev/null +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/OpencvDnnPipeline.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) Photon Vision. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.photonvision.vision.pipeline; + +import java.util.ArrayList; +import java.util.List; +import org.opencv.core.Core; +import org.opencv.core.Mat; +import org.opencv.core.MatOfFloat; +import org.opencv.core.MatOfInt; +import org.opencv.core.MatOfRect2d; +import org.opencv.core.Point; +import org.opencv.core.Rect2d; +import org.opencv.core.Scalar; +import org.opencv.core.Size; +import org.opencv.dnn.Dnn; +import org.opencv.dnn.Net; +import org.opencv.imgproc.Imgproc; +import org.opencv.utils.Converters; +import org.photonvision.common.util.ColorHelper; +import org.photonvision.vision.frame.Frame; +import org.photonvision.vision.frame.FrameThresholdType; +import org.photonvision.vision.pipe.CVPipe.CVPipeResult; +import org.photonvision.vision.pipe.impl.*; +import org.photonvision.vision.pipe.impl.OpencvDnnPipe.OpencvDnnPipeParams; +import org.photonvision.vision.pipeline.result.CVPipelineResult; +import org.photonvision.vision.target.TrackedTarget; +import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters; + +public class OpencvDnnPipeline extends CVPipeline { + private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe(); + private final OpencvDnnPipe dnnPipe = new OpencvDnnPipe(); + + private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE; + + public OpencvDnnPipeline() { + super(PROCESSING_TYPE); + settings = new DnnPipelineSettings(); + } + + public OpencvDnnPipeline(DnnPipelineSettings settings) { + super(PROCESSING_TYPE); + this.settings = settings; + } + + @Override + protected void setPipeParamsImpl() { + dnnPipe.setParams(new OpencvDnnPipeParams(settings.modelPath)); + } + + @Override + protected CVPipelineResult process(Frame input_frame, DnnPipelineSettings settings) { + long sumPipeNanosElapsed = 0; + + CVPipeResult> ret = dnnPipe.run(input_frame.colorImage); + sumPipeNanosElapsed += ret.nanosElapsed; + List targetList; + + // TODO deal with RKNN stuf here + targetList = ret.output; + var names = dnnPipe.getParams().getClassNames(); + + input_frame.colorImage.getMat().copyTo(input_frame.processedImage.getMat()); + + List targets = new ArrayList<>(); + + // This belongs in a collect & draw pipe but I'm lazy + for (var t : targetList) { + Imgproc.rectangle(input_frame.processedImage.getMat(), t.box.tl(), t.box.br(), new Scalar(0, 0, 255), 2); + + var name = String.format("%s (%f)", names.get(t.classIdx), t.confidence); + + Imgproc.putText( + input_frame.processedImage.getMat(), + name, + new Point(t.box.x + t.box.width / 2.0, t.box.y + t.box.height / 2.0), + 0, + 0.6, + ColorHelper.colorToScalar(java.awt.Color.white), + 2); + + targets.add( + new TrackedTarget( + t, + new TargetCalculationParameters( + false, null, null, null, null, frameStaticProperties))); + } + + var fpsResult = calculateFPSPipe.run(null); + var fps = fpsResult.output; + + return new CVPipelineResult(sumPipeNanosElapsed, fps, targets, input_frame); + } +} diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/PipelineType.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/PipelineType.java index c050a55d6f..34de3fac9a 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/PipelineType.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/PipelineType.java @@ -25,7 +25,7 @@ public enum PipelineType { ColoredShape(1, ColoredShapePipeline.class), AprilTag(2, AprilTagPipeline.class), Aruco(3, ArucoPipeline.class), - Dnn(4, DnnPipeline.class); + Dnn(4, RknnPipeline.class); public final int baseIndex; public final Class clazz; diff --git a/photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java b/photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java index c6d55211cd..916d768c79 100644 --- a/photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java +++ b/photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java @@ -211,7 +211,7 @@ private void setPipelineInternal(int newIndex) { break; case Dnn: logger.debug("Creating DNN Pipeline"); - currentUserPipeline = new DnnPipeline((DnnPipelineSettings) desiredPipelineSettings); + currentUserPipeline = new RknnPipeline((DnnPipelineSettings) desiredPipelineSettings); break; default: // Can be calib3d or drivermode, both of which are special cases diff --git a/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java b/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java index f53ab2e219..d5dbaed617 100644 --- a/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java +++ b/photon-core/src/main/java/org/photonvision/vision/target/TrackedTarget.java @@ -27,6 +27,7 @@ import org.opencv.core.MatOfPoint; import org.opencv.core.MatOfPoint2f; import org.opencv.core.Point; +import org.opencv.core.Rect2d; import org.opencv.core.RotatedRect; import org.photonvision.common.util.SerializationUtils; import org.photonvision.common.util.math.MathUtils; @@ -38,6 +39,7 @@ import org.photonvision.vision.opencv.Contour; import org.photonvision.vision.opencv.DualOffsetValues; import org.photonvision.vision.opencv.Releasable; +import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult; public class TrackedTarget implements Releasable { public final Contour m_mainContour; @@ -222,12 +224,16 @@ public TrackedTarget( m_targetOffsetPoint = new Point(box.x, box.y); m_robotOffsetPoint = new Point(); - m_pitch = - TargetCalculations.calculatePitch( - box.y + box.height / 2.0, params.cameraCenterPoint.y, params.verticalFocalLength); - m_yaw = - TargetCalculations.calculateYaw( - box.x + box.width / 2.0, params.cameraCenterPoint.x, params.horizontalFocalLength); + var yawPitch = + TargetCalculations.calculateYawPitch( + params.cameraCenterPoint.x, + box.x + box.width / 2.0, + params.horizontalFocalLength, + params.cameraCenterPoint.y, + box.y + box.height / 2.0, + params.verticalFocalLength); + m_yaw = yawPitch.getFirst(); + m_pitch = yawPitch.getSecond(); Point[] cornerPoints = new Point[] { @@ -248,6 +254,10 @@ public TrackedTarget( m_fiducialId = class_id; } + public TrackedTarget(NeuralNetworkPipeResult t, TargetCalculationParameters targetCalculationParameters) { + this(t.box, t.classIdx, t.confidence, targetCalculationParameters); + } + public void setFiducialId(int id) { m_fiducialId = id; }