upstream updates (#2)

* Load libquadmath on Windows (PhotonVision#1163) Nobody reads these, right? This probably won't make things worse. Surely. * Add RKNN / Object Detection Pipeline (PhotonVision#1144) Tested on Orange Pi 5 and Cool Pi 4B. Merge with parts of the OpenCV DNN PR. Adds support for YOLOv5s models for Rockchip CPUs with a NPU. Right now hard coded to a note model from alex_idk. Very much still incubating and largely untested. * Add LL3 image from photon-image-modifier (PhotonVision#1166) * LL3 * Update build.yml * Fix total ram reporting (PhotonVision#1161) * formatting * spotless * unpacked file * update readme * formatting of rknn-readme --------- Co-authored-by: Matt <[email protected]> Co-authored-by: Mohammad Durrani <[email protected]> Co-authored-by: Programmers3539 <[email protected]> Co-authored-by: Ryan Blue <[email protected]>
laviRZ · Jan 16, 2024 · 6b4f106 · 6b4f106
1 parent 13537a7
commit 6b4f106
Show file tree

Hide file tree

Showing 48 changed files with 835 additions and 94 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -281,6 +281,12 @@ jobs:
             image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_limelight.img.xz
             cpu: cortex-a7
             image_additional_mb: 4096
+          - os: ubuntu-latest
+            artifact-name: LinuxArm64
+            image_suffix: limelight3
+            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.5/photonvision_limelight3.img.xz
+            cpu: cortex-a7
+            image_additional_mb: 0
           - os: ubuntu-latest
             artifact-name: LinuxArm64
             image_suffix: orangepi5

diff --git a/.styleguide b/.styleguide
@@ -18,6 +18,7 @@ modifiableFileExclude {
   \.dll$
   \.webp$
   \.ico$
+  \.rknn$
   gradlew
 }
 

diff --git a/build.gradle b/build.gradle
@@ -30,9 +30,11 @@ ext {
     joglVersion = "2.4.0-rc-20200307"
     javalinVersion = "5.6.2"
     photonGlDriverLibVersion = "dev-v2023.1.0-9-g75fc678"
+    rknnVersion = "dev-v2024.0.0-30-g001b5ec"
     frcYear = "2024"
     mrcalVersion = "dev-v2024.0.0-7-gc976aaa";
 
+
     pubVersion = versionString
     isDev = pubVersion.startsWith("dev")
 

diff --git a/photon-client/index.html b/photon-client/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />

diff --git a/photon-client/package-lock.json b/photon-client/package-lock.json
diff --git a/photon-client/package.json b/photon-client/package.json
@@ -27,17 +27,17 @@
   },
   "devDependencies": {
     "@rushstack/eslint-patch": "^1.3.2",
-    "@vue/eslint-config-prettier": "^9.0.0",
-    "@vue/eslint-config-typescript": "^12.0.0",
-    "prettier": "^3.1.1",
     "@types/node": "^16.11.45",
     "@types/three": "^0.160.0",
     "@vitejs/plugin-vue2": "^2.3.1",
+    "@vue/eslint-config-prettier": "^9.0.0",
+    "@vue/eslint-config-typescript": "^12.0.0",
     "@vue/tsconfig": "^0.5.1",
     "deepmerge": "^4.3.1",
     "eslint": "^8.56.0",
     "eslint-plugin-vue": "^9.19.2",
     "npm-run-all": "^4.1.5",
+    "prettier": "3.2.2",
     "sass": "~1.32",
     "sass-loader": "^13.3.2",
     "terser": "^5.14.2",

diff --git a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue
@@ -7,6 +7,7 @@ import { computed, ref } from "vue";
 import PvIcon from "@/components/common/pv-icon.vue";
 import PvInput from "@/components/common/pv-input.vue";
 import { PipelineType } from "@/types/PipelineTypes";
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
 
 const changeCurrentCameraIndex = (index: number) => {
   useCameraSettingsStore().setCurrentCameraIndex(index, true);
@@ -27,6 +28,9 @@ const changeCurrentCameraIndex = (index: number) => {
     case PipelineType.RKNN:
       pipelineType.value = WebsocketPipelineType.RKNN;
       break;
+    case PipelineType.ObjectDetection:
+      pipelineType.value = WebsocketPipelineType.ObjectDetection;
+      break;
   }
 };
 
@@ -158,6 +162,9 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
     { name: "Aruco", value: WebsocketPipelineType.Aruco },
     { name: "RKNN", value: WebsocketPipelineType.RKNN }
   ];
+  if (useSettingsStore().general.rknnSupported) {
+    pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
+  }
 
   if (useCameraSettingsStore().isDriverMode) {
     pipelineTypes.push({ name: "Driver Mode", value: WebsocketPipelineType.DriverMode });
@@ -215,6 +222,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
     case PipelineType.RKNN:
       pipelineType.value = WebsocketPipelineType.RKNN;
       break;
+    case PipelineType.ObjectDetection:
+      pipelineType.value = WebsocketPipelineType.ObjectDetection;
+      break;
   }
 });
 </script>
@@ -362,7 +372,8 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
               { name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
               { name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
               { name: 'Aruco', value: WebsocketPipelineType.Aruco },
-              { name: 'RKNN', value: WebsocketPipelineType.RKNN }
+              { name: 'RKNN', value: WebsocketPipelineType.RKNN },
+              { name: 'Object Detection', value: WebsocketPipelineType.ObjectDetection }
             ]"
           />
         </v-card-text>

diff --git a/photon-client/src/components/dashboard/ConfigOptions.vue b/photon-client/src/components/dashboard/ConfigOptions.vue
@@ -8,6 +8,7 @@ import ThresholdTab from "@/components/dashboard/tabs/ThresholdTab.vue";
 import ContoursTab from "@/components/dashboard/tabs/ContoursTab.vue";
 import AprilTagTab from "@/components/dashboard/tabs/AprilTagTab.vue";
 import ArucoTab from "@/components/dashboard/tabs/ArucoTab.vue";
+import ObjectDetectionTab from "@/components/dashboard/tabs/ObjectDetectionTab.vue";
 import OutputTab from "@/components/dashboard/tabs/OutputTab.vue";
 import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue";
 import PnPTab from "@/components/dashboard/tabs/PnPTab.vue";
@@ -41,6 +42,10 @@ const allTabs = Object.freeze({
     tabName: "Aruco",
     component: ArucoTab
   },
+  objectDetectionTab: {
+    tabName: "Object Detection",
+    component: ObjectDetectionTab
+  },
   outputTab: {
     tabName: "Output",
     component: OutputTab
@@ -81,6 +86,7 @@ const getTabGroups = (): ConfigOption[][] => {
         allTabs.apriltagTab,
         allTabs.arucoTab,
         allTabs.rknnTab,
+        allTabs.objectDetectionTab,
         allTabs.outputTab
       ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
@@ -94,6 +100,7 @@ const getTabGroups = (): ConfigOption[][] => {
         allTabs.apriltagTab,
         allTabs.arucoTab,
         allTabs.rknnTab,
+        allTabs.objectDetectionTab,
         allTabs.outputTab
       ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
@@ -102,7 +109,14 @@ const getTabGroups = (): ConfigOption[][] => {
     return [
       [allTabs.inputTab],
       [allTabs.thresholdTab],
-      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.rknnTab, allTabs.outputTab],
+      [
+        allTabs.contoursTab,
+        allTabs.apriltagTab,
+        allTabs.arucoTab,
+        allTabs.rknnTab,
+        allTabs.objectDetectionTab,
+        allTabs.outputTab
+      ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   }
@@ -117,19 +131,22 @@ const tabGroups = computed<ConfigOption[][]>(() => {
   const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag;
   const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco;
   const isRKNN = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.RKNN;
+  const isObjectDetection =
+    useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.ObjectDetection;
 
   return getTabGroups()
     .map((tabGroup) =>
       tabGroup.filter(
         (tabConfig) =>
           !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated
-          !((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
-          !((isAprilTag || isAruco || isRKNN) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags or RKNN
-          !((isAprilTag || isAruco || isRKNN) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags or RKNN
+          !((!allow3d || isAprilTag || isAruco || isRKNN || isObjectDetection) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
+          !((isAprilTag || isAruco || isRKNN || isObjectDetection) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
+          !((isAprilTag || isAruco || isRKNN || isObjectDetection) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
           !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
           !(!isAruco && tabConfig.tabName === "Aruco") && //Filter out aruco unless we actually are doing Aruco
           !(!isRKNN && tabConfig.tabName === "RKNN") && //Filter out RKNN unless we actually are doing RKNN
-          !(isRKNN && tabConfig.tabName === "Output") //Filter out output tab if we're doing RKNN
+          !(!isRKNN && tabConfig.tabName === "Output") && //Filter out output tab if we're doing RKNN
+          !(!isObjectDetection && tabConfig.tabName === "Object Detection") //Filter out aruco unless we actually are doing Aruco
       )
     )
     .filter((it) => it.length); // Remove empty tab groups

diff --git a/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue b/photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue
@@ -0,0 +1,35 @@
+<script setup lang="ts">
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+import { PipelineType } from "@/types/PipelineTypes";
+import PvSlider from "@/components/common/pv-slider.vue";
+import { computed, getCurrentInstance } from "vue";
+import { useStateStore } from "@/stores/StateStore";
+
+// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
+// Defer reference to store access method
+const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+
+const interactiveCols = computed(
+  () =>
+    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+)
+  ? 9
+  : 8;
+</script>
+
+<template>
+  <div v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+    <pv-slider
+      v-model="currentPipelineSettings.confidence"
+      class="pt-2"
+      :slider-cols="interactiveCols"
+      label="Confidence"
+      tooltip="The minimum confidence for a detection to be considered valid. Bigger numbers mean fewer but more probable detections are allowed through."
+      :min="0"
+      :max="1"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
+    />
+  </div>
+</template>
diff --git a/photon-client/src/components/dashboard/tabs/TargetsTab.vue b/photon-client/src/components/dashboard/tabs/TargetsTab.vue
@@ -48,7 +48,12 @@ const resetCurrentBuffer = () => {
               >
                 Fiducial ID
               </th>
-              <template v-if="currentPipelineSettings.pipelineType === PipelineType.RKNN">
+              <template
+                v-if="
+                  currentPipelineSettings.pipelineType === PipelineType.RKNN ||
+                  currentPipelineSettings.pipelineType === PipelineType.ObjectDetection
+                "
+              >
                 <th class="text-center white--text">Class</th>
                 <th class="text-center white--text">Confidence</th>
               </template>
@@ -93,6 +98,18 @@ const resetCurrentBuffer = () => {
                 <td class="text-center">{{ target.fiducialId }}</td>
                 <td class="text-center">{{ target.ambiguity.toFixed(2) }}</td>
               </template>
+              <td
+                v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
+                class="text-center white--text"
+              >
+                {{ useStateStore().currentPipelineResults?.classNames[target.classId] }}
+              </td>
+              <td
+                v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection"
+                class="text-center white--text"
+              >
+                {{ target.confidence.toFixed(2) }}
+              </td>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <td class="text-center">{{ target.pitch.toFixed(2) }}&deg;</td>
                 <td class="text-center">{{ target.yaw.toFixed(2) }}&deg;</td>

diff --git a/photon-client/src/stores/settings/GeneralSettingsStore.ts b/photon-client/src/stores/settings/GeneralSettingsStore.ts
@@ -27,7 +27,8 @@ export const useSettingsStore = defineStore("settings", {
       gpuAcceleration: undefined,
       hardwareModel: undefined,
       hardwarePlatform: undefined,
-      mrCalWorking: true
+      mrCalWorking: true,
+      rknnSupported: false
     },
     network: {
       ntServerAddress: "",
@@ -99,7 +100,8 @@ export const useSettingsStore = defineStore("settings", {
         hardwareModel: data.general.hardwareModel || undefined,
         hardwarePlatform: data.general.hardwarePlatform || undefined,
         gpuAcceleration: data.general.gpuAcceleration || undefined,
-        mrCalWorking: data.general.mrCalWorking
+        mrCalWorking: data.general.mrCalWorking,
+        rknnSupported: data.general.rknnSupported
       };
       this.lighting = data.lighting;
       this.network = data.networkSettings;

diff --git a/photon-client/src/types/PhotonTrackingTypes.ts b/photon-client/src/types/PhotonTrackingTypes.ts
@@ -54,6 +54,8 @@ export interface PhotonTarget {
   ambiguity: number;
   // -1 if not set
   fiducialId: number;
+  confidence: number;
+  classId: number;
   // undefined if 3d isn't enabled
   pose?: Transform3d;
 }
@@ -70,4 +72,6 @@ export interface PipelineResult {
   targets: PhotonTarget[];
   // undefined if multitag failed or non-tag pipeline
   multitagResult?: MultitagResult;
+  // Object detection class names -- empty if not doing object detection
+  classNames: string[];
 }
diff --git a/photon-client/src/types/PipelineTypes.ts b/photon-client/src/types/PipelineTypes.ts
@@ -6,7 +6,8 @@ export enum PipelineType {
   ColoredShape = 3,
   AprilTag = 4,
   Aruco = 5,
-  RKNN = 6
+  ObjectDetection = 6,
+  RKNN = 7
 }
 
 export enum AprilTagFamily {
@@ -302,16 +303,41 @@ export const DefaultRKNNPipelineSettings: RKNNPipelineSettings = {
   selectedModel: "model"
 };
 
+export interface ObjectDetectionPipelineSettings extends PipelineSettings {
+  pipelineType: PipelineType.ObjectDetection;
+  confidence: number;
+  nms: number;
+  box_thresh: number;
+}
+export type ConfigurableObjectDetectionPipelineSettings = Partial<
+  Omit<ObjectDetectionPipelineSettings, "pipelineType">
+> &
+  ConfigurablePipelineSettings;
+export const DefaultObjectDetectionPipelineSettings: ObjectDetectionPipelineSettings = {
+  ...DefaultPipelineSettings,
+  pipelineType: PipelineType.ObjectDetection,
+  cameraGain: 20,
+  targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
+  ledMode: true,
+  outputShowMultipleTargets: false,
+  cameraExposure: 6,
+  confidence: 0.9,
+  nms: 0.45,
+  box_thresh: 0.25
+};
+
 export type ActivePipelineSettings =
   | ReflectivePipelineSettings
   | ColoredShapePipelineSettings
   | AprilTagPipelineSettings
   | ArucoPipelineSettings
-  | RKNNPipelineSettings;
+  | RKNNPipelineSettings
+  | ObjectDetectionPipelineSettings;
 
 export type ActiveConfigurablePipelineSettings =
   | ConfigurableReflectivePipelineSettings
   | ConfigurableColoredShapePipelineSettings
   | ConfigurableAprilTagPipelineSettings
   | ConfigurableArucoPipelineSettings
-  | ConfigurableRKNNPipelineSettings;
+  | ConfigurableRKNNPipelineSettings
+  | ConfigurableObjectDetectionPipelineSettings;
diff --git a/photon-client/src/types/SettingTypes.ts b/photon-client/src/types/SettingTypes.ts
@@ -7,6 +7,7 @@ export interface GeneralSettings {
   hardwareModel?: string;
   hardwarePlatform?: string;
   mrCalWorking: boolean;
+  rknnSupported: boolean;
 }
 
 export interface MetricData {

diff --git a/photon-client/src/types/WebsocketDataTypes.ts b/photon-client/src/types/WebsocketDataTypes.ts
@@ -103,5 +103,6 @@ export enum WebsocketPipelineType {
   ColoredShape = 1,
   AprilTag = 2,
   Aruco = 3,
-  RKNN = 4
+  ObjectDetection = 4,
+  RKNN = 5
 }