RKNN Object Detection

.gitignore

@@ -164,3 +164,5 @@ photonlib-java-examples/*/networktables.json.bck
 photon-server/src/main/resources/web/index.html
 
 venv
+
+!photon-core/src/main/resources/models.zip

photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue

@@ -24,6 +24,9 @@ const changeCurrentCameraIndex = (index: number) => {
     case PipelineType.Aruco:
       pipelineType.value = WebsocketPipelineType.Aruco;
       break;
+    case PipelineType.RKNN:
+      pipelineType.value = WebsocketPipelineType.RKNN;
+      break;
   }
 };
 
@@ -152,7 +155,8 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
     { name: "Reflective", value: WebsocketPipelineType.Reflective },
     { name: "Colored Shape", value: WebsocketPipelineType.ColoredShape },
     { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
-    { name: "Aruco", value: WebsocketPipelineType.Aruco }
+    { name: "Aruco", value: WebsocketPipelineType.Aruco },
+    { name: "RKNN", value: WebsocketPipelineType.RKNN }
   ];
 
   if (useCameraSettingsStore().isDriverMode) {
@@ -208,6 +212,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
     case PipelineType.Aruco:
       pipelineType.value = WebsocketPipelineType.Aruco;
       break;
+    case PipelineType.RKNN:
+      pipelineType.value = WebsocketPipelineType.RKNN;
+      break;
   }
 });
 </script>
@@ -354,7 +361,8 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
               { name: 'Reflective', value: WebsocketPipelineType.Reflective },
               { name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
               { name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
-              { name: 'Aruco', value: WebsocketPipelineType.Aruco }
+              { name: 'Aruco', value: WebsocketPipelineType.Aruco },
+              { name: 'RKNN', value: WebsocketPipelineType.RKNN }
             ]"
           />
         </v-card-text>

photon-client/src/components/dashboard/ConfigOptions.vue

@@ -12,6 +12,7 @@ import OutputTab from "@/components/dashboard/tabs/OutputTab.vue";
 import TargetsTab from "@/components/dashboard/tabs/TargetsTab.vue";
 import PnPTab from "@/components/dashboard/tabs/PnPTab.vue";
 import Map3DTab from "@/components/dashboard/tabs/Map3DTab.vue";
+import RKNNTab from "@/components/dashboard/tabs/RKNNTab.vue";
 import { WebsocketPipelineType } from "@/types/WebsocketDataTypes";
 
 interface ConfigOption {
@@ -55,6 +56,10 @@ const allTabs = Object.freeze({
   map3dTab: {
     tabName: "3D",
     component: Map3DTab
+  },
+  rknnTab: {
+    tabName: "RKNN",
+    component: RKNNTab
   }
 });
 
@@ -75,21 +80,29 @@ const getTabGroups = (): ConfigOption[][] => {
         allTabs.contoursTab,
         allTabs.apriltagTab,
         allTabs.arucoTab,
+        allTabs.rknnTab,
         allTabs.outputTab
       ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   } else if (lgAndDown) {
     return [
       [allTabs.inputTab],
-      [allTabs.thresholdTab, allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [
+        allTabs.thresholdTab,
+        allTabs.contoursTab,
+        allTabs.apriltagTab,
+        allTabs.arucoTab,
+        allTabs.rknnTab,
+        allTabs.outputTab
+      ],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   } else if (xl) {
     return [
       [allTabs.inputTab],
       [allTabs.thresholdTab],
-      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.outputTab],
+      [allTabs.contoursTab, allTabs.apriltagTab, allTabs.arucoTab, allTabs.rknnTab, allTabs.outputTab],
       [allTabs.targetsTab, allTabs.pnpTab, allTabs.map3dTab]
     ];
   }
@@ -103,17 +116,20 @@ const tabGroups = computed<ConfigOption[][]>(() => {
   const allow3d = useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled;
   const isAprilTag = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.AprilTag;
   const isAruco = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.Aruco;
+  const isRKNN = useCameraSettingsStore().currentWebsocketPipelineType === WebsocketPipelineType.RKNN;
 
   return getTabGroups()
     .map((tabGroup) =>
       tabGroup.filter(
         (tabConfig) =>
           !(!allow3d && tabConfig.tabName === "3D") && //Filter out 3D tab any time 3D isn't calibrated
           !((!allow3d || isAprilTag || isAruco) && tabConfig.tabName === "PnP") && //Filter out the PnP config tab if 3D isn't available, or we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags
-          !((isAprilTag || isAruco) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags
+          !((isAprilTag || isAruco || isRKNN) && tabConfig.tabName === "Threshold") && //Filter out threshold tab if we're doing AprilTags or RKNN
+          !((isAprilTag || isAruco || isRKNN) && tabConfig.tabName === "Contours") && //Filter out contours if we're doing AprilTags or RKNN
           !(!isAprilTag && tabConfig.tabName === "AprilTag") && //Filter out apriltag unless we actually are doing AprilTags
-          !(!isAruco && tabConfig.tabName === "Aruco") //Filter out aruco unless we actually are doing Aruco
+          !(!isAruco && tabConfig.tabName === "Aruco") && //Filter out aruco unless we actually are doing Aruco
+          !(!isRKNN && tabConfig.tabName === "RKNN") && //Filter out RKNN unless we actually are doing RKNN
+          !(isRKNN && tabConfig.tabName === "Output") //Filter out output tab if we're doing RKNN
       )
     )
     .filter((it) => it.length); // Remove empty tab groups

photon-client/src/components/dashboard/tabs/RKNNTab.vue

@@ -0,0 +1,62 @@
+<script setup lang="ts">
+import { PipelineType } from "@/types/PipelineTypes";
+import PvSlider from "@/components/common/pv-slider.vue";
+import pvSelect from "@/components/common/pv-select.vue";
+import { computed, getCurrentInstance } from "vue";
+import { useStateStore } from "@/stores/StateStore";
+import type { ActivePipelineSettings } from "@/types/PipelineTypes";
+import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
+
+// TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
+// Defer reference to store access method
+const currentPipelineSettings = computed<ActivePipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings
+);
+
+const currentSelectedModelIndex = computed(() => {
+  const cps = currentPipelineSettings.value;
+  return cps.pipelineType === PipelineType.RKNN
+    ? useCameraSettingsStore().availableModels.indexOf(cps.selectedModel)
+    : 0;
+});
+
+const getModelName = (index: number) => {
+  const cps = currentPipelineSettings.value;
+  return cps.pipelineType === PipelineType.RKNN ? useCameraSettingsStore().availableModels[index] : "";
+};
+
+console.log("available: " + useCameraSettingsStore().availableModels);
+const interactiveCols = computed(
+  () =>
+    (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
+    (!useStateStore().sidebarFolded || useCameraSettingsStore().isDriverMode)
+)
+  ? 9
+  : 8;
+</script>
+
+<template>
+  <div v-if="currentPipelineSettings.pipelineType === PipelineType.RKNN">
+    <pv-slider
+      v-model="currentPipelineSettings.confidenceThreshold"
+      class="pt-2"
+      :slider-cols="interactiveCols"
+      label="Confidence Threshold"
+      tooltip="Any detections with a confidence below this threshold will be discarded. Minimum value is 0.1."
+      :min="0.1"
+      :max="1"
+      :step="0.01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidenceThreshold: value }, false)"
+    />
+    <pv-select
+      :select-cols="interactiveCols"
+      :value="currentSelectedModelIndex"
+      label="RKNN Model"
+      tooltip="The RKNN model to use for inference."
+      :items="useCameraSettingsStore().availableModels"
+      @input="
+        (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ selectedModel: getModelName(value) }, false)
+      "
+    />
+  </div>
+</template>

photon-client/src/components/dashboard/tabs/TargetsTab.vue

@@ -48,6 +48,10 @@ const resetCurrentBuffer = () => {
               >
                 Fiducial ID
               </th>
+              <template v-if="currentPipelineSettings.pipelineType === PipelineType.RKNN">
+                <th class="text-center white--text">Class</th>
+                <th class="text-center white--text">Confidence</th>
+              </template>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <th class="text-center white--text">Pitch &theta;&deg;</th>
                 <th class="text-center white--text">Yaw &theta;&deg;</th>
@@ -85,6 +89,10 @@ const resetCurrentBuffer = () => {
               >
                 {{ target.fiducialId }}
               </td>
+              <template v-if="currentPipelineSettings.pipelineType === PipelineType.RKNN">
+                <td class="text-center">{{ target.fiducialId }}</td>
+                <td class="text-center">{{ target.ambiguity.toFixed(2) }}</td>
+              </template>
               <template v-if="!useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <td class="text-center">{{ target.pitch.toFixed(2) }}&deg;</td>
                 <td class="text-center">{{ target.yaw.toFixed(2) }}&deg;</td>

photon-client/src/components/settings/DeviceControlCard.vue

@@ -137,7 +137,8 @@ enum ImportType {
   HardwareConfig,
   HardwareSettings,
   NetworkConfig,
-  ApriltagFieldLayout
+  ApriltagFieldLayout,
+  RKNNModel
 }
 const showImportDialog = ref(false);
 const importType = ref<ImportType | number>(-1);
@@ -162,6 +163,9 @@ const handleSettingsImport = () => {
     case ImportType.ApriltagFieldLayout:
       settingsEndpoint = "/aprilTagFieldLayout";
       break;
+    case ImportType.RKNNModel:
+      settingsEndpoint = "/rknnModel";
+      break;
     default:
     case ImportType.AllSettings:
       settingsEndpoint = "";
@@ -259,7 +263,8 @@ const handleSettingsImport = () => {
                       'Hardware Config',
                       'Hardware Settings',
                       'Network Config',
-                      'Apriltag Layout'
+                      'Apriltag Layout',
+                      'RKNN Model'
                     ]"
                     :select-cols="10"
                     style="width: 100%"
@@ -270,7 +275,13 @@ const handleSettingsImport = () => {
                     v-model="importFile"
                     :disabled="importType === -1"
                     :error-messages="importType === -1 ? 'Settings type not selected' : ''"
-                    :accept="importType === ImportType.AllSettings ? '.zip' : '.json'"
+                    :accept="
+                      importType === ImportType.AllSettings
+                        ? '.zip'
+                        : importType === ImportType.RKNNModel
+                          ? '.rknn'
+                          : '.json'
+                    "
                   />
                 </v-row>
                 <v-row

photon-client/src/stores/settings/CameraSettingsStore.ts

@@ -67,6 +67,9 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
     },
     isCSICamera(): boolean {
       return this.currentCameraSettings.isCSICamera;
+    },
+    availableModels(): string[] {
+      return this.currentCameraSettings.availableModels || [];
     }
   },
   actions: {
@@ -104,7 +107,8 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
         pipelineNicknames: d.pipelineNicknames,
         currentPipelineIndex: d.currentPipelineIndex,
         pipelineSettings: d.currentPipelineSettings,
-        cameraQuirks: d.cameraQuirks
+        cameraQuirks: d.cameraQuirks,
+        availableModels: d.availableModels
       }));
     },
     /**

photon-client/src/types/PipelineTypes.ts

@@ -5,7 +5,8 @@ export enum PipelineType {
   Reflective = 2,
   ColoredShape = 3,
   AprilTag = 4,
-  Aruco = 5
+  Aruco = 5,
+  RKNN = 6
 }
 
 export enum AprilTagFamily {
@@ -281,14 +282,36 @@ export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
   doSingleTargetAlways: false
 };
 
+export interface RKNNPipelineSettings extends PipelineSettings {
+  pipelineType: PipelineType.RKNN;
+  confidenceThreshold: number;
+  selectedModel: string;
+}
+
+export type ConfigurableRKNNPipelineSettings = Partial<Omit<RKNNPipelineSettings, "pipelineType">> &
+  ConfigurablePipelineSettings;
+export const DefaultRKNNPipelineSettings: RKNNPipelineSettings = {
+  ...DefaultPipelineSettings,
+  pipelineType: PipelineType.RKNN,
+  cameraGain: 43,
+  targetModel: TargetModel.RapidReactCircularCargoBall,
+  ledMode: false,
+  outputShowMultipleTargets: false,
+  cameraExposure: 13,
+  confidenceThreshold: 0.35,
+  selectedModel: "model"
+};
+
 export type ActivePipelineSettings =
   | ReflectivePipelineSettings
   | ColoredShapePipelineSettings
   | AprilTagPipelineSettings
-  | ArucoPipelineSettings;
+  | ArucoPipelineSettings
+  | RKNNPipelineSettings;
 
 export type ActiveConfigurablePipelineSettings =
   | ConfigurableReflectivePipelineSettings
   | ConfigurableColoredShapePipelineSettings
   | ConfigurableAprilTagPipelineSettings
-  | ConfigurableArucoPipelineSettings;
+  | ConfigurableArucoPipelineSettings
+  | ConfigurableRKNNPipelineSettings;

photon-client/src/types/SettingTypes.ts

@@ -179,6 +179,8 @@ export interface CameraSettings {
 
   cameraQuirks: QuirkyCamera;
   isCSICamera: boolean;
+
+  availableModels?: string[];
 }
 
 export interface CameraSettingsChangeRequest {

photon-client/src/types/WebsocketDataTypes.ts

@@ -58,6 +58,7 @@ export interface WebsocketCameraSettingsUpdate {
   pipelineNicknames: string[];
   videoFormatList: WebsocketVideoFormat;
   cameraQuirks: QuirkyCamera;
+  availableModels?: string[];
 }
 export interface WebsocketNTUpdate {
   connected: boolean;
@@ -101,5 +102,6 @@ export enum WebsocketPipelineType {
   Reflective = 0,
   ColoredShape = 1,
   AprilTag = 2,
-  Aruco = 3
+  Aruco = 3,
+  RKNN = 4
 }

photon-core/src/main/java/org/photonvision/common/configuration/ConfigManager.java

@@ -217,6 +217,10 @@ public Path getCalibDir() {
         return Path.of(configDirectoryFile.toString(), "calibImgs");
     }
 
+    public Path getRKNNModelsPath() {
+        return Path.of(configDirectoryFile.toString() + "/models");
+    }
+
     public static final String LOG_PREFIX = "photonvision-";
     public static final String LOG_EXT = ".log";
     public static final String LOG_DATE_TIME_FORMAT = "yyyy-M-d_hh-mm-ss";
@@ -261,6 +265,17 @@ public boolean saveUploadedAprilTagFieldLayout(Path uploadPath) {
         return m_provider.saveUploadedAprilTagFieldLayout(uploadPath);
     }
 
+    public boolean saveUploadedRKNNModel(Path uploadPath) {
+        var modelPath = Path.of(getRKNNModelsPath().toString(), uploadPath.getFileName().toString());
+        try {
+            Files.copy(uploadPath, modelPath, StandardCopyOption.REPLACE_EXISTING);
+            return true;
+        } catch (IOException e) {
+            logger.error("Exception copying uploaded RKNN model!", e);
+            return false;
+        }
+    }
+
     public void requestSave() {
         logger.trace("Requesting save...");
         saveRequestTimestamp = System.currentTimeMillis();

photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java

@@ -181,5 +181,7 @@ public static class UICameraConfiguration {
         public boolean isFovConfigurable = true;
         public QuirkyCamera cameraQuirks;
         public boolean isCSICamera;
+
+        public String[] availableModels;
     }
 }