Run multitag on coprocessor

.gitignore

@@ -114,7 +114,6 @@ fabric.properties
 **/.settings
 **/.classpath
 **/.project
-**/settings
 **/dependency-reduced-pom.xml
 # photon-server/photon-vision.iml
 

photon-client/src/App.vue

@@ -30,7 +30,7 @@ const websocket = new AutoReconnectingWebsocket(
       useSettingsStore().updateMetricsFromWebsocket(data.metrics);
     }
     if (data.updatePipelineResult !== undefined) {
-      useStateStore().updatePipelineResultsFromWebsocket(data.updatePipelineResult);
+      useStateStore().updateBackendResultsFromWebsocket(data.updatePipelineResult);
     }
     if (data.mutatePipelineSettings !== undefined && data.cameraIndex !== undefined) {
       useCameraSettingsStore().changePipelineSettingsInStore(data.mutatePipelineSettings, data.cameraIndex);

photon-client/src/components/app/photon-log-view.vue

@@ -71,7 +71,7 @@ document.addEventListener("keydown", (e) => {
           </v-btn>
         </v-btn-toggle>
         <v-card-text v-if="logs.length === 0" style="font-size: 18px; font-weight: 600">
-          There are no Logs to show
+          There are no logs to show
         </v-card-text>
         <v-virtual-scroll v-else :items="logs" item-height="50" height="600">
           <template #default="{ item }">

photon-client/src/components/cameras/CamerasView.vue

@@ -30,7 +30,7 @@ const driverMode = computed<boolean>({
 });
 
 const fpsTooLow = computed<boolean>(() => {
-  const currFPS = useStateStore().pipelineResults?.fps || 0;
+  const currFPS = useStateStore().currentPipelineResults?.fps || 0;
   const targetFPS = useCameraSettingsStore().currentVideoFormat.fps;
   const driverMode = useCameraSettingsStore().isDriverMode;
   const gpuAccel = useSettingsStore().general.gpuAcceleration !== undefined;
@@ -58,8 +58,8 @@ const fpsTooLow = computed<boolean>(() => {
             style="font-size: 1rem; padding: 0; margin: 0"
           >
             <span class="pr-1">
-              {{ Math.round(useStateStore().pipelineResults?.fps || 0) }}&nbsp;FPS &ndash;
-              {{ Math.min(Math.round(useStateStore().pipelineResults?.latency || 0), 9999) }} ms latency
+              {{ Math.round(useStateStore().currentPipelineResults?.fps || 0) }}&nbsp;FPS &ndash;
+              {{ Math.min(Math.round(useStateStore().currentPipelineResults?.latency || 0), 9999) }} ms latency
             </span>
           </v-chip>
         </div>

photon-client/src/components/dashboard/CamerasCard.vue

@@ -21,7 +21,7 @@ const driverMode = computed<boolean>({
 });
 
 const fpsTooLow = computed<boolean>(() => {
-  const currFPS = useStateStore().pipelineResults?.fps || 0;
+  const currFPS = useStateStore().currentPipelineResults?.fps || 0;
   const targetFPS = useCameraSettingsStore().currentVideoFormat.fps;
   const driverMode = useCameraSettingsStore().isDriverMode;
   const gpuAccel = useSettingsStore().general.gpuAcceleration !== undefined;
@@ -46,7 +46,7 @@ const fpsTooLow = computed<boolean>(() => {
           style="font-size: 1rem; padding: 0; margin: 0"
         >
           <span class="pr-1">
-            Processing @ {{ Math.round(useStateStore().pipelineResults?.fps || 0) }}&nbsp;FPS &ndash;
+            Processing @ {{ Math.round(useStateStore().currentPipelineResults?.fps || 0) }}&nbsp;FPS &ndash;
           </span>
           <span
             v-if="
@@ -61,7 +61,7 @@ const fpsTooLow = computed<boolean>(() => {
             stop viewing the raw stream for better performance
           </span>
           <span v-else>
-            {{ Math.min(Math.round(useStateStore().pipelineResults?.latency || 0), 9999) }} ms latency
+            {{ Math.min(Math.round(useStateStore().currentPipelineResults?.latency || 0), 9999) }} ms latency
           </span>
         </v-chip>
       </div>

photon-client/src/components/dashboard/tabs/Map3DTab.vue

@@ -4,7 +4,7 @@ import type { PhotonTarget } from "@/types/PhotonTrackingTypes";
 import { useStateStore } from "@/stores/StateStore";
 import Photon3dVisualizer from "@/components/app/photon-3d-visualizer.vue";
 
-const trackedTargets = computed<PhotonTarget[]>(() => useStateStore().pipelineResults?.targets || []);
+const trackedTargets = computed<PhotonTarget[]>(() => useStateStore().currentPipelineResults?.targets || []);
 </script>
 
 <template>

photon-client/src/components/dashboard/tabs/OutputTab.vue

@@ -40,6 +40,8 @@ const offsetPoints = computed<MetricItem[]>(() => {
   }
 });
 
+const currentPipelineSettings = useCameraSettingsStore().currentPipelineSettings;
+
 const interactiveCols = computed(
   () =>
     (getCurrentInstance()?.proxy.$vuetify.breakpoint.mdAndDown || false) &&
@@ -75,13 +77,37 @@ const interactiveCols = computed(
     <cv-switch
       v-model="useCameraSettingsStore().currentPipelineSettings.outputShowMultipleTargets"
       label="Show Multiple Targets"
-      tooltip="If enabled, up to five targets will be displayed and sent to user code, instead of just one"
+      tooltip="If enabled, up to five targets will be displayed and sent via PhotonLib, instead of just one"
       :disabled="isTagPipeline"
       :switch-cols="interactiveCols"
       @input="
         (value) => useCameraSettingsStore().changeCurrentPipelineSetting({ outputShowMultipleTargets: value }, false)
       "
     />
+    <cv-switch
+      v-if="
+        currentPipelineSettings.pipelineType === PipelineType.AprilTag &&
+        useCameraSettingsStore().isCurrentVideoFormatCalibrated
+      "
+      v-model="currentPipelineSettings.doMultiTarget"
+      label="Do Multi-Target Estimation"
+      tooltip="If enabled, all visible fiducial targets will be used to provide a single pose estimate from their combined model."
+      :switch-cols="interactiveCols"
+      :disabled="!isTagPipeline"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ doMultiTarget: value }, false)"
+    />
+    <cv-switch
+      v-if="
+        currentPipelineSettings.pipelineType === PipelineType.AprilTag &&
+        useCameraSettingsStore().isCurrentVideoFormatCalibrated
+      "
+      v-model="currentPipelineSettings.doSingleTargetAlways"
+      label="Always Do Single-Target Estimation"
+      tooltip="If disabled, visible fiducial targets used for multi-target estimation will not also be used for single-target estimation."
+      :switch-cols="interactiveCols"
+      :disabled="!isTagPipeline || !currentPipelineSettings.doMultiTarget"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ doSingleTargetAlways: value }, false)"
+    />
     <v-divider />
     <table
       v-if="useCameraSettingsStore().currentPipelineSettings.offsetRobotOffsetMode !== RobotOffsetPointMode.None"

photon-client/src/components/dashboard/tabs/TargetsTab.vue

@@ -8,11 +8,10 @@ import { useStateStore } from "@/stores/StateStore";
   <div>
     <v-row align="start" class="pb-4" style="height: 300px">
       <!-- Simple table height must be set here and in the CSS for the fixed-header to work -->
-      <v-simple-table fixed-header height="100%" dense dark>
+      <v-simple-table fixed-header dense dark>
         <template #default>
           <thead style="font-size: 1.25rem">
             <tr>
-              <th class="text-center">Target Count</th>
               <th
                 v-if="
                   useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
@@ -44,8 +43,7 @@ import { useStateStore } from "@/stores/StateStore";
             </tr>
           </thead>
           <tbody>
-            <tr v-for="(target, index) in useStateStore().pipelineResults?.targets" :key="index">
-              <td>{{ index }}</td>
+            <tr v-for="(target, index) in useStateStore().currentPipelineResults?.targets" :key="index">
               <td
                 v-if="
                   useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag ||
@@ -63,21 +61,45 @@ import { useStateStore } from "@/stores/StateStore";
               <template v-else-if="useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled">
                 <td>{{ target.pose?.x.toFixed(2) }}&nbsp;m</td>
                 <td>{{ target.pose?.y.toFixed(2) }}&nbsp;m</td>
-                <td>{{ ((target.pose?.angle_z * 180.0) / Math.PI).toFixed(2) }}&deg;</td>
+                <td>{{ (((target.pose?.angle_z || 0) * 180.0) / Math.PI).toFixed(2) }}&deg;</td>
               </template>
               <template
                 v-if="
                   useCameraSettingsStore().currentPipelineType === PipelineType.AprilTag &&
                   useCameraSettingsStore().currentPipelineSettings.solvePNPEnabled
                 "
               >
-                <td>{{ target.ambiguity?.toFixed(2) }}%</td>
+                <td>{{ target.ambiguity >= 0 ? target.ambiguity?.toFixed(2) + "%" : "(In Multi-Target)" }}</td>
               </template>
             </tr>
           </tbody>
         </template>
       </v-simple-table>
     </v-row>
+    <v-row
+      v-if="
+        useCameraSettingsStore().currentPipelineSettings.pipelineType === PipelineType.AprilTag &&
+        useCameraSettingsStore().currentPipelineSettings.doMultiTarget
+      "
+      align="start"
+      class="pb-4 white--text"
+    >
+      <v-card-subtitle>Multi-tag pose, field-to-camera</v-card-subtitle>
+      <v-simple-table fixed-header height="100%" dense dark>
+        <thead style="font-size: 1.25rem">
+          <th class="text-center">X meters</th>
+          <th class="text-center">Y meters</th>
+          <th class="text-center">Z Angle &theta;&deg;</th>
+          <th class="text-center">Tags</th>
+        </thead>
+        <tbody>
+          <td>{{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.x.toFixed(2) }}</td>
+          <td>{{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.y.toFixed(2) }}</td>
+          <td>{{ useStateStore().currentPipelineResults?.multitagResult?.bestTransform.angle_z.toFixed(2) }}</td>
+          <td>{{ useStateStore().currentPipelineResults?.multitagResult?.fiducialIDsUsed }}</td>
+        </tbody>
+      </v-simple-table>
+    </v-row>
   </div>
 </template>
 

photon-client/src/components/settings/ApriltagControlCard.vue

@@ -0,0 +1,93 @@
+<script setup lang="ts">
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
+import { Euler, Quaternion as ThreeQuat } from "three";
+import type { Quaternion } from "@/types/PhotonTrackingTypes";
+
+const quatToEuler = (quat: Quaternion): Euler => {
+  const three_quat = new ThreeQuat(quat.X, quat.Y, quat.Z, quat.W);
+  return new Euler().setFromQuaternion(three_quat, "ZYX");
+};
+
+// Convert from radians to degrees.
+const degrees = (radians: number): number => (radians * 180) / Math.PI;
+</script>
+
+<template>
+  <v-card dark class="mb-3 pr-6 pb-3" style="background-color: #006492">
+    <v-card-title>Apriltag Layout</v-card-title>
+    <div class="ml-5">
+      <p>Field width: {{ useSettingsStore().currentFieldLayout.field.width.toFixed(2) }} meters</p>
+      <p>Field length: {{ useSettingsStore().currentFieldLayout.field.length.toFixed(2) }} meters</p>
+
+      <!-- Simple table height must be set here and in the CSS for the fixed-header to work -->
+      <v-simple-table fixed-header height="100%" dense dark>
+        <template #default>
+          <thead style="font-size: 1.25rem">
+            <tr>
+              <th class="text-center">ID</th>
+              <th class="text-center">X, meters</th>
+              <th class="text-center">Y, meters</th>
+              <th class="text-center">Z, meters</th>
+              <th class="text-center">θ<sub>x</sub> angle, &deg;</th>
+              <th class="text-center">θ<sub>y</sub> angle, &deg;</th>
+              <th class="text-center">θ<sub>z</sub> angle, &deg;</th>
+            </tr>
+          </thead>
+          <tbody>
+            <tr v-for="(tag, index) in useSettingsStore().currentFieldLayout.tags" :key="index">
+              <td>{{ tag.ID }}</td>
+              <td v-for="(val, idx) in Object.values(tag.pose.translation).slice(0, 3).map(degrees)" :key="idx">
+                {{ val.toFixed(2) }}
+              </td>
+              <td
+                v-for="(val, idx) in Object.values(quatToEuler(tag.pose.rotation.quaternion)).slice(0, 3).map(degrees)"
+                :key="idx"
+              >
+                {{ val.toFixed(2) }}
+              </td>
+            </tr>
+          </tbody>
+        </template>
+      </v-simple-table>
+    </div>
+  </v-card>
+</template>
+
+<style scoped lang="scss">
+.v-data-table {
+  width: 100%;
+  height: 100%;
+  text-align: center;
+  background-color: #006492 !important;
+
+  th,
+  td {
+    background-color: #006492 !important;
+    font-size: 1rem !important;
+  }
+
+  td {
+    font-family: monospace !important;
+  }
+
+  tbody :hover td {
+    background-color: #005281 !important;
+  }
+
+  ::-webkit-scrollbar {
+    width: 0;
+    height: 0.55em;
+    border-radius: 5px;
+  }
+
+  ::-webkit-scrollbar-track {
+    -webkit-box-shadow: inset 0 0 6px rgba(0, 0, 0, 0.3);
+    border-radius: 10px;
+  }
+
+  ::-webkit-scrollbar-thumb {
+    background-color: #ffd843;
+    border-radius: 10px;
+  }
+}
+</style>

photon-client/src/components/settings/DeviceControlCard.vue

@@ -135,7 +135,8 @@ enum ImportType {
   AllSettings,
   HardwareConfig,
   HardwareSettings,
-  NetworkConfig
+  NetworkConfig,
+  ApriltagFieldLayout
 }
 const showImportDialog = ref(false);
 const importType = ref<ImportType | number>(-1);
@@ -157,6 +158,9 @@ const handleSettingsImport = () => {
     case ImportType.NetworkConfig:
       settingsEndpoint = "/networkConfig";
       break;
+    case ImportType.ApriltagFieldLayout:
+      settingsEndpoint = "/aprilTagFieldLayout";
+      break;
     default:
     case ImportType.AllSettings:
       settingsEndpoint = "";
@@ -249,7 +253,13 @@ const handleSettingsImport = () => {
                     v-model="importType"
                     label="Type"
                     tooltip="Select the type of settings file you are trying to upload"
-                    :items="['All Settings', 'Hardware Config', 'Hardware Settings', 'Network Config']"
+                    :items="[
+                      'All Settings',
+                      'Hardware Config',
+                      'Hardware Settings',
+                      'Network Config',
+                      'Apriltag Layout'
+                    ]"
                     :select-cols="10"
                     style="width: 100%"
                   />

photon-client/src/components/settings/MetricsCard.vue

@@ -72,12 +72,12 @@ const fetchMetrics = () => {
       if (error.request) {
         useStateStore().showSnackbarMessage({
           color: "error",
-          message: "Unable to fetch Metrics! The backend didn't respond."
+          message: "Unable to fetch metrics! The backend didn't respond."
         });
       } else {
         useStateStore().showSnackbarMessage({
           color: "error",
-          message: "An error occurred while trying to fetch Metrics."
+          message: "An error occurred while trying to fetch metrics."
         });
       }
     })

photon-client/src/stores/StateStore.ts

@@ -24,7 +24,7 @@ interface StateStore {
   logMessages: LogMessage[];
   currentCameraIndex: number;
 
-  pipelineResults?: PipelineResult;
+  backendResults: Record<string, PipelineResult>;
 
   colorPickingMode: boolean;
 
@@ -58,7 +58,7 @@ export const useStateStore = defineStore("state", {
       logMessages: [],
       currentCameraIndex: 0,
 
-      pipelineResults: undefined,
+      backendResults: {},
 
       colorPickingMode: false,
 
@@ -77,6 +77,11 @@ export const useStateStore = defineStore("state", {
       }
     };
   },
+  getters: {
+    currentPipelineResults(): PipelineResult | undefined {
+      return this.backendResults[this.currentCameraIndex.toString()];
+    }
+  },
   actions: {
     setSidebarFolded(value: boolean) {
       this.sidebarFolded = value;
@@ -95,12 +100,11 @@ export const useStateStore = defineStore("state", {
         clients: data.clients
       };
     },
-    updatePipelineResultsFromWebsocket(data: WebsocketPipelineResultUpdate) {
-      for (const cameraIndex in data) {
-        if (parseInt(cameraIndex) === this.currentCameraIndex) {
-          this.pipelineResults = data[cameraIndex];
-        }
-      }
+    updateBackendResultsFromWebsocket(data: WebsocketPipelineResultUpdate) {
+      this.backendResults = {
+        ...this.backendResults,
+        ...data
+      };
     },
     updateCalibrationStateValuesFromWebsocket(data: WebsocketCalibrationData) {
       this.calibrationData = {