From 7f09f9e4f5b4237ef4b9dde7fdcb747115315659 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 3 Jan 2024 14:32:04 -0700 Subject: [PATCH] Save calibration data and show preliminary GUI (#1078) * Serialize all calibration data * Run lint * typing nit * fix code * move these tables around some * Add cool formatting * add request to get snapshots by resolution and camera * re-enable all resolutions * add wip so i can change computers (SQUASH ME AND KILL ME AHHHH) * Get everything working but viewing snapshots * Update RequestHandler.java * Update CameraCalibrationInfoCard.vue * Update CameraCalibrationInfoCard.vue * add observation viewer * round * fix illiegal import * Swap to PNG and serialize insolution * move import/export buttons TO THE TOP * Update WebsocketDataTypes.ts * Add snapshotname to observation * Refactor to serialize snapshot image itself * Run lint * Use new base64 image data in info card * Update SettingTypes.ts * Create calibration json -> mrcal converter script * Update calibrationUtils.py * Fix calibrate NPEs in teest * Run lint * Always run cornersubpix * Update CameraCalibrationInfoCard.vue Update CameraCalibrationInfoCard.vue * Update OpenCVHelp.java * Update OpenCVHelp.java * Replace test mode camera JSONs * Run wpiformat * Revert intrinsics but keep other data * Remove misc comments * Rename JsonMat->JsonImageMat and add calobject_warp * Update Server.java * Rename cameraExtrinsics to distCoeffs * fix typing issues * use util methods * Formatting fixes * fix styling * move to devTools * remove unneeded or unused imports * Remove fixed-right css If its really that big of a deal, we can add it back later, kind of a drag to fix rn. * Create util method * Remove extra legacy calibration things --------- Co-authored-by: Sriman Achanta <68172138+srimanachanta@users.noreply.github.com> --- .gitignore | 2 + devTools/calibrationUtils.py | 134 ++++++++++ .../cameras/CameraCalibrationCard.vue | 61 +++-- .../cameras/CameraCalibrationInfoCard.vue | 251 ++++++++++++++++++ .../src/components/cameras/CamerasView.vue | 7 +- .../stores/settings/CameraSettingsStore.ts | 42 ++- .../src/types/PhotonTrackingTypes.ts | 41 +-- photon-client/src/types/SettingTypes.ts | 96 ++++++- photon-client/src/types/WebsocketDataTypes.ts | 22 +- .../configuration/PhotonConfiguration.java | 4 +- .../networktables/NTDataPublisher.java | 2 +- .../photonvision/common/util/ColorHelper.java | 4 + .../photonvision/common/util/TestUtils.java | 8 +- .../common/util/file/JacksonUtils.java | 14 + .../common/util/math/MathUtils.java | 21 ++ .../vision/calibration/BoardObservation.java | 71 +++++ .../CameraCalibrationCoefficients.java | 77 +++--- .../vision/calibration/JsonImageMat.java | 79 ++++++ .../{JsonMat.java => JsonMatOfDouble.java} | 11 +- .../vision/pipe/impl/Calibrate3dPipe.java | 114 +++++--- .../vision/pipe/impl/DrawCalibrationPipe.java | 62 +++++ .../pipe/impl/FindBoardCornersPipe.java | 72 +++-- .../vision/pipeline/Calibrate3dPipeline.java | 74 +++--- .../vision/pipeline/OutputStreamPipeline.java | 16 +- .../vision/pipeline/UICalibrationData.java | 30 +-- .../result/CalibrationPipelineResult.java | 35 +++ .../vision/processes/PipelineManager.java | 14 +- .../vision/processes/VisionModule.java | 23 +- .../VisionModuleChangeSubscriber.java | 13 +- .../vision/target/TrackedTarget.java | 8 + .../vision/pipeline/Calibrate3dPipeTest.java | 33 +-- .../vision/processes/PipelineManagerTest.java | 3 +- .../simulation/SimCameraProperties.java | 2 +- .../photonvision/server/RequestHandler.java | 78 +++++- .../java/org/photonvision/server/Server.java | 9 +- .../resources/calibration/lifecam240p.json | 2 +- .../resources/calibration/lifecam480p.json | 2 +- test-resources/calibration/laptop.json | 2 +- test-resources/calibration/laptop_1280.json | 2 +- test-resources/calibration/lifecam240p.json | 35 +-- test-resources/calibration/lifecam480p.json | 35 +-- test-resources/calibration/lifecam_1280.json | 32 +-- .../calibration/limelight_1280_720.json | 2 +- 43 files changed, 1248 insertions(+), 397 deletions(-) create mode 100644 devTools/calibrationUtils.py create mode 100644 photon-client/src/components/cameras/CameraCalibrationInfoCard.vue create mode 100644 photon-core/src/main/java/org/photonvision/vision/calibration/BoardObservation.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/calibration/JsonImageMat.java rename photon-core/src/main/java/org/photonvision/vision/calibration/{JsonMat.java => JsonMatOfDouble.java} (91%) create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipe/impl/DrawCalibrationPipe.java create mode 100644 photon-core/src/main/java/org/photonvision/vision/pipeline/result/CalibrationPipelineResult.java diff --git a/.gitignore b/.gitignore index 766874b000..8eb27213b4 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,5 @@ photonlib-cpp-examples/*/networktables.json.bck photonlib-java-examples/*/networktables.json.bck *.sqlite photon-server/src/main/resources/web/index.html + +venv diff --git a/devTools/calibrationUtils.py b/devTools/calibrationUtils.py new file mode 100644 index 0000000000..97ba70c425 --- /dev/null +++ b/devTools/calibrationUtils.py @@ -0,0 +1,134 @@ +import base64 +from dataclasses import dataclass +import json +import os +import cv2 +import numpy as np + + +@dataclass +class Resolution: + width: int + height: int + + +@dataclass +class JsonMatOfDoubles: + rows: int + cols: int + type: int + data: list[float] + + +@dataclass +class JsonMat: + rows: int + cols: int + type: int + data: str # Base64-encoded PNG data + + +@dataclass +class Point2: + x: float + y: float + + +@dataclass +class Translation3d: + x: float + y: float + z: float + + +@dataclass +class Quaternion: + X: float + Y: float + Z: float + W: float + + +@dataclass +class Rotation3d: + quaternion: Quaternion + + +@dataclass +class Pose3d: + translation: Translation3d + rotation: Rotation3d + + +@dataclass +class Point3: + x: float + y: float + z: float + + +@dataclass +class Observation: + # Expected feature 3d location in the camera frame + locationInObjectSpace: list[Point3] + # Observed location in pixel space + locationInImageSpace: list[Point2] + # (measured location in pixels) - (expected from FK) + reprojectionErrors: list[Point2] + # Solver optimized board poses + optimisedCameraToObject: Pose3d + # If we should use this observation when re-calculating camera calibration + includeObservationInCalibration: bool + snapshotName: str + # The actual image the snapshot is from + snapshotData: JsonMat + + +@dataclass +class CameraCalibration: + resolution: Resolution + cameraIntrinsics: JsonMatOfDoubles + distCoeffs: JsonMatOfDoubles + observations: list[Observation] + + +def convert_photon_to_mrcal(photon_cal_json_path: str, output_folder: str): + """ + Unpack a Photon calibration JSON (eg, photon_calibration_Microsoft_LifeCam_HD-3000_800x600.json) into + the output_folder directory with images and corners.vnl file for use with mrcal. + """ + with open(photon_cal_json_path, "r") as cal_json: + # Convert to nested objects instead of nameddicts on json-loads + class Generic: + @classmethod + def from_dict(cls, dict): + obj = cls() + obj.__dict__.update(dict) + return obj + + camera_cal_data: CameraCalibration = json.loads( + cal_json.read(), object_hook=Generic.from_dict + ) + + # Create output_folder if not exists + if not os.path.exists(output_folder): + os.makedirs(output_folder) + + # Decode each image and save it as a png + for obs in camera_cal_data.observations: + image = obs.snapshotData.data + decoded_data = base64.b64decode(image) + np_data = np.frombuffer(decoded_data, np.uint8) + img = cv2.imdecode(np_data, cv2.IMREAD_UNCHANGED) + cv2.imwrite(f"{output_folder}/{obs.snapshotName}", img) + + # And create a VNL file for use with mrcal + with open(f"{output_folder}/corners.vnl", "w+") as vnl_file: + vnl_file.write("# filename x y level\n") + + for obs in camera_cal_data.observations: + for corner in obs.locationInImageSpace: + # Always level zero + vnl_file.write(f"{obs.snapshotName} {corner.x} {corner.y} 0\n") + + vnl_file.flush() diff --git a/photon-client/src/components/cameras/CameraCalibrationCard.vue b/photon-client/src/components/cameras/CameraCalibrationCard.vue index e2d5e7d1f1..0b5163dcb8 100644 --- a/photon-client/src/components/cameras/CameraCalibrationCard.vue +++ b/photon-client/src/components/cameras/CameraCalibrationCard.vue @@ -1,7 +1,7 @@ @@ -494,6 +512,7 @@ const endCalibration = () => { tbody :hover td { background-color: #005281 !important; + cursor: pointer; } ::-webkit-scrollbar { diff --git a/photon-client/src/components/cameras/CameraCalibrationInfoCard.vue b/photon-client/src/components/cameras/CameraCalibrationInfoCard.vue new file mode 100644 index 0000000000..4177ca5c09 --- /dev/null +++ b/photon-client/src/components/cameras/CameraCalibrationInfoCard.vue @@ -0,0 +1,251 @@ + + + + + diff --git a/photon-client/src/components/cameras/CamerasView.vue b/photon-client/src/components/cameras/CamerasView.vue index f1dbfef7d8..4b4530f985 100644 --- a/photon-client/src/components/cameras/CamerasView.vue +++ b/photon-client/src/components/cameras/CamerasView.vue @@ -41,7 +41,12 @@ const fpsTooLow = computed(() => {