From 4734d8838f74ed29de4cb4c87a3f48a40e949efc Mon Sep 17 00:00:00 2001 From: Stephen Winn Date: Tue, 20 Aug 2024 16:51:59 +0900 Subject: [PATCH 1/5] [WIP] Added steps to include TIFFTAG data in the output .tif --- opendm/dem/utils.py | 8 ++++++++ opendm/orthophoto.py | 7 +++++++ stages/odm_dem.py | 3 +++ stages/odm_orthophoto.py | 3 +++ 4 files changed, 21 insertions(+) diff --git a/opendm/dem/utils.py b/opendm/dem/utils.py index 9fb383a92..507e44ad9 100644 --- a/opendm/dem/utils.py +++ b/opendm/dem/utils.py @@ -1,3 +1,4 @@ +import rasterio def get_dem_vars(args): return { @@ -8,3 +9,10 @@ def get_dem_vars(args): 'BIGTIFF': 'IF_SAFER', 'NUM_THREADS': args.max_concurrency, } + +def update_tags(dem_file): + + # - Update the geotiff tags in place using rasterio + with rasterio.open(dem_file, 'r+') as rst: + rst.update_tags(TIFFTAG_DATETIME='2024:01:01 00:00+00:00') + rst.update_tags(TIFFTAG_SOFTWARE='OpenDroneMap') diff --git a/opendm/orthophoto.py b/opendm/orthophoto.py index 319a504cb..663f3cb24 100644 --- a/opendm/orthophoto.py +++ b/opendm/orthophoto.py @@ -27,6 +27,13 @@ def get_orthophoto_vars(args): 'NUM_THREADS': args.max_concurrency } +def update_tags(orthophoto_file): + + # - Update the geotiff tags in place using rasterio + with rasterio.open(orthophoto_file, 'r+') as rst: + rst.update_tags(TIFFTAG_DATETIME='2024:01:01 00:00+00:00') + rst.update_tags(TIFFTAG_SOFTWARE='OpenDroneMap') + def build_overviews(orthophoto_file): log.ODM_INFO("Building Overviews") kwargs = {'orthophoto': orthophoto_file} diff --git a/stages/odm_dem.py b/stages/odm_dem.py index 6964e036d..f96712151 100755 --- a/stages/odm_dem.py +++ b/stages/odm_dem.py @@ -80,6 +80,9 @@ def process(self, args, outputs): dem_geotiff_path = os.path.join(odm_dem_root, "{}.tif".format(product)) bounds_file_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') + # update dem tags + utils.update_tags(dem_geotiff_path) + if args.crop > 0 or args.boundary: # Crop DEM Cropper.crop(bounds_file_path, dem_geotiff_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space) diff --git a/stages/odm_orthophoto.py b/stages/odm_orthophoto.py index e66d89db1..916770648 100644 --- a/stages/odm_orthophoto.py +++ b/stages/odm_orthophoto.py @@ -109,6 +109,9 @@ def process(self, args, outputs): '-outputCornerFile "{corners}" {bands} {depth_idx} {inpaint} ' '{utm_offsets} {a_srs} {vars} {gdal_configs} '.format(**kwargs), env_vars={'OMP_NUM_THREADS': args.max_concurrency}) + # update output orthophoto tags + orthophoto.update_tags(tree.odm_orthophoto_tif) + # Create georeferenced GeoTiff if reconstruction.is_georeferenced(): bounds_file_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') From 635a8a362b531a284f5330b82dc5b90f83d8c1bc Mon Sep 17 00:00:00 2001 From: Stephen Winn Date: Thu, 5 Sep 2024 14:28:52 +0900 Subject: [PATCH 2/5] Revert "[WIP] Added steps to include TIFFTAG data in the output .tif" This reverts commit 4734d8838f74ed29de4cb4c87a3f48a40e949efc. --- opendm/dem/utils.py | 8 -------- opendm/orthophoto.py | 7 ------- stages/odm_dem.py | 3 --- stages/odm_orthophoto.py | 3 --- 4 files changed, 21 deletions(-) diff --git a/opendm/dem/utils.py b/opendm/dem/utils.py index 507e44ad9..9fb383a92 100644 --- a/opendm/dem/utils.py +++ b/opendm/dem/utils.py @@ -1,4 +1,3 @@ -import rasterio def get_dem_vars(args): return { @@ -9,10 +8,3 @@ def get_dem_vars(args): 'BIGTIFF': 'IF_SAFER', 'NUM_THREADS': args.max_concurrency, } - -def update_tags(dem_file): - - # - Update the geotiff tags in place using rasterio - with rasterio.open(dem_file, 'r+') as rst: - rst.update_tags(TIFFTAG_DATETIME='2024:01:01 00:00+00:00') - rst.update_tags(TIFFTAG_SOFTWARE='OpenDroneMap') diff --git a/opendm/orthophoto.py b/opendm/orthophoto.py index 663f3cb24..319a504cb 100644 --- a/opendm/orthophoto.py +++ b/opendm/orthophoto.py @@ -27,13 +27,6 @@ def get_orthophoto_vars(args): 'NUM_THREADS': args.max_concurrency } -def update_tags(orthophoto_file): - - # - Update the geotiff tags in place using rasterio - with rasterio.open(orthophoto_file, 'r+') as rst: - rst.update_tags(TIFFTAG_DATETIME='2024:01:01 00:00+00:00') - rst.update_tags(TIFFTAG_SOFTWARE='OpenDroneMap') - def build_overviews(orthophoto_file): log.ODM_INFO("Building Overviews") kwargs = {'orthophoto': orthophoto_file} diff --git a/stages/odm_dem.py b/stages/odm_dem.py index f96712151..6964e036d 100755 --- a/stages/odm_dem.py +++ b/stages/odm_dem.py @@ -80,9 +80,6 @@ def process(self, args, outputs): dem_geotiff_path = os.path.join(odm_dem_root, "{}.tif".format(product)) bounds_file_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') - # update dem tags - utils.update_tags(dem_geotiff_path) - if args.crop > 0 or args.boundary: # Crop DEM Cropper.crop(bounds_file_path, dem_geotiff_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space) diff --git a/stages/odm_orthophoto.py b/stages/odm_orthophoto.py index 916770648..e66d89db1 100644 --- a/stages/odm_orthophoto.py +++ b/stages/odm_orthophoto.py @@ -109,9 +109,6 @@ def process(self, args, outputs): '-outputCornerFile "{corners}" {bands} {depth_idx} {inpaint} ' '{utm_offsets} {a_srs} {vars} {gdal_configs} '.format(**kwargs), env_vars={'OMP_NUM_THREADS': args.max_concurrency}) - # update output orthophoto tags - orthophoto.update_tags(tree.odm_orthophoto_tif) - # Create georeferenced GeoTiff if reconstruction.is_georeferenced(): bounds_file_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') From 6ecc8274f81fc6b209b429bc4911ba03e324b470 Mon Sep 17 00:00:00 2001 From: Stephen Winn Date: Thu, 5 Sep 2024 17:28:07 +0900 Subject: [PATCH 3/5] Add TIFFTAG_* information to the orthphoto/dsm/dtm Reverted previous changes and moved all steps into the ODMPostProcess step. --- stages/odm_postprocess.py | 58 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/stages/odm_postprocess.py b/stages/odm_postprocess.py index 95fac779f..1fab36069 100644 --- a/stages/odm_postprocess.py +++ b/stages/odm_postprocess.py @@ -1,9 +1,14 @@ import os +import rasterio +import json +import numpy as np +from datetime import datetime from osgeo import gdal from opendm import io from opendm import log from opendm import types +from opendm import context from opendm.utils import copy_paths, get_processing_results_paths from opendm.ogctiles import build_3dtiles @@ -14,6 +19,56 @@ def process(self, args, outputs): log.ODM_INFO("Post Processing") + # -- TIFFTAG information - add datetime and software version to .tif metadata + + # Gather information + # ODM version ... + with open(os.path.join(context.root_path, 'VERSION')) as version_file: + version = version_file.read().strip() + # Datetimes ... + shots_file = tree.path("odm_report", "shots.geojson") + if not args.skip_report and os.path.isfile(shots_file): + # Open file + with open(shots_file, 'r') as f: + odm_shots = json.loads(f.read()) + # Compute mean time + cts = [] + for feat in odm_shots["features"]: + ct = feat["properties"]["capture_time"] + cts.append(ct) + mean_dt = datetime.fromtimestamp(np.mean(cts)) + # Format it + CAPTURE_DATETIME = mean_dt.strftime('%Y:%m:%d %H:%M') + '+00:00' #UTC + else: + #try instead with images.json file + images_file = tree.path('images.json') + if os.path.isfile(images_file): + # Open file + with open(images_file, 'r') as f: + imgs = json.loads(f.read()) + # Compute mean time + cts = [] + for img in imgs: + ct = img["utc_time"]/1000. #ms to s + cts.append(ct) + mean_dt = datetime.fromtimestamp(np.mean(cts)) + # Format it + CAPTURE_DATETIME = mean_dt.strftime('%Y:%m:%d %H:%M') + '+00:00' #UTC + else: + CAPTURE_DATETIME = None + + # Add it + for product in [tree.odm_orthophoto_tif, + tree.path("odm_dem", "dsm.tif"), + tree.path("odm_dem", "dtm.tif")]: + for pdt in [product, product.replace('.tif', '.original.tif')]: + if os.path.isfile(pdt): + log.ODM_INFO("Adding TIFFTAGs to {} ...".format(pdt)) + with rasterio.open(pdt, 'r+') as rst: + rst.update_tags(TIFFTAG_DATETIME=CAPTURE_DATETIME) + rst.update_tags(TIFFTAG_SOFTWARE='OpenDroneMap {}'.format(version)) + + # -- GCP info if not outputs['large']: # TODO: support for split-merge? @@ -45,11 +100,14 @@ def process(self, args, outputs): else: log.ODM_WARNING("Cannot open %s for writing, skipping GCP embedding" % product) + # -- 3D tiles if getattr(args, '3d_tiles'): build_3dtiles(args, tree, reconstruction, self.rerun()) + # -- Copy to if args.copy_to: try: copy_paths([os.path.join(args.project_path, p) for p in get_processing_results_paths()], args.copy_to, self.rerun()) except Exception as e: log.ODM_WARNING("Cannot copy to %s: %s" % (args.copy_to, str(e))) + From f0ddaba12e73657143e12ffc165dcdc46794bcf0 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 5 Sep 2024 14:27:16 -0400 Subject: [PATCH 4/5] Refactoring, use odm_version(), add seconds --- opendm/log.py | 2 ++ opendm/photo.py | 11 ++++++ stages/odm_postprocess.py | 71 ++++++++++----------------------------- 3 files changed, 31 insertions(+), 53 deletions(-) diff --git a/opendm/log.py b/opendm/log.py index 8e04d7791..fbcbfed51 100644 --- a/opendm/log.py +++ b/opendm/log.py @@ -6,6 +6,7 @@ import dateutil.parser import shutil import multiprocessing +from repoze.lru import lru_cache from opendm.arghelpers import double_quote, args_to_dict from vmem import virtual_memory @@ -30,6 +31,7 @@ lock = threading.Lock() +@lru_cache(maxsize=None) def odm_version(): with open(os.path.join(os.path.dirname(__file__), "..", "VERSION")) as f: return f.read().split("\n")[0].strip() diff --git a/opendm/photo.py b/opendm/photo.py index d6921c0ae..7549458fe 100644 --- a/opendm/photo.py +++ b/opendm/photo.py @@ -21,6 +21,17 @@ projections = ['perspective', 'fisheye', 'fisheye_opencv', 'brown', 'dual', 'equirectangular', 'spherical'] +def find_mean_utc_time(photos): + utc_times = [] + for p in photos: + if p.utc_time is not None: + utc_times.append(p.utc_time / 1000.0) + if len(utc_times) == 0: + return None + + return np.mean(utc_times) + + def find_largest_photo_dims(photos): max_mp = 0 max_dims = None diff --git a/stages/odm_postprocess.py b/stages/odm_postprocess.py index 1fab36069..4b425eb20 100644 --- a/stages/odm_postprocess.py +++ b/stages/odm_postprocess.py @@ -8,7 +8,7 @@ from opendm import io from opendm import log from opendm import types -from opendm import context +from opendm import photo from opendm.utils import copy_paths, get_processing_results_paths from opendm.ogctiles import build_3dtiles @@ -19,56 +19,25 @@ def process(self, args, outputs): log.ODM_INFO("Post Processing") - # -- TIFFTAG information - add datetime and software version to .tif metadata + rasters = [tree.odm_orthophoto_tif, + tree.path("odm_dem", "dsm.tif"), + tree.path("odm_dem", "dtm.tif")] - # Gather information - # ODM version ... - with open(os.path.join(context.root_path, 'VERSION')) as version_file: - version = version_file.read().strip() - # Datetimes ... - shots_file = tree.path("odm_report", "shots.geojson") - if not args.skip_report and os.path.isfile(shots_file): - # Open file - with open(shots_file, 'r') as f: - odm_shots = json.loads(f.read()) - # Compute mean time - cts = [] - for feat in odm_shots["features"]: - ct = feat["properties"]["capture_time"] - cts.append(ct) - mean_dt = datetime.fromtimestamp(np.mean(cts)) - # Format it - CAPTURE_DATETIME = mean_dt.strftime('%Y:%m:%d %H:%M') + '+00:00' #UTC - else: - #try instead with images.json file - images_file = tree.path('images.json') - if os.path.isfile(images_file): - # Open file - with open(images_file, 'r') as f: - imgs = json.loads(f.read()) - # Compute mean time - cts = [] - for img in imgs: - ct = img["utc_time"]/1000. #ms to s - cts.append(ct) - mean_dt = datetime.fromtimestamp(np.mean(cts)) - # Format it - CAPTURE_DATETIME = mean_dt.strftime('%Y:%m:%d %H:%M') + '+00:00' #UTC - else: - CAPTURE_DATETIME = None + mean_capture_time = photo.find_mean_utc_time(reconstruction.photos) + mean_capture_dt = None + if mean_capture_time is not None: + mean_capture_dt = datetime.fromtimestamp(mean_capture_time).strftime('%Y:%m:%d %H:%M:%S') + '+00:00' - # Add it - for product in [tree.odm_orthophoto_tif, - tree.path("odm_dem", "dsm.tif"), - tree.path("odm_dem", "dtm.tif")]: - for pdt in [product, product.replace('.tif', '.original.tif')]: - if os.path.isfile(pdt): - log.ODM_INFO("Adding TIFFTAGs to {} ...".format(pdt)) - with rasterio.open(pdt, 'r+') as rst: - rst.update_tags(TIFFTAG_DATETIME=CAPTURE_DATETIME) - rst.update_tags(TIFFTAG_SOFTWARE='OpenDroneMap {}'.format(version)) + # Add TIFF tags + for product in rasters: + if os.path.isfile(product): + log.ODM_INFO("Adding TIFFTAGs to {}".format(product)) + with rasterio.open(product, 'r+') as rst: + if mean_capture_dt is not None: + rst.update_tags(TIFFTAG_DATETIME=mean_capture_dt) + rst.update_tags(TIFFTAG_SOFTWARE='ODM {}'.format(log.odm_version())) - # -- GCP info + # GCP info if not outputs['large']: # TODO: support for split-merge? @@ -83,9 +52,7 @@ def process(self, args, outputs): with open(gcp_gml_export_file) as f: gcp_xml = f.read() - for product in [tree.odm_orthophoto_tif, - tree.path("odm_dem", "dsm.tif"), - tree.path("odm_dem", "dtm.tif")]: + for product in rasters: if os.path.isfile(product): ds = gdal.Open(product) if ds is not None: @@ -100,11 +67,9 @@ def process(self, args, outputs): else: log.ODM_WARNING("Cannot open %s for writing, skipping GCP embedding" % product) - # -- 3D tiles if getattr(args, '3d_tiles'): build_3dtiles(args, tree, reconstruction, self.rerun()) - # -- Copy to if args.copy_to: try: copy_paths([os.path.join(args.project_path, p) for p in get_processing_results_paths()], args.copy_to, self.rerun()) From 33e850086070a30626c4c3a5e7673f308ddb2e9b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 5 Sep 2024 14:31:08 -0400 Subject: [PATCH 5/5] Cleanup unused imports --- stages/odm_postprocess.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stages/odm_postprocess.py b/stages/odm_postprocess.py index 4b425eb20..1f1da8da3 100644 --- a/stages/odm_postprocess.py +++ b/stages/odm_postprocess.py @@ -1,7 +1,5 @@ import os import rasterio -import json -import numpy as np from datetime import datetime from osgeo import gdal