From 995f039082d2ddc010c977490105f599d4147fee Mon Sep 17 00:00:00 2001 From: CSD-RD Date: Thu, 22 Nov 2018 14:31:04 +0100 Subject: [PATCH 1/6] Added support for masks in DenseReconstruction. --- apps/DensifyPointCloud/DensifyPointCloud.cpp | 2 +- libs/MVS/Scene.h | 2 +- libs/MVS/SceneDensify.cpp | 30 +++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/apps/DensifyPointCloud/DensifyPointCloud.cpp b/apps/DensifyPointCloud/DensifyPointCloud.cpp index 82874cef6..16bdde305 100644 --- a/apps/DensifyPointCloud/DensifyPointCloud.cpp +++ b/apps/DensifyPointCloud/DensifyPointCloud.cpp @@ -217,7 +217,7 @@ int main(int argc, LPCTSTR* argv) return EXIT_FAILURE; } TD_TIMER_START(); - if (!scene.DenseReconstruction()) + if (!scene.DenseReconstruction(std::vector())) return EXIT_FAILURE; VERBOSE("Densifying point-cloud completed: %u points (%s)", scene.pointcloud.points.GetSize(), TD_TIMER_GET_FMT().c_str()); diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index 41d50c255..e250e7f6f 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -75,7 +75,7 @@ class MVS_API Scene bool ExportCamerasMLP(const String& fileName, const String& fileNameScene) const; // Dense reconstruction - bool DenseReconstruction(); + bool DenseReconstruction(std::vector& theMasks, std::vector> &images = std::vector>(), const uint32_t &width = 0, const uint32_t &height = 0, const uint32_t &depth = 0); void DenseReconstructionEstimate(void*); void DenseReconstructionFilter(void*); diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index d54c03049..ef2301f72 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -130,6 +130,7 @@ class DepthMapsData bool SelectViews(IndexArr& images, IndexArr& imagesMap, IndexArr& neighborsMap); bool SelectViews(DepthData& depthData); + bool SetMasks(DepthData& depthData, BitMatrix& mask); bool InitViews(DepthData& depthData, uint32_t idxNeighbor, uint32_t numNeighbors); bool InitDepthMap(DepthData& depthData); bool EstimateDepthMap(uint32_t idxImage); @@ -326,6 +327,14 @@ bool DepthMapsData::SelectViews(DepthData& depthData) } // SelectViews /*----------------------------------------------------------------*/ +bool DepthMapsData::SetMasks(DepthData& depthData, BitMatrix& mask) +{ + depthData.mask.create(mask.size()); + depthData.mask.swap(mask); + return true; +}// SetMasks +/*----------------------------------------------------------------*/ + // select target image for the reference image (the first image in "images") // and initialize images data; // if idxNeighbor is not NO_ID, only the reference image and the given neighbor are initialized; @@ -1499,7 +1508,7 @@ struct DenseDepthMapData { static void* DenseReconstructionEstimateTmp(void*); static void* DenseReconstructionFilterTmp(void*); -bool Scene::DenseReconstruction() +bool Scene::DenseReconstruction(std::vector& theMasks, std::vector> &imageVector, const uint32_t &width, const uint32_t &height, const uint32_t &depth) { DenseDepthMapData data(*this); @@ -1596,6 +1605,25 @@ bool Scene::DenseReconstruction() imagesMap[data.images[idx]] = NO_ID; data.images.RemoveAt(idx); } + + if (!theMasks.empty()) + { +#ifdef DENSE_USE_OPENMP +#pragma omp parallel for + for (int_t ID = 0; ID < (int_t)data.images.GetSize(); ++ID) + { + const uint32_t idx((uint32_t)ID); +#else + FOREACH(idx, data.images) + { +#endif + const uint32_t idxImage(data.images[idx]); + DepthData& depthData(data.detphMaps.arrDepthData[idxImage]); + + data.detphMaps.SetMasks(depthData, theMasks[idxImage]); + } + } + // globally select a target view for each reference image if (OPTDENSE::nNumViews == 1 && !data.detphMaps.SelectViews(data.images, imagesMap, data.neighborsMap)) { VERBOSE("error: no valid images to be dense reconstructed"); From 91c3abf75dad0fe32f70b4e81b914bd7e33d90ad Mon Sep 17 00:00:00 2001 From: CSD-RD Date: Thu, 22 Nov 2018 14:35:26 +0100 Subject: [PATCH 2/6] Changed implementation to add support for more generic ostream/istream instead of ofstream/ifstream. --- libs/Common/Types.inl | 16 +++++++------- libs/MVS/DepthMap.cpp | 50 +++++++++++++++++++++++++++++++++++++++++++ libs/MVS/DepthMap.h | 7 ++++++ libs/MVS/Interface.h | 13 +++++++++++ libs/MVS/Scene.cpp | 18 ++++++++++++++++ libs/MVS/Scene.h | 5 +++++ 6 files changed, 101 insertions(+), 8 deletions(-) diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index 6a7c15761..3944df8e4 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -3378,23 +3378,23 @@ enum ARCHIVE_TYPE { // export the current state of the given reconstruction object template -bool SerializeSave(const TYPE& obj, std::ofstream& fs, ARCHIVE_TYPE type, unsigned flags=0) +bool SerializeSave(const TYPE& obj, std::ostream& os, ARCHIVE_TYPE type, unsigned flags=0) { // serialize out the current state switch (type) { case ARCHIVE_TEXT: { - boost::archive::text_oarchive ar(fs, flags); + boost::archive::text_oarchive ar(os, flags); ar << obj; break; } case ARCHIVE_BINARY: { - boost::archive::binary_oarchive ar(fs, flags); + boost::archive::binary_oarchive ar(os, flags); ar << obj; break; } case ARCHIVE_BINARY_ZIP: { namespace io = boost::iostreams; io::filtering_streambuf ffs; ffs.push(io::zlib_compressor(io::zlib::best_speed)); - ffs.push(fs); + ffs.push(os); boost::archive::binary_oarchive ar(ffs, flags); ar << obj; break; } @@ -3417,24 +3417,24 @@ bool SerializeSave(const TYPE& obj, const SEACAVE::String& fileName, ARCHIVE_TYP // import the state to the given reconstruction object template -bool SerializeLoad(TYPE& obj, std::ifstream& fs, ARCHIVE_TYPE type, unsigned flags=0) +bool SerializeLoad(TYPE& obj, std::istream& is, ARCHIVE_TYPE type, unsigned flags=0) { try { // serialize in the saved state switch (type) { case ARCHIVE_TEXT: { - boost::archive::text_iarchive ar(fs, flags); + boost::archive::text_iarchive ar(is, flags); ar >> obj; break; } case ARCHIVE_BINARY: { - boost::archive::binary_iarchive ar(fs, flags); + boost::archive::binary_iarchive ar(is, flags); ar >> obj; break; } case ARCHIVE_BINARY_ZIP: { namespace io = boost::iostreams; io::filtering_streambuf ffs; ffs.push(io::zlib_decompressor()); - ffs.push(fs); + ffs.push(is); boost::archive::binary_iarchive ar(ffs, flags); ar >> obj; break; } diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 61fc24bea..e8e04b927 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -188,11 +188,23 @@ bool DepthData::Save(const String& fileName) const ASSERT(IsValid() && !depthMap.empty() && !confMap.empty()); return SerializeSave(*this, fileName, ARCHIVE_BINARY_ZIP); } +bool DepthData::Save(std::ostream &os) const +{ + ASSERT(IsValid() && !depthMap.empty() && !confMap.empty()); + return SerializeSave(*this, os, ARCHIVE_BINARY_ZIP); +} bool DepthData::Load(const String& fileName) { ASSERT(IsValid()); return SerializeLoad(*this, fileName, ARCHIVE_BINARY_ZIP); } +bool DepthData::Load(std::istream &is) +{ + ASSERT(IsValid()); + bool ok = SerializeLoad(*this, is, ARCHIVE_BINARY_ZIP); + is.seekg(0, is.beg); + return ok; +} /*----------------------------------------------------------------*/ @@ -209,6 +221,14 @@ unsigned DepthData::IncRef(const String& fileName) return 0; return ++references; } +unsigned DepthData::IncRef(std::istream &is) +{ + Lock l(cs); + ASSERT(!IsEmpty() || references == 0); + if (IsEmpty() && !Load(is)) + return 0; + return ++references; +} unsigned DepthData::DecRef() { Lock l(cs); @@ -712,12 +732,27 @@ bool MVS::SaveDepthMap(const String& fileName, const DepthMap& depthMap) return SerializeSave(depthMap, fileName, ARCHIVE_BINARY_ZIP); } // SaveDepthMap /*----------------------------------------------------------------*/ +// save the depth map in a stream +bool MVS::SaveDepthMap(std::ostream &os, const DepthMap &depthMap) +{ + ASSERT(!depthMap.empty()); + return SerializeSave(depthMap, os, ARCHIVE_BINARY_ZIP); +} // SaveDepthMap +/*----------------------------------------------------------------*/ // load the depth map from our .dmap file format bool MVS::LoadDepthMap(const String& fileName, DepthMap& depthMap) { return SerializeLoad(depthMap, fileName, ARCHIVE_BINARY_ZIP); } // LoadDepthMap /*----------------------------------------------------------------*/ +// load the depth map from a stream +bool MVS::LoadDepthMap(std::istream &is, DepthMap &depthMap) +{ + bool ok = SerializeLoad(depthMap, is, ARCHIVE_BINARY_ZIP); + is.seekg(is.beg); + return ok; +} // LoadDepthMap +/*----------------------------------------------------------------*/ // save the normal map in our .nmap file format bool MVS::SaveNormalMap(const String& fileName, const NormalMap& normalMap) @@ -740,12 +775,27 @@ bool MVS::SaveConfidenceMap(const String& fileName, const ConfidenceMap& confMap return SerializeSave(confMap, fileName, ARCHIVE_BINARY_ZIP); } // SaveConfidenceMap /*----------------------------------------------------------------*/ +// save the confidence map in stream +bool MVS::SaveConfidenceMap(std::ostream &os, const ConfidenceMap& confMap) +{ + ASSERT(!confMap.empty()); + return SerializeSave(confMap, os, ARCHIVE_BINARY_ZIP); +} // SaveConfidenceMap +/*----------------------------------------------------------------*/ // load the confidence map from our .cmap file format bool MVS::LoadConfidenceMap(const String& fileName, ConfidenceMap& confMap) { return SerializeLoad(confMap, fileName, ARCHIVE_BINARY_ZIP); } // LoadConfidenceMap /*----------------------------------------------------------------*/ +// load the confidence map from stream +bool MVS::LoadConfidenceMap(std::istream &is, ConfidenceMap& confMap) +{ + bool ok = SerializeLoad(confMap, is, ARCHIVE_BINARY_ZIP); + is.seekg(is.beg); + return ok; +} // LoadConfidenceMap +/*----------------------------------------------------------------*/ diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 3a4b95885..7e0a3a26f 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -150,10 +150,13 @@ struct MVS_API DepthData { void GetNormal(const Point2f& x, Point3f& N, const TImage* pPointMap=NULL) const; bool Save(const String& fileName) const; + bool Save(std::ostream &os) const; bool Load(const String& fileName); + bool Load(std::istream &is); unsigned GetRef(); unsigned IncRef(const String& fileName); + unsigned IncRef(std::istream &is); unsigned DecRef(); #ifdef _USE_BOOST @@ -348,11 +351,15 @@ MVS_API void EstimatePointColors(const ImageArr& images, PointCloud& pointcloud) MVS_API void EstimatePointNormals(const ImageArr& images, PointCloud& pointcloud, int numNeighbors=16/*K-nearest neighbors*/); MVS_API bool SaveDepthMap(const String& fileName, const DepthMap& depthMap); +MVS_API bool SaveDepthMap(std::ostream &os, const DepthMap &depthMap); MVS_API bool LoadDepthMap(const String& fileName, DepthMap& depthMap); +MVS_API bool LoadDepthMap(std::istream &is, DepthMap &depthMap); MVS_API bool SaveNormalMap(const String& fileName, const NormalMap& normalMap); MVS_API bool LoadNormalMap(const String& fileName, NormalMap& normalMap); MVS_API bool SaveConfidenceMap(const String& fileName, const ConfidenceMap& confMap); +MVS_API bool SaveConfidenceMap(std::ostream &os, const ConfidenceMap & confMap); MVS_API bool LoadConfidenceMap(const String& fileName, ConfidenceMap& confMap); +MVS_API bool LoadConfidenceMap(std::istream &is, ConfidenceMap& confMap); MVS_API bool ExportDepthMap(const String& fileName, const DepthMap& depthMap, Depth minDepth=FLT_MAX, Depth maxDepth=0); MVS_API bool ExportNormalMap(const String& fileName, const NormalMap& normalMap); diff --git a/libs/MVS/Interface.h b/libs/MVS/Interface.h index 5fa6dc4b8..f42ad7844 100644 --- a/libs/MVS/Interface.h +++ b/libs/MVS/Interface.h @@ -133,6 +133,19 @@ bool SerializeLoad(_Tp& obj, const std::string& fileName) { return true; } +// Main exporter & importer +template +bool SerializeSave(const _Tp& obj, std::ostream &stream) { + ARCHIVE::ArchiveSave serializer(stream); + serializer & obj; + return true; +} +template +bool SerializeLoad(_Tp& obj, std::istream &stream) { + ARCHIVE::ArchiveLoad serializer(stream); + serializer & obj; + return true; +} #define ARCHIVE_DEFINE_TYPE(TYPE) \ template<> \ diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index fdc2d70dc..e57440ac2 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -63,7 +63,25 @@ bool Scene::LoadInterface(const String & fileName) if (!ARCHIVE::SerializeLoad(obj, fileName)) return false; + return LoadInterface(obj); +} // LoadInterface + +bool Scene::LoadInterface(std::istream &is, const uint32_t &width, const uint32_t &height) +{ + TD_TIMER_STARTD(); + Interface obj; + + // serialize in the current state + if (!ARCHIVE::SerializeLoad(obj, is)) + return false; + + return LoadInterface(obj, width, height); +} // LoadInterface + +bool Scene::LoadInterface(Interface &obj, const uint32_t &width, const uint32_t &height) +{ // import platforms and cameras + TD_TIMER_STARTD(); ASSERT(!obj.platforms.empty()); platforms.Reserve((uint32_t)obj.platforms.size()); for (Interface::PlatformArr::const_iterator itPlatform=obj.platforms.begin(); itPlatform!=obj.platforms.end(); ++itPlatform) { diff --git a/libs/MVS/Scene.h b/libs/MVS/Scene.h index e250e7f6f..c9ffae116 100644 --- a/libs/MVS/Scene.h +++ b/libs/MVS/Scene.h @@ -46,6 +46,8 @@ namespace MVS { +struct Interface; + class MVS_API Scene { public: @@ -64,6 +66,7 @@ class MVS_API Scene void Release(); bool LoadInterface(const String& fileName); + bool LoadInterface(std::istream &is, const uint32_t &width = 0, const uint32_t &height = 0); bool SaveInterface(const String& fileName) const; bool Load(const String& fileName); @@ -104,6 +107,8 @@ class MVS_API Scene ar & mesh; } #endif +private: + bool LoadInterface(Interface &obj, const uint32_t &width = 0, const uint32_t &height = 0); }; /*----------------------------------------------------------------*/ From 6930ca4542be444b1879b82edd3d1c15197ae1ba Mon Sep 17 00:00:00 2001 From: CSD-RD Date: Thu, 22 Nov 2018 15:27:10 +0100 Subject: [PATCH 3/6] Added support to access already opened streams instead of reloading images from files. --- libs/MVS/Image.cpp | 52 +++++++++++++++++++++ libs/MVS/Image.h | 2 + libs/MVS/Scene.cpp | 2 +- libs/MVS/SceneDensify.cpp | 98 ++++++++++++++++++++++++++++----------- 4 files changed, 126 insertions(+), 28 deletions(-) diff --git a/libs/MVS/Image.cpp b/libs/MVS/Image.cpp index e83a75d67..ca88ebc93 100644 --- a/libs/MVS/Image.cpp +++ b/libs/MVS/Image.cpp @@ -127,6 +127,52 @@ bool Image::ReloadImage(unsigned nMaxResolution, bool bLoadPixels) } // ReloadImage /*----------------------------------------------------------------*/ +// reads again the image data from the input vector +bool Image::ReloadImage(const uint32_t &_width, const uint32_t &_height, const uint32_t &depth, unsigned nMaxResolution, bool bLoadPixels, const std::vector &imageData) +{ + width = _width; + height = _height; + + if (bLoadPixels) + { + image.create(height, width); + + uint8_t *pData = image.data; + + const uint8_t *buffer = imageData.data(); + + switch (depth) + { + case 1: + { + if (!CImage::FilterFormat(pData, PF_R8G8B8, 3, buffer, PF_GRAY8, depth, imageData.size())) + { + return false; + } + } + break; + case 3: + memcpy(pData, buffer, imageData.size()); + break; + case 4: + { + if (!CImage::FilterFormat(pData, PF_R8G8B8, 3, buffer, PF_R8G8B8A8, depth, imageData.size())) + { + return false; + } + } + break; + default: + return false; + } + } + + scale = ResizeImage(nMaxResolution); + + return true; +} // ReloadImage + /*----------------------------------------------------------------*/ + // free the image data void Image::ReleaseImage() { @@ -184,6 +230,12 @@ unsigned Image::RecomputeMaxResolution(unsigned& level, unsigned minImageSize) c } // RecomputeMaxResolution /*----------------------------------------------------------------*/ +// compute image scale for a given max and min resolution, using the current image file data +unsigned Image::RecomputeMaxResolution(const uint32_t &width, const uint32_t &height, unsigned& level, unsigned minImageSize) const +{ + return Image8U3::computeMaxResolution(MAXF(width, height), level, minImageSize); +} // RecomputeMaxResolution + /*----------------------------------------------------------------*/ // compute the camera extrinsics from the platform pose and the relative camera pose to the platform Camera Image::GetCamera(const PlatformArr& platforms, const Image8U::Size& resolution) const diff --git a/libs/MVS/Image.h b/libs/MVS/Image.h index a0081be1a..0cadf5c87 100644 --- a/libs/MVS/Image.h +++ b/libs/MVS/Image.h @@ -100,10 +100,12 @@ class MVS_API Image static bool ReadImage(IMAGEPTR pImage, Image8U3& image); bool LoadImage(const String& fileName, unsigned nMaxResolution=0); bool ReloadImage(unsigned nMaxResolution=0, bool bLoadPixels=true); + bool ReloadImage(const uint32_t &width, const uint32_t &height, const uint32_t &depth, unsigned nMaxResolution=0, bool bLoadPixels=false, const std::vector &imageData=std::vector()); void ReleaseImage(); float ResizeImage(unsigned nMaxResolution=0); unsigned ComputeMaxResolution(unsigned& level, unsigned minImageSize) const; unsigned RecomputeMaxResolution(unsigned& level, unsigned minImageSize) const; + unsigned RecomputeMaxResolution(const uint32_t &width, const uint32_t &height, unsigned& level, unsigned minImageSize) const; Camera GetCamera(const PlatformArr& platforms, const Image8U::Size& resolution) const; void UpdateCamera(const PlatformArr& platforms); diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index e57440ac2..31f965c84 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -129,7 +129,7 @@ bool Scene::LoadInterface(Interface &obj, const uint32_t &width, const uint32_t imageData.cameraID = image.cameraID; #if 1 // load image - if (!imageData.ReloadImage(0, false)) + if ((width == 0 && height == 0) ? !imageData.ReloadImage(0, false) : !imageData.ReloadImage(width, height, 0, 0, false)) return false; imageData.UpdateCamera(platforms); #else diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index ef2301f72..c38efdf4c 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -138,8 +138,8 @@ class DepthMapsData bool RemoveSmallSegments(DepthData& depthData); bool GapInterpolation(DepthData& depthData); - bool FilterDepthMap(DepthData& depthData, const IndexArr& idxNeighbors, bool bAdjust=true); - void FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal); + bool FilterDepthMap(DepthData& depthData, const IndexArr& idxNeighbors, bool bAdjust=true, std::map *streamFiltered=NULL, std::map *streamConfidence=NULL); + void FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal, std::map *streamDepthMap=NULL); protected: static void* STCALL ScoreDepthMapTmp(void*); @@ -1029,7 +1029,7 @@ bool DepthMapsData::GapInterpolation(DepthData& depthData) // filter depth-map, one pixel at a time, using confidence based fusion or neighbor pixels -bool DepthMapsData::FilterDepthMap(DepthData& depthDataRef, const IndexArr& idxNeighbors, bool bAdjust) +bool DepthMapsData::FilterDepthMap(DepthData& depthDataRef, const IndexArr& idxNeighbors, bool bAdjust, std::map *streamFilteredMap, std::map *streamConfidenceMap) { TD_TIMER_STARTD(); @@ -1271,9 +1271,27 @@ bool DepthMapsData::FilterDepthMap(DepthData& depthDataRef, const IndexArr& idxN } } } - if (!SaveDepthMap(ComposeDepthFilePath(imageRef.pImageData-scene.images.Begin(), "filtered.dmap"), newDepthMap) || - !SaveConfidenceMap(ComposeDepthFilePath(imageRef.pImageData-scene.images.Begin(), "filtered.cmap"), newConfMap)) - return false; + + if(streamFilteredMap!=NULL) + { + if (!SaveDepthMap((*streamFilteredMap)[imageRef.pImageData - scene.images.Begin()], newDepthMap)) + return false; + } + else + { + if (!SaveDepthMap(ComposeDepthFilePath(imageRef.pImageData-scene.images.Begin(), "filtered.dmap"), newDepthMap)) + return false; + } + if(streamConfidenceMap!=NULL) + { + if (!SaveConfidenceMap((*streamConfidenceMap)[imageRef.pImageData - scene.images.Begin()], newConfMap)) + return false; + } + else + { + if (!SaveConfidenceMap(ComposeDepthFilePath(imageRef.pImageData-scene.images.Begin(), "filtered.cmap"), newConfMap)) + return false; + } #if TD_VERBOSE != TD_VERBOSE_OFF DEBUG("Depth map %3u filtered using %u other images: %u/%u depths discarded (%s)", imageRef.pImageData-scene.images.Begin(), N, nDiscarded, nProcessed, TD_TIMER_GET_FMT().c_str()); @@ -1300,7 +1318,7 @@ struct Proj { }; typedef SEACAVE::cList ProjArr; typedef SEACAVE::cList ProjsArr; -void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) +void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal, std::map *streamDepthMap) { TD_TIMER_STARTD(); @@ -1311,8 +1329,16 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal) DepthData& depthData = arrDepthData[i]; if (!depthData.IsValid()) continue; - if (depthData.IncRef(ComposeDepthFilePath(i, "dmap")) == 0) - return; + if(streamDepthMap!=NULL) + { + if (depthData.IncRef((*streamDepthMap)[i]) == 0) + return; + } + else + { + if (depthData.IncRef(ComposeDepthFilePath(i, "dmap")) == 0) + return; + } ASSERT(!depthData.IsEmpty()); IndexScore& connection = connections.AddEmpty(); connection.idx = i; @@ -1506,12 +1532,23 @@ struct DenseDepthMapData { } }; +struct DepthMapDataStream +{ + DepthMapDataStream(DenseDepthMapData *inData = nullptr) + : data(inData) + {} + DenseDepthMapData *data; + std::map streamDepthMap; + std::map streamFilteredMap; + std::map streamConfidenceMap; +}; + static void* DenseReconstructionEstimateTmp(void*); static void* DenseReconstructionFilterTmp(void*); bool Scene::DenseReconstruction(std::vector& theMasks, std::vector> &imageVector, const uint32_t &width, const uint32_t &height, const uint32_t &depth) { DenseDepthMapData data(*this); - + DepthMapDataStream d(&data); { // maps global view indices to our list of views to be processed IndexArr imagesMap; @@ -1645,12 +1682,12 @@ bool Scene::DenseReconstruction(std::vector& theMasks, std::vector threads(2); FOREACHPTR(pThread, threads) - pThread->start(DenseReconstructionEstimateTmp, (void*)&data); + pThread->start(DenseReconstructionEstimateTmp, (void*)&d); FOREACHPTR(pThread, threads) pThread->join(); } else { // single-thread execution - DenseReconstructionEstimate((void*)&data); + DenseReconstructionEstimate((void*)&d); } GET_LOGCONSOLE().Play(); if (!data.events.IsEmpty()) @@ -1671,12 +1708,12 @@ bool Scene::DenseReconstruction(std::vector& theMasks, std::vector threads(MINF(nMaxThreads, (unsigned)data.images.GetSize())); FOREACHPTR(pThread, threads) - pThread->start(DenseReconstructionFilterTmp, (void*)&data); + pThread->start(DenseReconstructionFilterTmp, (void*)&d); FOREACHPTR(pThread, threads) pThread->join(); } else { // single-thread execution - DenseReconstructionFilter((void*)&data); + DenseReconstructionFilter((void*)&d); } GET_LOGCONSOLE().Play(); if (!data.events.IsEmpty()) @@ -1686,7 +1723,7 @@ bool Scene::DenseReconstruction(std::vector& theMasks, std::vector 2) { // print number of points with 3+ views @@ -1720,7 +1757,7 @@ bool Scene::DenseReconstruction(std::vector& theMasks, std::vectordata; dataThreads.scene.DenseReconstructionEstimate(arg); return NULL; } @@ -1728,7 +1765,9 @@ void* DenseReconstructionEstimateTmp(void* arg) { // initialize the dense reconstruction with the sparse point cloud void Scene::DenseReconstructionEstimate(void* pData) { - DenseDepthMapData& data = *((DenseDepthMapData*)pData); + DenseDepthMapData& data = *((DepthMapDataStream*)pData)->data; + std::map &streamDepthMap = ((DepthMapDataStream*)pData)->streamDepthMap; + while (true) { CAutoPtr evt(data.events.GetEvent()); switch (evt->GetID()) { @@ -1750,7 +1789,8 @@ void Scene::DenseReconstructionEstimate(void* pData) break; } // try to load already compute depth-map for this image - if (depthData.Load(ComposeDepthFilePath(idx, "dmap"))) { + std::map::iterator pos = streamDepthMap.find(idx); + if (pos != streamDepthMap.end()) { if (OPTDENSE::nOptimize & (OPTDENSE::OPTIMIZE)) { // optimize depth-map data.events.AddEventFirst(new EVTOptimizeDepthMap(evtImage.idxImage)); @@ -1825,7 +1865,7 @@ void Scene::DenseReconstructionEstimate(void* pData) } #endif // save compute depth-map for this image - depthData.Save(ComposeDepthFilePath(idx, "dmap")); + depthData.Save(streamDepthMap[idx]); depthData.ReleaseImages(); depthData.Release(); data.progress->operator++(); @@ -1842,7 +1882,7 @@ void Scene::DenseReconstructionEstimate(void* pData) /*----------------------------------------------------------------*/ void* DenseReconstructionFilterTmp(void* arg) { - DenseDepthMapData& dataThreads = *((DenseDepthMapData*)arg); + const DenseDepthMapData& dataThreads = *((DepthMapDataStream*)arg)->data; dataThreads.scene.DenseReconstructionFilter(arg); return NULL; } @@ -1850,7 +1890,11 @@ void* DenseReconstructionFilterTmp(void* arg) { // filter estimated depth-maps void Scene::DenseReconstructionFilter(void* pData) { - DenseDepthMapData& data = *((DenseDepthMapData*)pData); + DenseDepthMapData& data = *((DepthMapDataStream*)pData)->data; + std::map &streamDepthMap = ((DepthMapDataStream*)pData)->streamDepthMap; + std::map &streamFilteredMap = ((DepthMapDataStream*)pData)->streamFilteredMap; + std::map &streamConfidenceMap = ((DepthMapDataStream*)pData)->streamConfidenceMap; + CAutoPtr evt; while ((evt=data.events.GetEvent(0)) != NULL) { switch (evt->GetID()) { @@ -1863,7 +1907,7 @@ void Scene::DenseReconstructionFilter(void* pData) break; } // make sure all depth-maps are loaded - depthData.IncRef(ComposeDepthFilePath(idx, "dmap")); + depthData.IncRef(streamDepthMap[idx]); const unsigned numMaxNeighbors(8); IndexArr idxNeighbors(0, depthData.neighbors.GetSize()); FOREACH(n, depthData.neighbors) { @@ -1871,7 +1915,7 @@ void Scene::DenseReconstructionFilter(void* pData) DepthData& depthDataPair = data.detphMaps.arrDepthData[idxView]; if (!depthDataPair.IsValid()) continue; - if (depthDataPair.IncRef(ComposeDepthFilePath(idxView, "dmap")) == 0) { + if (depthDataPair.IncRef(streamDepthMap[idxView]) == 0) { // signal error and terminate data.events.AddEventFirst(new EVTFail); return; @@ -1881,7 +1925,7 @@ void Scene::DenseReconstructionFilter(void* pData) break; } // filter the depth-map for this image - if (data.detphMaps.FilterDepthMap(depthData, idxNeighbors, OPTDENSE::bFilterAdjust)) { + if (data.detphMaps.FilterDepthMap(depthData, idxNeighbors, OPTDENSE::bFilterAdjust, &streamFilteredMap, &streamConfidenceMap)) { // load the filtered maps after all depth-maps were filtered data.events.AddEvent(new EVTAdjustDepthMap(evtImage.idxImage)); } @@ -1902,9 +1946,9 @@ void Scene::DenseReconstructionFilter(void* pData) ASSERT(depthData.IsValid()); data.sem.Wait(); // load filtered maps - if (depthData.IncRef(ComposeDepthFilePath(idx, "dmap")) == 0 || - !LoadDepthMap(ComposeDepthFilePath(idx, "filtered.dmap"), depthData.depthMap) || - !LoadConfidenceMap(ComposeDepthFilePath(idx, "filtered.cmap"), depthData.confMap)) + if (depthData.IncRef(streamDepthMap[idx]) == 0 || + !LoadDepthMap(streamFilteredMap[idx], depthData.depthMap) || + !LoadConfidenceMap(streamConfidenceMap[idx], depthData.confMap)) { // signal error and terminate data.events.AddEventFirst(new EVTFail); From fd5049035091a8fc1fdcd9ebe72215344424c6ea Mon Sep 17 00:00:00 2001 From: CSD-RD Date: Thu, 22 Nov 2018 15:27:43 +0100 Subject: [PATCH 4/6] Fixed bug when points are infinite in FuseDepthMaps. --- libs/MVS/SceneDensify.cpp | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index c38efdf4c..2e1be68b8 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -1457,9 +1457,26 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateNormal, } else { // this point is valid, store it point = X*(REAL(1)/confidence); - // invalidate all neighbor depths that do not agree with it - for (Depth* pDepth: invalidDepths) - *pDepth = 0; + if (ISFINITE(point)) + { + // invalidate all neighbor depths that do not agree with it + for (Depth* pDepth : invalidDepths) + *pDepth = 0; + } + else + { + // remove point + FOREACH(v, views) { + const uint32_t idxImageB(views[v]); + const ImageRef x(pointProjs[v].GetCoord()); + ASSERT(arrDepthIdx[idxImageB].isInside(x) && arrDepthIdx[idxImageB](x).idx != NO_ID); + arrDepthIdx[idxImageB](x).idx = NO_ID; + } + projs.RemoveLast(); + pointcloud.pointWeights.RemoveLast(); + pointcloud.pointViews.RemoveLast(); + pointcloud.points.RemoveLast(); + } } } } From 1616a910eec0f317b36f3d59095dd7f4c8ac1e73 Mon Sep 17 00:00:00 2001 From: Luis Kabongo Date: Thu, 29 Nov 2018 10:36:40 +0100 Subject: [PATCH 5/6] Fixed missing implementation for masks support in DenseReconstruction. --- libs/MVS/SceneDensify.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 52e3a8215..cd4274339 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -711,10 +711,10 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) // init integral images and index to image-ref map for the reference data Image64F imageSum0; cv::integral(image.image, imageSum0, CV_64F); - if (prevDepthMapSize != size) { + if (!depthData.mask.empty() || prevDepthMapSize != size) { prevDepthMapSize = size; - BitMatrix mask; - DepthEstimator::MapMatrix2ZigzagIdx(size, coords, mask, MAXF(64,(int)nMaxThreads*8)); + //BitMatrix mask; + DepthEstimator::MapMatrix2ZigzagIdx(size, coords, depthData.mask, MAXF(64,(int)nMaxThreads*8)); } // init threads From af9bab497d4d1a20c2cfea0ea7fd2fa920aa636f Mon Sep 17 00:00:00 2001 From: CSD-RD Date: Thu, 29 Nov 2018 10:36:40 +0100 Subject: [PATCH 6/6] Fixed missing implementation for masks support in DenseReconstruction. --- libs/MVS/SceneDensify.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 52e3a8215..cd4274339 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -711,10 +711,10 @@ bool DepthMapsData::EstimateDepthMap(IIndex idxImage) // init integral images and index to image-ref map for the reference data Image64F imageSum0; cv::integral(image.image, imageSum0, CV_64F); - if (prevDepthMapSize != size) { + if (!depthData.mask.empty() || prevDepthMapSize != size) { prevDepthMapSize = size; - BitMatrix mask; - DepthEstimator::MapMatrix2ZigzagIdx(size, coords, mask, MAXF(64,(int)nMaxThreads*8)); + //BitMatrix mask; + DepthEstimator::MapMatrix2ZigzagIdx(size, coords, depthData.mask, MAXF(64,(int)nMaxThreads*8)); } // init threads