From afb59d6bd3c4b61cd1821d21900525a976d6442d Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Wed, 4 Oct 2023 19:55:04 +0545 Subject: [PATCH 1/2] Implements logic to support negative url --- backend/core/utils.py | 34 ++++++++++++------- backend/core/views.py | 21 ++++++++++++ .../TrainingDS/DatasetNew/DatasetNew.js | 10 ++---- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/backend/core/utils.py b/backend/core/utils.py index b5d7dd94..6477efd7 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -122,18 +122,13 @@ def is_dir_empty(directory_path): def download_image(url, base_path, source_name): response = requests.get(url) - # Check if the URL request was successful - if response.status_code != 200: - raise ValueError( - f"Failed to download image from {url}. Status code: {response.status_code}" - ) - image = response.content - url = re.sub(r"\.(png|jpeg)$", "", url) - url_splitted_list = url.split("/") + pattern = r"/(\d+)/(\d+)/(\d+)(?:\.\w+)?" + match = re.search(pattern, url) + # filename = z-x-y + filename = f"{base_path}/{source_name}-{match.group(2)}-{match.group(3)}-{match.group(1)}.png" - filename = f"{base_path}/{source_name}-{url_splitted_list[-2]}-{url_splitted_list[-1]}-{url_splitted_list[-3]}.png" with open(filename, "wb") as f: f.write(image) @@ -178,15 +173,30 @@ def download_imagery(start: list, end: list, zm_level, base_path, source="maxar" # add multiple logic on supported sources here else: # source should be url as string , like this : https://tiles.openaerialmap.org/62dbd947d8499800053796ec/0/62dbd947d8499800053796ed/{z}/{x}/{y} - download_url = source.format( - x=download_path[0], y=download_path[1], z=zm_level - ) + if "{-y}" in source: + ## negative TMS + source_value = source.replace("{-y}", "{y}") + # conversion from normal tms + y_value = int((2**zm_level) - download_path[1] - 1) + + else: + # If it doesn't, use the positive y-coordinate + y_value = download_path[1] + source_value = source + download_url = source_value.format( + x=download_path[0], y=y_value, z=zm_level) download_urls.append(download_url) start_y = start_y - 1 # decrease the y start_x = start_x + 1 # increase the x + # Use the ThreadPoolExecutor to download the images in parallel + + # with concurrent.futures.ThreadPoolExecutor() as executor: + # for url in download_urls: + # executor.submit(download_image, url, base_path, source_name) + with concurrent.futures.ThreadPoolExecutor() as executor: futures = [ executor.submit(download_image, url, base_path, source_name) diff --git a/backend/core/views.py b/backend/core/views.py index ecb25da0..c6321083 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -245,6 +245,27 @@ class LabelViewSet(viewsets.ModelViewSet): ) filterset_fields = ["aoi", "aoi__dataset"] + def create(self, request, *args, **kwargs): + aoi_id = request.data.get("aoi") + geom = request.data.get("geom") + + # Check if a label with the same AOI and geometry exists + existing_label = Label.objects.filter(aoi=aoi_id, geom=geom).first() + + if existing_label: + # If it exists, update the existing label + serializer = LabelSerializer(existing_label, data=request.data) + else: + # If it doesn't exist, create a new label + serializer = LabelSerializer(data=request.data) + + if serializer.is_valid(): + serializer.save() + return Response( + serializer.data, status=status.HTTP_200_OK + ) # 200 for update, 201 for create + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + class RawdataApiFeedbackView(APIView): authentication_classes = [OsmAuthentication] diff --git a/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js b/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js index 91957014..c80913ec 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js @@ -91,17 +91,11 @@ const DatasetNew = (props) => { fullWidth onChange={(e) => { let trimmedValue = e.target.value.trim(); - // let regUrl = /^(https?|chrome):\/\/[^\s$.?#].[^\s]*$/; - // let endsWithPng = - // trimmedValue.endsWith(".png") || - // trimmedValue.endsWith(".jpeg"); - // if (endsWithPng) { - // trimmedValue = trimmedValue.slice(0, -4); - // } let hasZXY = trimmedValue.includes("{z}/{x}/{y}"); + let hasZXYNegative = trimmedValue.includes("{z}/{x}/{-y}"); let isValid = regUrl.test(trimmedValue) && - hasZXY && + (hasZXY || hasZXYNegative) && trimmedValue !== "" && trimmedValue != null; setError(!isValid); From 46801badeeb0febf9ca8ff57ee50716354d7e14a Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Thu, 5 Oct 2023 16:43:59 +0545 Subject: [PATCH 2/2] Updated the installation doc with local tiles --- docs/Docker-installation.md | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/Docker-installation.md b/docs/Docker-installation.md index ece44360..96913b15 100644 --- a/docs/Docker-installation.md +++ b/docs/Docker-installation.md @@ -130,3 +130,57 @@ Docker Compose is created with redis , worker , postgis database , api and fron ``` Frontend will be available on 5000 port , Backend will be on 8000 , Flower will be on 5500 + +10. Want to run your local tiles ? + + You can use [titler](https://github.com/developmentseed/titiler) , [gdals2tiles](https://gdal.org/programs/gdal2tiles.html) or nginx to run your own TMS server and add following to docker compose in order to access your localhost through docker containers . Add those to API and Worker . Make sure you update the .env variable accordingly + + ``` + network_mode: "host" + ``` + Example docker compose : + + ``` + backend-api: + build: + context: ./backend + dockerfile: Dockerfile_CPU + container_name: api + command: python manage.py runserver 0.0.0.0:8000 + + ports: + - 8000:8000 + volumes: + - ./backend:/app + - ${RAMP_HOME}:/RAMP_HOME + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + depends_on: + - redis + - postgres + network_mode: "host" + + backend-worker: + build: + context: ./backend + dockerfile: Dockerfile_CPU + container_name: worker + command: celery -A aiproject worker --loglevel=INFO --concurrency=1 + + volumes: + - ./backend:/app + - ${RAMP_HOME}:/RAMP_HOME + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + depends_on: + - backend-api + - redis + - postgres + network_mode: "host" + ``` + + Example .env after host change : + + ``` + DATABASE_URL=postgis://postgres:admin@localhost:5434/ai + CELERY_BROKER_URL="redis://localhost:6379/0" + CELERY_RESULT_BACKEND="redis://localhost:6379/0" + ``` \ No newline at end of file