From 9e0a3510dd92ca0a5b2053f51bcef03ed372ffb5 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Aug 2023 20:42:18 +0545 Subject: [PATCH 1/2] Implemented API for apply/feedback --- backend/core/serializers.py | 28 +++++++++-- backend/core/tasks.py | 94 +++++++++++++------------------------ backend/core/views.py | 22 +++++---- docker-compose.yml | 4 +- 4 files changed, 70 insertions(+), 78 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 223413e5..9d655aca 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -120,12 +120,18 @@ class Meta: # read_only_fields = ("created_at", "osm_id") -class LabelFileSerializer( - GeoFeatureModelSerializer -): # serializers are used to translate models objects to api +class LabelFileSerializer(GeoFeatureModelSerializer): class Meta: model = Label - geo_field = "geom" # this will be used as geometry in order to create geojson api , geofeatureserializer will let you create api in geojson + geo_field = "geom" + # auto_bbox = True + fields = ("osm_id",) + + +class FeedbackLabelFileSerializer(GeoFeatureModelSerializer): + class Meta: + model = FeedbackLabel + geo_field = "geom" # auto_bbox = True fields = ("osm_id",) @@ -158,7 +164,19 @@ class FeedbackParamSerializer(serializers.Serializer): training_id = serializers.IntegerField(required=True) epochs = serializers.IntegerField(required=False) batch_size = serializers.IntegerField(required=False) - freeze_layers = serializers.BooleanField(required=False) + zoom_level = serializers.ListField(required=False) + + def validate(self, data): + """ + Check supplied data + """ + if "zoom_level" in data: + for i in data["zoom_level"]: + if int(i) < 19 or int(i) > 21: + raise serializers.ValidationError( + "Zoom level Supported between 19-21" + ) + return data class PredictionParamSerializer(serializers.Serializer): diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 8f020b81..4cd34642 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -10,8 +10,12 @@ import ramp.utils import tensorflow as tf from celery import shared_task -from core.models import AOI, Feedback, Label, Training -from core.serializers import FeedbackFileSerializer, LabelFileSerializer +from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training +from core.serializers import ( + FeedbackFileSerializer, + FeedbackLabelFileSerializer, + LabelFileSerializer, +) from core.utils import bbox, download_imagery, get_start_end_download_coords from django.conf import settings from django.contrib.gis.db.models.aggregates import Extent @@ -65,34 +69,32 @@ def train_model( shutil.rmtree(training_input_image_source) os.makedirs(training_input_image_source) if feedback: - feedback_objects = Feedback.objects.filter( - training__id=feedback, - validated=True, - ) - bbox_feedback = feedback_objects.aggregate(Extent("geom"))[ - "geom__extent" - ] - bbox_geo = GEOSGeometry( - f"POLYGON(({bbox_feedback[0]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[1]}))" - ) - print(training_input_image_source) - print(bbox_feedback) - with open( - os.path.join(training_input_image_source, "labels_bbox.geojson"), - "w", - encoding="utf-8", - ) as f: - f.write(bbox_geo.geojson) + try: + aois = FeedbackAOI.objects.filter(training=feedback) + except FeedbackAOI.DoesNotExist: + raise ValueError( + f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first", + ) + + else: + try: + aois = AOI.objects.filter(dataset=dataset_id) + except AOI.DoesNotExist: + raise ValueError( + f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", + ) + for obj in aois: + bbox_coords = bbox(obj.geom.coords[0]) for z in zoom_level: zm_level = z print( f"""Running Download process for - feedback {training_id} - dataset : {dataset_id} , zoom : {zm_level}""" + aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}""" ) try: tile_size = DEFAULT_TILE_SIZE # by default - bbox_coords = list(bbox_feedback) + start, end = get_start_end_download_coords( bbox_coords, zm_level, tile_size ) @@ -107,49 +109,17 @@ def train_model( except Exception as ex: raise ex - else: - try: - aois = AOI.objects.filter(dataset=dataset_id) - except AOI.DoesNotExist: - raise ValueError( - f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", - ) - - for obj in aois: - bbox_coords = bbox(obj.geom.coords[0]) - for z in zoom_level: - zm_level = z - print( - f"""Running Download process for - aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}""" - ) - try: - tile_size = DEFAULT_TILE_SIZE # by default - - start, end = get_start_end_download_coords( - bbox_coords, zm_level, tile_size - ) - # start downloading - download_imagery( - start, - end, - zm_level, - base_path=training_input_image_source, - source=source_imagery, - ) - except Exception as ex: - raise ex - ## -----------LABEL GENERATOR--------- - logging.debug("Label Generator started") + logging.info("Label Generator started") + aoi_list = [r.id for r in aois] + logging.info(aoi_list) + if feedback: - feedback_objects = Feedback.objects.filter( - training__id=feedback, - validated=True, - ) - serialized_field = FeedbackFileSerializer(feedback_objects, many=True) + label = FeedbackLabel.objects.filter(feedback_aoi__in=aoi_list) + logging.info(label) + + serialized_field = FeedbackLabelFileSerializer(label, many=True) else: - aoi_list = [r.id for r in aois] label = Label.objects.filter(aoi__in=aoi_list) serialized_field = LabelFileSerializer(label, many=True) diff --git a/backend/core/views.py b/backend/core/views.py index 765f1485..66b3fc63 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -244,8 +244,8 @@ class LabelViewSet(viewsets.ModelViewSet): class RawdataApiFeedbackView(APIView): - # authentication_classes = [OsmAuthentication] - # permission_classes = [IsOsmAuthenticated] + authentication_classes = [OsmAuthentication] + permission_classes = [IsOsmAuthenticated] def post(self, request, feedbackaoi_id, *args, **kwargs): """Downloads available osm data as labels within given feedback aoi @@ -408,6 +408,15 @@ def run_task_status(request, run_id: str): class FeedbackView(APIView): + """Applies Associated feedback to Training Published Checkpoint + + Args: + APIView (_type_): _description_ + + Returns: + _type_: _description_ + """ + authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] @@ -421,12 +430,7 @@ def post(self, request, *args, **kwargs): training_id = deserialized_data["training_id"] training_instance = Training.objects.get(id=training_id) - unique_zoom_levels = ( - Feedback.objects.filter(training__id=training_id, validated=True) - .values("zoom_level") - .distinct() - ) - zoom_level = [z["zoom_level"] for z in unique_zoom_levels] + zoom_level = deserialized_data.get("zoom_level", [19, 20]) epochs = deserialized_data.get("epochs", 20) batch_size = deserialized_data.get("batch_size", 8) instance = Training.objects.create( @@ -448,7 +452,7 @@ def post(self, request, *args, **kwargs): zoom_level=instance.zoom_level, source_imagery=instance.source_imagery, feedback=training_id, - freeze_layers=instance.freeze_layers, + freeze_layers=True, # True by default for feedback ) if not instance.source_imagery: instance.source_imagery = instance.model.dataset.source_imagery diff --git a/docker-compose.yml b/docker-compose.yml index 80dfc78e..263647eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: volumes: - ./backend:/app - ${RAMP_HOME}:/RAMP_HOME - # - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE depends_on: - redis - postgres @@ -55,7 +55,7 @@ services: volumes: - ./backend:/app - ${RAMP_HOME}:/RAMP_HOME - # - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE depends_on: - backend-api - redis From 105d7dde53e97ca9361f65e203fd250e0a6fe94b Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Aug 2023 20:54:55 +0545 Subject: [PATCH 2/2] Added feedback aoi gpx and changed url for feedback submit --- backend/core/urls.py | 6 +++++- backend/core/utils.py | 29 +++++++++++++++++++++++++++++ backend/core/views.py | 30 ++++++++++++++---------------- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/backend/core/urls.py b/backend/core/urls.py index d1466c2a..cde78096 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -11,6 +11,7 @@ FeedbackLabelViewset, FeedbackView, FeedbackViewset, + GenerateFeedbackAOIGpxView, GenerateGpxView, LabelViewSet, ModelViewSet, @@ -49,10 +50,13 @@ path("training/status//", run_task_status), path("training/publish//", publish_training), path("prediction/", PredictionView.as_view()), - path("apply/feedback/", FeedbackView.as_view()), + path("feedback/training/submit/", FeedbackView.as_view()), path("status/", APIStatus.as_view()), path("geojson2osm/", geojson2osmconverter, name="geojson2osmconverter"), path("aoi/gpx//", GenerateGpxView.as_view()), + path( + "feedback-aoi/gpx//", GenerateFeedbackAOIGpxView.as_view() + ), path("workspace/", TrainingWorkspaceView.as_view()), path( "workspace/download//", TrainingWorkspaceDownloadView.as_view() diff --git a/backend/core/utils.py b/backend/core/utils.py index 58dcd259..4d6567e9 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -2,12 +2,14 @@ import json import math import os +from datetime import datetime from uuid import uuid4 from xml.dom import ValidationErr from zipfile import ZipFile import requests from django.conf import settings +from gpxpy.gpx import GPX, GPXTrack, GPXTrackSegment, GPXWaypoint from tqdm import tqdm from .models import AOI, FeedbackAOI, FeedbackLabel, Label @@ -243,6 +245,33 @@ def remove_file(path: str) -> None: os.unlink(path) +def gpx_generator(geom_json): + """Generates GPX for give geojson geometry + + Args: + geom_json (_type_): _description_ + + Returns: + xml: gpx + """ + + gpx = GPX() + gpx_track = GPXTrack() + gpx.tracks.append(gpx_track) + gpx_segment = GPXTrackSegment() + gpx_track.segments.append(gpx_segment) + for point in geom_json["coordinates"][0]: + # Append each point as a GPXWaypoint to the GPXTrackSegment + gpx_segment.points.append(GPXWaypoint(point[1], point[0])) + gpx.creator = "fAIr" + gpx_track.name = "Don't Edit this Boundary" + gpx_track.description = "Map inside this boundary and go back to fAIr UI" + gpx.time = datetime.now() + gpx.link = "https://github.com/hotosm/fAIr" + gpx.link_text = "AI Assisted Mapping - fAIr : HOTOSM" + return gpx.to_xml() + + def process_feature(feature, aoi_id, foreign_key_id, feedback=False): """Multi thread process of features""" properties = feature["properties"] diff --git a/backend/core/views.py b/backend/core/views.py index 66b3fc63..c9c4dd16 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -27,7 +27,6 @@ from django_filters.rest_framework import DjangoFilterBackend from drf_yasg.utils import swagger_auto_schema from geojson2osm import geojson2osm -from gpxpy.gpx import GPX, GPXTrack, GPXTrackSegment, GPXWaypoint from hot_fair_utilities import polygonize, predict, vectorize from login.authentication import OsmAuthentication from login.permissions import IsOsmAuthenticated @@ -66,6 +65,7 @@ download_imagery, get_dir_size, get_start_end_download_coords, + gpx_generator, process_rawdata, request_rawdata, ) @@ -624,21 +624,19 @@ def get(self, request, aoi_id: int): # Convert the polygon field to GPX format geom_json = json.loads(aoi.geom.json) # Create a new GPX object - gpx = GPX() - gpx_track = GPXTrack() - gpx.tracks.append(gpx_track) - gpx_segment = GPXTrackSegment() - gpx_track.segments.append(gpx_segment) - for point in geom_json["coordinates"][0]: - # Append each point as a GPXWaypoint to the GPXTrackSegment - gpx_segment.points.append(GPXWaypoint(point[1], point[0])) - gpx.creator = "fAIr Backend" - gpx_track.name = f"AOI of id {aoi_id} , Don't Edit this Boundary" - gpx_track.description = "This is coming from AI Assisted Mapping - fAIr : HOTOSM , Map inside this boundary and go back to fAIr UI" - gpx.time = datetime.now() - gpx.link = "https://github.com/hotosm/fAIr" - gpx.link_text = "AI Assisted Mapping - fAIr : HOTOSM" - return HttpResponse(gpx.to_xml(), content_type="application/xml") + gpx_xml=gpx_generator(geom_json) + return HttpResponse(gpx_xml, content_type="application/xml") + + +class GenerateFeedbackAOIGpxView(APIView): + def get(self, request, feedback_aoi_id: int): + aoi = get_object_or_404(FeedbackAOI, id=feedback_aoi_id) + # Convert the polygon field to GPX format + geom_json = json.loads(aoi.geom.json) + # Create a new GPX object + gpx_xml=gpx_generator(geom_json) + return HttpResponse(gpx_xml, content_type="application/xml") + class TrainingWorkspaceView(APIView):