From 1f84e0fd16150f49d074c23d86512be2015dd74a Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 2 Oct 2024 22:38:12 +0200 Subject: [PATCH 01/12] feat(filters): filters on model api endpoints , adds pagination as well as centroid of models --- backend/aiproject/settings.py | 12 ++++++++---- backend/core/serializers.py | 23 +++++++++++++++++++++-- backend/core/urls.py | 2 ++ backend/core/views.py | 29 ++++++++++++++++++++++++----- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/backend/aiproject/settings.py b/backend/aiproject/settings.py index 2c4cf2b7..106104eb 100644 --- a/backend/aiproject/settings.py +++ b/backend/aiproject/settings.py @@ -10,12 +10,13 @@ https://docs.aiproject.com/en/3.1/ref/settings/ """ -import os import logging +import os +from socket import gethostbyname, gethostname + import dj_database_url import environ from corsheaders.defaults import default_headers -from socket import gethostbyname, gethostname env = environ.Env() @@ -102,6 +103,7 @@ CORS_ORIGIN_WHITELIST = ALLOWED_ORIGINS CORS_ORIGIN_ALLOW_ALL = env("CORS_ORIGIN_ALLOW_ALL", default=False) +DEFAULT_PAGINATION_SIZE = env("DEFAULT_PAGINATION_SIZE", default=50) REST_FRAMEWORK = { "DEFAULT_SCHEMA_CLASS": "rest_framework.schemas.coreapi.AutoSchema", @@ -110,6 +112,8 @@ "rest_framework.authentication.BasicAuthentication", "login.authentication.OsmAuthentication", ], + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", + "PAGE_SIZE": DEFAULT_PAGINATION_SIZE, } ROOT_URLCONF = "aiproject.urls" @@ -208,7 +212,7 @@ } } # get ramp home and set it to environ -RAMP_HOME = env("RAMP_HOME",default=None) +RAMP_HOME = env("RAMP_HOME", default=None) if RAMP_HOME: os.environ["RAMP_HOME"] = RAMP_HOME @@ -220,4 +224,4 @@ ENABLE_PREDICTION_API = env("ENABLE_PREDICTION_API", default=False) -TEST_RUNNER = 'tests.test_runners.NoDestroyTestRunner' +TEST_RUNNER = "tests.test_runners.NoDestroyTestRunner" diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 7465bcf2..0045408b 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -1,11 +1,10 @@ from django.conf import settings +from login.models import OsmUser from rest_framework import serializers from rest_framework_gis.serializers import ( GeoFeatureModelSerializer, # this will be used if we used to serialize as geojson ) -from login.models import OsmUser - from .models import * # from .tasks import train_model @@ -48,6 +47,26 @@ def create(self, validated_data): return super().create(validated_data) +class ModelCentroidSerializer(serializers.ModelSerializer): + geometry = serializers.SerializerMethodField() + + class Meta: + model = Model + fields = ("id", "name", "geometry") + + def get_geometry(self, obj): + """ + Get the centroid of the AOI linked to the dataset of the given model. + """ + aoi = AOI.objects.filter(dataset=obj.dataset).first() + if aoi and aoi.geom: + return { + "type": "Point", + "coordinates": aoi.geom.centroid.coords, + } + return None + + class AOISerializer( GeoFeatureModelSerializer ): # serializers are used to translate models objects to api diff --git a/backend/core/urls.py b/backend/core/urls.py index 8212eade..758a5976 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -16,6 +16,7 @@ GenerateFeedbackAOIGpxView, GenerateGpxView, LabelViewSet, + ModelCentroidView, ModelViewSet, RawdataApiAOIView, RawdataApiFeedbackView, @@ -51,6 +52,7 @@ "label/feedback/osm/fetch//", RawdataApiFeedbackView.as_view(), ), + path("models/centroid/", ModelCentroidView.as_view(), name="model-centroid"), # path("download//", download_training_data), path("training/status//", run_task_status), path("training/publish//", publish_training), diff --git a/backend/core/views.py b/backend/core/views.py index d0bd9634..054b6a97 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -26,18 +26,18 @@ from django_filters.rest_framework import DjangoFilterBackend from drf_yasg.utils import swagger_auto_schema from geojson2osm import geojson2osm +from login.authentication import OsmAuthentication +from login.permissions import IsOsmAuthenticated from orthogonalizer import othogonalize_poly from osmconflator import conflate_geojson -from rest_framework import decorators, serializers, status, viewsets +from rest_framework import decorators, filters, serializers, status, viewsets from rest_framework.decorators import api_view from rest_framework.exceptions import ValidationError +from rest_framework.generics import ListAPIView from rest_framework.response import Response from rest_framework.views import APIView from rest_framework_gis.filters import InBBoxFilter, TMSTileFilter -from login.authentication import OsmAuthentication -from login.permissions import IsOsmAuthenticated - from .models import ( AOI, ApprovedPredictions, @@ -59,6 +59,7 @@ FeedbackParamSerializer, FeedbackSerializer, LabelSerializer, + ModelCentroidSerializer, ModelSerializer, PredictionParamSerializer, ) @@ -236,8 +237,26 @@ class ModelViewSet( permission_classes = [IsOsmAuthenticated] permission_allowed_methods = ["GET"] queryset = Model.objects.all() + filter_backends = ( + InBBoxFilter, # it will take bbox like this api/v1/model/?in_bbox=-90,29,-89,35 , + DjangoFilterBackend, + filters.SearchFilter, + ) serializer_class = ModelSerializer # connecting serializer - filterset_fields = ["status"] + filterset_fields = ["status", "created_at", "last_modified", "created_by"] + search_fields = ["name"] + + +class ModelCentroidView(ListAPIView): + queryset = Model.objects.all() + serializer_class = ModelCentroidSerializer + filter_backends = ( + # InBBoxFilter, + DjangoFilterBackend, + filters.SearchFilter, + ) + filterset_fields = ["id"] + search_fields = ["name"] class AOIViewSet(viewsets.ModelViewSet): From 125ad491cb9e4c2ab776dc600a48ad99ac08f1c8 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 2 Oct 2024 22:47:10 +0200 Subject: [PATCH 02/12] perf(gtlt): greather than and leess than filter on timestamps --- backend/core/views.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/core/views.py b/backend/core/views.py index 054b6a97..1a84f4b2 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -242,8 +242,13 @@ class ModelViewSet( DjangoFilterBackend, filters.SearchFilter, ) - serializer_class = ModelSerializer # connecting serializer - filterset_fields = ["status", "created_at", "last_modified", "created_by"] + serializer_class = ModelSerializer + filterset_fields = { + "status": ["exact"], + "created_at": ["exact", "gt", "gte", "lt", "lte"], + "last_modified": ["exact", "gt", "gte", "lt", "lte"], + "created_by": ["exact"], + } search_fields = ["name"] From 3945683d586dba440acb9449e596fd445aaa76e6 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 2 Oct 2024 23:04:20 +0200 Subject: [PATCH 03/12] perf(id): adds id filter in model --- backend/core/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/core/views.py b/backend/core/views.py index 1a84f4b2..80465952 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -248,6 +248,7 @@ class ModelViewSet( "created_at": ["exact", "gt", "gte", "lt", "lte"], "last_modified": ["exact", "gt", "gte", "lt", "lte"], "created_by": ["exact"], + "id": ["exact"], } search_fields = ["name"] From 0f0676938c5e39de921203114de1fb6562aa2d6c Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 2 Oct 2024 23:19:24 +0200 Subject: [PATCH 04/12] fix(userinfo): added missing userinfo in models directly instead of userid , adds users endpoint as well --- backend/core/serializers.py | 34 ++++++++++++++++++---------------- backend/core/urls.py | 2 ++ backend/core/views.py | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 0045408b..33d03d76 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -28,9 +28,27 @@ def create(self, validated_data): return super().create(validated_data) +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = OsmUser + fields = [ + "osm_id", + "username", + # "is_superuser", + # "is_active", + # "is_staff", + "date_joined", + # "email", + "img_url", + # "user_permissions", + ] + + class ModelSerializer( serializers.ModelSerializer ): # serializers are used to translate models objects to api + created_by = UserSerializer(read_only=True) + class Meta: model = Model fields = "__all__" # defining all the fields to be included in curd for now , we can restrict few if we want @@ -333,19 +351,3 @@ def validate(self, data): data["area_threshold"] ) return data - - -class UserSerializer(serializers.ModelSerializer): - class Meta: - model = OsmUser - fields = [ - "osm_id", - "username", - "is_superuser", - "is_active", - "is_staff", - "date_joined", - "email", - "img_url", - "user_permissions", - ] diff --git a/backend/core/urls.py b/backend/core/urls.py index 758a5976..b54bc169 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -23,6 +23,7 @@ TrainingViewSet, TrainingWorkspaceDownloadView, TrainingWorkspaceView, + UsersView, download_training_data, geojson2osmconverter, publish_training, @@ -52,6 +53,7 @@ "label/feedback/osm/fetch//", RawdataApiFeedbackView.as_view(), ), + path("users/", UsersView.as_view(), name="user-list-view"), path("models/centroid/", ModelCentroidView.as_view(), name="model-centroid"), # path("download//", download_training_data), path("training/status//", run_task_status), diff --git a/backend/core/views.py b/backend/core/views.py index 80465952..6efcf9d3 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -47,6 +47,7 @@ FeedbackLabel, Label, Model, + OsmUser, Training, ) from .serializers import ( @@ -62,6 +63,7 @@ ModelCentroidSerializer, ModelSerializer, PredictionParamSerializer, + UserSerializer, ) from .tasks import train_model from .utils import get_dir_size, gpx_generator, process_rawdata, request_rawdata @@ -265,6 +267,20 @@ class ModelCentroidView(ListAPIView): search_fields = ["name"] +class UsersView(ListAPIView): + authentication_classes = [OsmAuthentication] + permission_classes = [IsOsmAuthenticated] + queryset = OsmUser.objects.all() + serializer_class = UserSerializer + filter_backends = ( + # InBBoxFilter, + DjangoFilterBackend, + filters.SearchFilter, + ) + filterset_fields = ["id"] + search_fields = ["username"] + + class AOIViewSet(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] From d1d39b7706f871d600c2096db7d089454fa9e1ae Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 2 Oct 2024 23:22:57 +0200 Subject: [PATCH 05/12] refactor(id): id on usersview to search --- backend/core/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/views.py b/backend/core/views.py index 6efcf9d3..32f191da 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -278,7 +278,7 @@ class UsersView(ListAPIView): filters.SearchFilter, ) filterset_fields = ["id"] - search_fields = ["username"] + search_fields = ["username", "id"] class AOIViewSet(viewsets.ModelViewSet): From 0d413b369d4ef477ca673e6680ac39330ed7c233 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 3 Oct 2024 08:01:18 +0200 Subject: [PATCH 06/12] perf(order): ordering on modelset --- backend/core/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/core/views.py b/backend/core/views.py index 32f191da..1d26b9ce 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -243,6 +243,7 @@ class ModelViewSet( InBBoxFilter, # it will take bbox like this api/v1/model/?in_bbox=-90,29,-89,35 , DjangoFilterBackend, filters.SearchFilter, + filters.OrderingFilter, ) serializer_class = ModelSerializer filterset_fields = { @@ -252,6 +253,7 @@ class ModelViewSet( "created_by": ["exact"], "id": ["exact"], } + ordering_fields = ["created_at", "last_modified", "id", "status"] search_fields = ["name"] From 7310afdb3bac9dee89fca4879b560361c5ef4f9c Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 7 Oct 2024 12:01:13 +0200 Subject: [PATCH 07/12] Add model description --- backend/core/models.py | 1 + backend/core/serializers.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 7ad9afd6..60247ff9 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -56,6 +56,7 @@ class ModelStatus(models.IntegerChoices): name = models.CharField(max_length=255) created_at = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) + description = models.TextField(max_length=500, null=True, blank=True) created_by = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) published_training = models.PositiveIntegerField(null=True, blank=True) status = models.IntegerField(default=-1, choices=ModelStatus.choices) # diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 33d03d76..0757ba7c 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -1,10 +1,11 @@ from django.conf import settings -from login.models import OsmUser from rest_framework import serializers from rest_framework_gis.serializers import ( GeoFeatureModelSerializer, # this will be used if we used to serialize as geojson ) +from login.models import OsmUser + from .models import * # from .tasks import train_model @@ -48,10 +49,11 @@ class ModelSerializer( serializers.ModelSerializer ): # serializers are used to translate models objects to api created_by = UserSerializer(read_only=True) + accuracy = serializers.SerializerMethodField() class Meta: model = Model - fields = "__all__" # defining all the fields to be included in curd for now , we can restrict few if we want + fields = "__all__" read_only_fields = ( "created_at", "last_modified", @@ -64,6 +66,12 @@ def create(self, validated_data): validated_data["created_by"] = user return super().create(validated_data) + def get_accuracy(self, obj): + training = Training.objects.filter(id=obj.published_training).first() + if training: + return training.accuracy + return None + class ModelCentroidSerializer(serializers.ModelSerializer): geometry = serializers.SerializerMethodField() From 161c7e7c596b4e495fc268598593932e7f514ab8 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 7 Oct 2024 12:04:29 +0200 Subject: [PATCH 08/12] Add inline comments --- backend/core/serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 0757ba7c..b10e98cb 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -66,7 +66,9 @@ def create(self, validated_data): validated_data["created_by"] = user return super().create(validated_data) - def get_accuracy(self, obj): + def get_accuracy( + self, obj + ): ### Don't make this to production , add accuracy and centroid to model itself so that API call will be faster for getting all models without foreign table call training = Training.objects.filter(id=obj.published_training).first() if training: return training.accuracy From 3c75fc764092efc10bbfe382627d15c2f1ad10ed Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 7 Oct 2024 14:53:42 +0200 Subject: [PATCH 09/12] Featuer : add geojson response for the model centroid --- backend/core/serializers.py | 5 +++-- backend/core/views.py | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index b10e98cb..0169fe3d 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -68,18 +68,19 @@ def create(self, validated_data): def get_accuracy( self, obj - ): ### Don't make this to production , add accuracy and centroid to model itself so that API call will be faster for getting all models without foreign table call + ): ## this might have performance problem when db grows bigger , consider adding indexes / view in db training = Training.objects.filter(id=obj.published_training).first() if training: return training.accuracy return None -class ModelCentroidSerializer(serializers.ModelSerializer): +class ModelCentroidSerializer(GeoFeatureModelSerializer): geometry = serializers.SerializerMethodField() class Meta: model = Model + geo_field = "geometry" fields = ("id", "name", "geometry") def get_geometry(self, obj): diff --git a/backend/core/views.py b/backend/core/views.py index 1d26b9ce..c89e5974 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -26,8 +26,6 @@ from django_filters.rest_framework import DjangoFilterBackend from drf_yasg.utils import swagger_auto_schema from geojson2osm import geojson2osm -from login.authentication import OsmAuthentication -from login.permissions import IsOsmAuthenticated from orthogonalizer import othogonalize_poly from osmconflator import conflate_geojson from rest_framework import decorators, filters, serializers, status, viewsets @@ -38,6 +36,9 @@ from rest_framework.views import APIView from rest_framework_gis.filters import InBBoxFilter, TMSTileFilter +from login.authentication import OsmAuthentication +from login.permissions import IsOsmAuthenticated + from .models import ( AOI, ApprovedPredictions, @@ -258,7 +259,7 @@ class ModelViewSet( class ModelCentroidView(ListAPIView): - queryset = Model.objects.all() + queryset = Model.objects.filter(status=0) ## only deliver the published model serializer_class = ModelCentroidSerializer filter_backends = ( # InBBoxFilter, @@ -267,6 +268,7 @@ class ModelCentroidView(ListAPIView): ) filterset_fields = ["id"] search_fields = ["name"] + pagination_class = None class UsersView(ListAPIView): From 6547a7672cf446f7a3e5b88850f264ee8539e97b Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 7 Oct 2024 16:36:02 +0200 Subject: [PATCH 10/12] Add id in serializer --- backend/core/serializers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 0169fe3d..37255889 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -95,6 +95,14 @@ def get_geometry(self, obj): } return None + def to_representation(self, instance): + """ + Override to_representation to customize GeoJSON structure. + """ + representation = super().to_representation(instance) + representation["properties"]["id"] = representation["id"] + return representation + class AOISerializer( GeoFeatureModelSerializer From 5afff97acbd5255898963d7a39ab50cde4fd68c8 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 7 Oct 2024 16:52:28 +0200 Subject: [PATCH 11/12] pop id as it is already being added in response models --- backend/core/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 37255889..d2844113 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -100,7 +100,7 @@ def to_representation(self, instance): Override to_representation to customize GeoJSON structure. """ representation = super().to_representation(instance) - representation["properties"]["id"] = representation["id"] + representation["properties"]["id"] = representation.pop("id") return representation From 9e51af59ce553f32fa7d7d39993d4fbdf051c3a3 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 7 Oct 2024 16:57:26 +0200 Subject: [PATCH 12/12] add mid to the properties instead of id --- backend/core/serializers.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index d2844113..3f4cc556 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -77,11 +77,12 @@ def get_accuracy( class ModelCentroidSerializer(GeoFeatureModelSerializer): geometry = serializers.SerializerMethodField() + mid = serializers.IntegerField(source="id") class Meta: model = Model geo_field = "geometry" - fields = ("id", "name", "geometry") + fields = ("mid", "name", "geometry") def get_geometry(self, obj): """ @@ -95,13 +96,13 @@ def get_geometry(self, obj): } return None - def to_representation(self, instance): - """ - Override to_representation to customize GeoJSON structure. - """ - representation = super().to_representation(instance) - representation["properties"]["id"] = representation.pop("id") - return representation + # def to_representation(self, instance): + # """ + # Override to_representation to customize GeoJSON structure. + # """ + # representation = super().to_representation(instance) + # representation["properties"]["id"] = representation.pop("id") + # return representation class AOISerializer(