From 19b3e45019d9d5c5a042167626598b8f4d824fb1 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 8 Oct 2024 14:26:21 +0200 Subject: [PATCH 01/23] Add centroid lat and lon --- backend/core/serializers.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 3f4cc556..629a0ba9 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -1,11 +1,11 @@ +import mercantile from django.conf import settings +from login.models import OsmUser from rest_framework import serializers from rest_framework_gis.serializers import ( GeoFeatureModelSerializer, # this will be used if we used to serialize as geojson ) -from login.models import OsmUser - from .models import * # from .tasks import train_model @@ -50,6 +50,7 @@ class ModelSerializer( ): # serializers are used to translate models objects to api created_by = UserSerializer(read_only=True) accuracy = serializers.SerializerMethodField() + tile = serializers.SerializerMethodField() class Meta: model = Model @@ -66,6 +67,17 @@ def create(self, validated_data): validated_data["created_by"] = user return super().create(validated_data) + def get_tile(self, obj): + aoi = AOI.objects.filter(dataset=obj.dataset).first() + if aoi and aoi.geom: + centroid = aoi.geom.centroid.coords + try: + tile = mercantile.tile(centroid[0], centroid[1], zoom=18) + return [tile.x, tile.y, 18] + except: + pass + return None + def get_accuracy( self, obj ): ## this might have performance problem when db grows bigger , consider adding indexes / view in db @@ -82,7 +94,8 @@ class ModelCentroidSerializer(GeoFeatureModelSerializer): class Meta: model = Model geo_field = "geometry" - fields = ("mid", "name", "geometry") + fields = ("mid", "geometry") + # fields = ("mid", "name", "geometry") def get_geometry(self, obj): """ From db90cce93066b7d0d6a0cebc71b15b17ea323665 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 8 Oct 2024 14:26:40 +0200 Subject: [PATCH 02/23] Add mercantile requirements --- backend/api-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/api-requirements.txt b/backend/api-requirements.txt index 17187cea..e74dc0a7 100644 --- a/backend/api-requirements.txt +++ b/backend/api-requirements.txt @@ -23,5 +23,5 @@ fairpredictor==0.0.26 rasterio==1.3.8 numpy<2.0.0 - +mercantile==1.2.1 From d9d4f62ea5bd4aadbb7cea386c235fa88a3f4b07 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 8 Oct 2024 14:53:51 +0200 Subject: [PATCH 03/23] Added thumnail_url --- backend/core/serializers.py | 41 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 629a0ba9..c49a72a0 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -45,12 +45,10 @@ class Meta: ] -class ModelSerializer( - serializers.ModelSerializer -): # serializers are used to translate models objects to api +class ModelSerializer(serializers.ModelSerializer): created_by = UserSerializer(read_only=True) accuracy = serializers.SerializerMethodField() - tile = serializers.SerializerMethodField() + thumbnail_url = serializers.SerializerMethodField() class Meta: model = Model @@ -67,20 +65,31 @@ def create(self, validated_data): validated_data["created_by"] = user return super().create(validated_data) - def get_tile(self, obj): - aoi = AOI.objects.filter(dataset=obj.dataset).first() - if aoi and aoi.geom: - centroid = aoi.geom.centroid.coords - try: - tile = mercantile.tile(centroid[0], centroid[1], zoom=18) - return [tile.x, tile.y, 18] - except: - pass + def get_training(self, obj): + if not hasattr(self, "_cached_training"): + self._cached_training = Training.objects.filter( + id=obj.published_training + ).first() + return self._cached_training + + def get_thumbnail_url(self, obj): + training = Training.objects.filter(id=obj.published_training).first() + + if training: + if training.source_imagery: + aoi = AOI.objects.filter(dataset=obj.dataset).first() + if aoi and aoi.geom: + centroid = ( + aoi.geom.centroid.coords + ) ## Centroid can be stored in db table if required when project grows bigger + try: + tile = mercantile.tile(centroid[0], centroid[1], zoom=18) + return training.source_imagery.format(x=tile.x, y=tile.y, z=18) + except Exception as ex: + pass return None - def get_accuracy( - self, obj - ): ## this might have performance problem when db grows bigger , consider adding indexes / view in db + def get_accuracy(self, obj): training = Training.objects.filter(id=obj.published_training).first() if training: return training.accuracy From a15bf2a1a4171d189f11ac0b0817c012b0a38fd1 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 8 Oct 2024 14:55:25 +0200 Subject: [PATCH 04/23] Commented get_training function for now --- backend/core/serializers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index c49a72a0..fbcd1efc 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -65,12 +65,12 @@ def create(self, validated_data): validated_data["created_by"] = user return super().create(validated_data) - def get_training(self, obj): - if not hasattr(self, "_cached_training"): - self._cached_training = Training.objects.filter( - id=obj.published_training - ).first() - return self._cached_training + # def get_training(self, obj): + # if not hasattr(self, "_cached_training"): + # self._cached_training = Training.objects.filter( + # id=obj.published_training + # ).first() + # return self._cached_training def get_thumbnail_url(self, obj): training = Training.objects.filter(id=obj.published_training).first() From 6ba4c0b666dc34331a5e7cd0ff38ac734ed361b1 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 10 Oct 2024 13:06:06 +0200 Subject: [PATCH 05/23] Addes tippecanoe to generate vector tiles from aois and labels along with the centroid of the training is stored now --- backend/core/models.py | 2 +- backend/core/tasks.py | 25 +++++++++++++++++++------ backend/requirements.txt | 3 ++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 60247ff9..913d5fe6 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -2,7 +2,6 @@ from django.contrib.postgres.fields import ArrayField from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models - from login.models import OsmUser # Create your models here. @@ -89,6 +88,7 @@ class Training(models.Model): chips_length = models.PositiveIntegerField(default=0) batch_size = models.PositiveIntegerField() freeze_layers = models.BooleanField(default=False) + centroid = geomodels.PointField(srid=4326, null=True, blank=True) class Feedback(models.Model): diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 05b674a1..e553d139 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -2,18 +2,13 @@ import logging import os import shutil +import subprocess import sys import tarfile import traceback from shutil import rmtree from celery import shared_task -from django.conf import settings -from django.contrib.gis.db.models.aggregates import Extent -from django.contrib.gis.geos import GEOSGeometry -from django.shortcuts import get_object_or_404 -from django.utils import timezone - from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training from core.serializers import ( AOISerializer, @@ -23,6 +18,11 @@ LabelFileSerializer, ) from core.utils import bbox, is_dir_empty +from django.conf import settings +from django.contrib.gis.db.models.aggregates import Extent +from django.contrib.gis.geos import GEOSGeometry +from django.shortcuts import get_object_or_404 +from django.utils import timezone logger = logging.getLogger(__name__) @@ -135,6 +135,10 @@ def train_model( raise ValueError( f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", ) + first_aoi_centroid = aois[0].geom.centroid + training_instance.centroid = first_aoi_centroid + training_instance.save() + for obj in aois: bbox_coords = bbox(obj.geom.coords[0]) for z in zoom_level: @@ -309,6 +313,15 @@ def train_model( ) as f: f.write(json.dumps(aoi_serializer.data)) + tippecanoe_command = f"tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 --named-layer={os.path.join(output_path, "aois.geojson")} --named-layer={os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed" + logging.info("Starting to generate vector tiles for aois and labels") + try: + subprocess.check_output(tippecanoe_command) + except subprocess.CalledProcessError as ex: + logger.error(ex.output) + raise ex + logging.info("Vector tile generation done !") + # copy aois and labels to preprocess output before compressing it to tar shutil.copyfile( os.path.join(output_path, "aois.geojson"), diff --git a/backend/requirements.txt b/backend/requirements.txt index e6edc6ac..d7b4e8cc 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,3 +1,4 @@ -r api-requirements.txt hot-fair-utilities==1.3.0 -tflite-runtime==2.14.0 \ No newline at end of file +tflite-runtime==2.14.0 +tippecanoe==2.45.0 \ No newline at end of file From 0ab6df3b0ab634d3385b812cb61c24351d3d7edf Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 10 Oct 2024 20:37:52 +0200 Subject: [PATCH 06/23] Updated docker compose file and added dockerfile frontend --- backend/core/tasks.py | 4 ++-- docker-compose-cpu.yml | 2 +- docker-compose.yml | 2 +- frontend/Dockerfile.frontend | 15 +++++++++++++++ setup-ramp.sh | 32 +++++++++++++++----------------- 5 files changed, 34 insertions(+), 21 deletions(-) create mode 100644 frontend/Dockerfile.frontend diff --git a/backend/core/tasks.py b/backend/core/tasks.py index e553d139..91ca85b9 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -313,7 +313,7 @@ def train_model( ) as f: f.write(json.dumps(aoi_serializer.data)) - tippecanoe_command = f"tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 --named-layer={os.path.join(output_path, "aois.geojson")} --named-layer={os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed" + tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 --named-layer={os.path.join(output_path, "aois.geojson")} --named-layer={os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" logging.info("Starting to generate vector tiles for aois and labels") try: subprocess.check_output(tippecanoe_command) @@ -321,7 +321,7 @@ def train_model( logger.error(ex.output) raise ex logging.info("Vector tile generation done !") - + # copy aois and labels to preprocess output before compressing it to tar shutil.copyfile( os.path.join(output_path, "aois.geojson"), diff --git a/docker-compose-cpu.yml b/docker-compose-cpu.yml index 20777bcc..a6e6eae9 100644 --- a/docker-compose-cpu.yml +++ b/docker-compose-cpu.yml @@ -69,7 +69,7 @@ services: context: ./frontend dockerfile: Dockerfile.frontend container_name: frontend - command: npm start -- --host 0.0.0.0 --port 3000 + command: npm run dev -- --host 0.0.0.0 --port 3000 ports: - 3000:3000 depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index ac37dbee..cb0c6460 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,7 +74,7 @@ services: context: ./frontend dockerfile: Dockerfile.frontend container_name: frontend - command: npm start -- --host 0.0.0.0 --port 3000 + command: npm run dev -- --host 0.0.0.0 --port 3000 ports: - 3000:3000 depends_on: diff --git a/frontend/Dockerfile.frontend b/frontend/Dockerfile.frontend new file mode 100644 index 00000000..0738b662 --- /dev/null +++ b/frontend/Dockerfile.frontend @@ -0,0 +1,15 @@ + +FROM node:20.18 + +WORKDIR /app + + +COPY . /app + + +RUN npm install --force + + +# RUN npm run build + +# EXPOSE 3000 \ No newline at end of file diff --git a/setup-ramp.sh b/setup-ramp.sh index 817844ee..84abdc31 100644 --- a/setup-ramp.sh +++ b/setup-ramp.sh @@ -2,29 +2,27 @@ ## To run this activate your venv and hit bash setup-ramp.sh -# Step 1: Create a new folder called 'ramp' outside fAIr -mkdir -p ramp -# Step 2: Install gdown for downloading files from Google Drive -pip install gdown +if [ ! -d "ramp_base" ]; then -# Step 3: Download BaseModel Checkpoint from Google Drive -gdown --fuzzy https://drive.google.com/uc?id=1YQsY61S_rGfJ_f6kLQq4ouYE2l3iRe1k + mkdir -p ramp_base -# Step 4: Clone the Ramp code repository -git clone https://github.com/kshitijrajsharma/ramp-code-fAIr.git ramp-code -# Step 5: Unzip the downloaded BaseModel checkpoint into the 'ramp' directory inside the cloned repository -unzip checkpoint.tf.zip -d ramp-code/ramp + pip install gdown -# Step 6: Define the current location for environment variables -RAMP_HOME="$(pwd)/ramp" + + gdown --fuzzy https://drive.google.com/uc?id=1YQsY61S_rGfJ_f6kLQq4ouYE2l3iRe1k + + git clone https://github.com/kshitijrajsharma/ramp-code-fAIr.git "$(pwd)/ramp_base/ramp-code" + + + unzip checkpoint.tf.zip -d "$(pwd)/ramp_base/ramp-code/ramp" + + echo "Setup complete. Please run 'source .env' to apply the environment variables." +fi + +RAMP_HOME="$(pwd)/ramp_base" TRAINING_WORKSPACE="$(pwd)/trainings" -# Step 7: Create a '.env' file with the exported variables echo "export RAMP_HOME=$RAMP_HOME" > .env echo "export TRAINING_WORKSPACE=$TRAINING_WORKSPACE" >> .env - -# Print success message -echo "Setup complete. Please run 'source .env' to apply the environment variables." - From 28b81d84bd56dfc8541edc9b841fdd6cc3173193 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 09:50:42 +0200 Subject: [PATCH 07/23] Feat : Search by model id --- backend/core/views.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/core/views.py b/backend/core/views.py index c89e5974..f7d7cfc4 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -26,6 +26,8 @@ from django_filters.rest_framework import DjangoFilterBackend from drf_yasg.utils import swagger_auto_schema from geojson2osm import geojson2osm +from login.authentication import OsmAuthentication +from login.permissions import IsOsmAuthenticated from orthogonalizer import othogonalize_poly from osmconflator import conflate_geojson from rest_framework import decorators, filters, serializers, status, viewsets @@ -36,9 +38,6 @@ from rest_framework.views import APIView from rest_framework_gis.filters import InBBoxFilter, TMSTileFilter -from login.authentication import OsmAuthentication -from login.permissions import IsOsmAuthenticated - from .models import ( AOI, ApprovedPredictions, @@ -255,7 +254,7 @@ class ModelViewSet( "id": ["exact"], } ordering_fields = ["created_at", "last_modified", "id", "status"] - search_fields = ["name"] + search_fields = ["name", "id"] class ModelCentroidView(ListAPIView): From 0dc50aec437ae1db93f22a7f5e767c9145615418 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 10:11:27 +0200 Subject: [PATCH 08/23] Feat : Added feedback count to training /get/id/ endpoint --- backend/core/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/core/views.py b/backend/core/views.py index f7d7cfc4..bf93ed8d 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -192,6 +192,16 @@ class TrainingViewSet( serializer_class = TrainingSerializer # connecting serializer filterset_fields = ["model", "status"] + def retrieve(self, request, *args, **kwargs): + instance = self.get_object() + serializer = self.get_serializer(instance) + feedback_count = Feedback.objects.filter( + training=instance.id + ).count() # cal feedback count + data = serializer.data + data["feedback_count"] = feedback_count + return Response(data, status=status.HTTP_200_OK) + class FeedbackViewset(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] From 1bbb0e9bb51fc2af56a76d0fd0543305169abb75 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 10:20:17 +0200 Subject: [PATCH 09/23] Feat : Add banners from backend --- backend/core/models.py | 19 +++++++++++++++++++ backend/core/serializers.py | 14 ++++++++++++++ backend/core/urls.py | 2 ++ backend/core/views.py | 14 ++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/backend/core/models.py b/backend/core/models.py index 913d5fe6..1f7c1c17 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -2,6 +2,7 @@ from django.contrib.postgres.fields import ArrayField from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from django.utils import timezone from login.models import OsmUser # Create your models here. @@ -149,3 +150,21 @@ class ApprovedPredictions(models.Model): approved_by = models.ForeignKey( OsmUser, to_field="osm_id", on_delete=models.CASCADE ) + + +class Banner(models.Model): + message = models.CharField(max_length=255) + start_date = models.DateTimeField(default=timezone.now) + end_date = models.DateTimeField(null=True, blank=True) + is_active = models.BooleanField(default=True) + + def is_displayable(self): + now = timezone.now() + return ( + self.is_active + and (self.start_date <= now) + and (self.end_date is None or self.end_date >= now) + ) + + def __str__(self): + return self.message diff --git a/backend/core/serializers.py b/backend/core/serializers.py index fbcd1efc..7586aa00 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -393,3 +393,17 @@ def validate(self, data): data["area_threshold"] ) return data + + +class BannerSerializer(serializers.ModelSerializer): + class Meta: + model = Banner + fields = [ + "id", + "message", + "start_date", + "end_date", + "is_active", + "is_displayable", + ] + read_only_fields = ["is_displayable"] diff --git a/backend/core/urls.py b/backend/core/urls.py index b54bc169..f8dc8ac6 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -7,6 +7,7 @@ from .views import ( # APIStatus, AOIViewSet, ApprovedPredictionsViewSet, + BannerViewSet, ConflateGeojson, DatasetViewSet, FeedbackAOIViewset, @@ -44,6 +45,7 @@ router.register(r"feedback", FeedbackViewset) router.register(r"feedback-aoi", FeedbackAOIViewset) router.register(r"feedback-label", FeedbackLabelViewset) +router.register(r"banners", BannerViewSet) urlpatterns = [ diff --git a/backend/core/views.py b/backend/core/views.py index bf93ed8d..788bb182 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -23,6 +23,7 @@ StreamingHttpResponse, ) from django.shortcuts import get_object_or_404, redirect +from django.utils import timezone from django_filters.rest_framework import DjangoFilterBackend from drf_yasg.utils import swagger_auto_schema from geojson2osm import geojson2osm @@ -41,6 +42,7 @@ from .models import ( AOI, ApprovedPredictions, + Banner, Dataset, Feedback, FeedbackAOI, @@ -53,6 +55,7 @@ from .serializers import ( AOISerializer, ApprovedPredictionsSerializer, + BannerSerializer, DatasetSerializer, FeedbackAOISerializer, FeedbackFileSerializer, @@ -838,3 +841,14 @@ def get(self, request, lookup_dir): os.path.basename(base_dir) ) return response + + +class BannerViewSet(viewsets.ModelViewSet): + queryset = Banner.objects.all() + serializer_class = BannerSerializer + + def get_queryset(self): + now = timezone.now() + return Banner.objects.filter(is_active=True, start_date__lte=now).filter( + end_date__gte=now + ) | Banner.objects.filter(is_active=True, end_date__isnull=True) From d568faf9fca548060a24ddc1d42b875a14544b56 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 10:28:33 +0200 Subject: [PATCH 10/23] Add admin view and permission set for retrieving the data --- backend/core/admin.py | 14 ++++++++++++++ backend/core/views.py | 3 +++ 2 files changed, 17 insertions(+) diff --git a/backend/core/admin.py b/backend/core/admin.py index ccd5af55..350f3d14 100644 --- a/backend/core/admin.py +++ b/backend/core/admin.py @@ -47,3 +47,17 @@ class FeedbackAOIAdmin(geoadmin.OSMGeoAdmin): @admin.register(Feedback) class FeedbackAdmin(geoadmin.OSMGeoAdmin): list_display = ["feedback_type", "training", "user", "created_at"] + + +@admin.register(Banner) +class BannerAdmin(admin.ModelAdmin): + list_display = ("message", "start_date", "end_date", "is_active", "is_displayable") + list_filter = ("is_active", "start_date", "end_date") + search_fields = ("message",) + readonly_fields = ("is_displayable",) + + def is_displayable(self, obj): + return obj.is_displayable() + + is_displayable.boolean = True + is_displayable.short_description = "Currently Displayable" diff --git a/backend/core/views.py b/backend/core/views.py index 788bb182..186dc251 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -846,6 +846,9 @@ def get(self, request, lookup_dir): class BannerViewSet(viewsets.ModelViewSet): queryset = Banner.objects.all() serializer_class = BannerSerializer + authentication_classes = [OsmAuthentication] + permission_classes = [IsOsmAuthenticated] + permission_allowed_methods = ["GET"] def get_queryset(self): now = timezone.now() From 382274f02e54444589e40856131db02731e567ae Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 11:27:51 +0200 Subject: [PATCH 11/23] Refactor ::: created_by -> user --- backend/core/admin.py | 6 +++--- backend/core/models.py | 10 ++++----- backend/core/serializers.py | 10 ++++----- backend/core/views.py | 14 ++++++------ backend/login/permissions.py | 38 ++++++++++++++++++++++++++++++--- backend/tests/factories.py | 18 ++++++++-------- backend/tests/test_endpoints.py | 29 +++++++++++++------------ 7 files changed, 78 insertions(+), 47 deletions(-) diff --git a/backend/core/admin.py b/backend/core/admin.py index 350f3d14..9c0f3bfa 100644 --- a/backend/core/admin.py +++ b/backend/core/admin.py @@ -8,12 +8,12 @@ @admin.register(Dataset) class DatasetAdmin(geoadmin.OSMGeoAdmin): - list_display = ["name", "created_by"] + list_display = ["name", "user"] @admin.register(Model) class ModelAdmin(geoadmin.OSMGeoAdmin): - list_display = ["get_dataset_id", "name", "status", "created_at", "created_by"] + list_display = ["get_dataset_id", "name", "status", "created_at", "user"] def get_dataset_id(self, obj): return obj.dataset.id @@ -28,7 +28,7 @@ class TrainingAdmin(geoadmin.OSMGeoAdmin): "description", "status", "zoom_level", - "created_by", + "user", "accuracy", ] list_filter = ["status"] diff --git a/backend/core/models.py b/backend/core/models.py index 1f7c1c17..77dc577c 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -15,7 +15,7 @@ class DatasetStatus(models.IntegerChoices): DRAFT = -1 name = models.CharField(max_length=255) - created_by = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) + user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) last_modified = models.DateTimeField(auto_now=True) created_at = models.DateTimeField(auto_now_add=True) source_imagery = models.URLField(blank=True, null=True) @@ -57,7 +57,7 @@ class ModelStatus(models.IntegerChoices): created_at = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) description = models.TextField(max_length=500, null=True, blank=True) - created_by = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) + user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) published_training = models.PositiveIntegerField(null=True, blank=True) status = models.IntegerField(default=-1, choices=ModelStatus.choices) # @@ -81,7 +81,7 @@ class Training(models.Model): models.PositiveIntegerField(), size=4, ) - created_by = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) + user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) started_at = models.DateTimeField(null=True, blank=True) finished_at = models.DateTimeField(null=True, blank=True) accuracy = models.FloatField(null=True, blank=True) @@ -147,9 +147,7 @@ class ApprovedPredictions(models.Model): srid=4326 ) ## Making this geometry field to support point/line prediction later on approved_at = models.DateTimeField(auto_now_add=True) - approved_by = models.ForeignKey( - OsmUser, to_field="osm_id", on_delete=models.CASCADE - ) + user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) class Banner(models.Model): diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 7586aa00..ec2a22d8 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -18,14 +18,14 @@ class Meta: model = Dataset fields = "__all__" # defining all the fields to be included in curd for now , we can restrict few if we want read_only_fields = ( - "created_by", + "user", "created_at", "last_modified", ) def create(self, validated_data): user = self.context["request"].user - validated_data["created_by"] = user + validated_data["user"] = user return super().create(validated_data) @@ -46,7 +46,7 @@ class Meta: class ModelSerializer(serializers.ModelSerializer): - created_by = UserSerializer(read_only=True) + user = UserSerializer(read_only=True) accuracy = serializers.SerializerMethodField() thumbnail_url = serializers.SerializerMethodField() @@ -56,13 +56,13 @@ class Meta: read_only_fields = ( "created_at", "last_modified", - "created_by", + "user", "published_training", ) def create(self, validated_data): user = self.context["request"].user - validated_data["created_by"] = user + validated_data["user"] = user return super().create(validated_data) # def get_training(self, obj): diff --git a/backend/core/views.py b/backend/core/views.py index 186dc251..8ab580a2 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -28,7 +28,7 @@ from drf_yasg.utils import swagger_auto_schema from geojson2osm import geojson2osm from login.authentication import OsmAuthentication -from login.permissions import IsOsmAuthenticated +from login.permissions import IsAdminUser, IsOsmAuthenticated, IsStaffUser from orthogonalizer import othogonalize_poly from osmconflator import conflate_geojson from rest_framework import decorators, filters, serializers, status, viewsets @@ -107,7 +107,7 @@ class Meta: read_only_fields = ( "created_at", "status", - "created_by", + "user", "started_at", "finished_at", "accuracy", @@ -144,7 +144,7 @@ def create(self, validated_data): ) user = self.context["request"].user - validated_data["created_by"] = user + validated_data["user"] = user # create the model instance multimasks = validated_data.get("multimasks", False) input_contact_spacing = validated_data.get("input_contact_spacing", 0.75) @@ -263,7 +263,7 @@ class ModelViewSet( "status": ["exact"], "created_at": ["exact", "gt", "gte", "lt", "lte"], "last_modified": ["exact", "gt", "gte", "lt", "lte"], - "created_by": ["exact"], + "user": ["exact"], "id": ["exact"], } ordering_fields = ["created_at", "last_modified", "id", "status"] @@ -364,7 +364,7 @@ class ApprovedPredictionsViewSet(viewsets.ModelViewSet): def create(self, request, *args, **kwargs): training_id = request.data.get("training") geom = request.data.get("geom") - request.data["approved_by"] = self.request.user.osm_id + request.data["user"] = self.request.user.osm_id existing_approved_feature = ApprovedPredictions.objects.filter( training=training_id, geom=geom @@ -589,7 +589,7 @@ def post(self, request, *args, **kwargs): model=training_instance.model, status="SUBMITTED", description=f"Feedback of Training {training_id}", - created_by=self.request.user, + user=self.request.user, zoom_level=zoom_level, epochs=epochs, batch_size=batch_size, @@ -847,7 +847,7 @@ class BannerViewSet(viewsets.ModelViewSet): queryset = Banner.objects.all() serializer_class = BannerSerializer authentication_classes = [OsmAuthentication] - permission_classes = [IsOsmAuthenticated] + permission_classes = [IsStaffUser, IsAdminUser] permission_allowed_methods = ["GET"] def get_queryset(self): diff --git a/backend/login/permissions.py b/backend/login/permissions.py index fd8b482c..fef9fd74 100644 --- a/backend/login/permissions.py +++ b/backend/login/permissions.py @@ -8,9 +8,41 @@ class IsOsmAuthenticated(permissions.BasePermission): def has_permission(self, request, view): permission_allowed_methods = getattr(view, "permission_allowed_methods", []) - if request.method in permission_allowed_methods: # if request method is set to allowed give them permission + if request.method in permission_allowed_methods: return True - if request.user: + # If the user is authenticated, allow access + if request.user and request.user.is_authenticated: + # Global access for staff and admin users + if request.user.is_staff or request.user.is_superuser: + return True + + return True + + return False + + def has_object_permission(self, request, view, obj): + # Allow read-only access for any authenticated user + if request.method in permissions.SAFE_METHODS: + return True + + # Allow modification (PUT, DELETE) if the user is staff or admin + if request.user.is_staff or request.user.is_superuser: + return True + + # Check if the object has a 'creator' field and if the user is the creator + if hasattr(obj, "creator") and obj.creator == request.user: return True - return False \ No newline at end of file + return False + + +class IsAdminUser(permissions.BasePermission): + def has_permission(self, request, view): + return ( + request.user and request.user.is_authenticated and request.user.is_superuser + ) + + +class IsStaffUser(permissions.BasePermission): + def has_permission(self, request, view): + return request.user and request.user.is_authenticated and request.user.is_staff diff --git a/backend/tests/factories.py b/backend/tests/factories.py index 829c1b4e..0a407535 100644 --- a/backend/tests/factories.py +++ b/backend/tests/factories.py @@ -1,16 +1,16 @@ import factory -from login.models import OsmUser -from django.contrib.gis.geos import Polygon from core.models import ( - Dataset, AOI, - Label, - Model, - Training, + Dataset, Feedback, FeedbackAOI, FeedbackLabel, + Label, + Model, + Training, ) +from django.contrib.gis.geos import Polygon +from login.models import OsmUser class OsmUserFactory(factory.django.DjangoModelFactory): @@ -26,7 +26,7 @@ class Meta: name = "My test dataset" source_imagery = "https://tiles.openaerialmap.org/5ac4fc6f26964b0010033112/0/5ac4fc6f26964b0010033113/{z}/{x}/{y}" - created_by = factory.SubFactory(OsmUserFactory) + user = factory.SubFactory(OsmUserFactory) class AoiFactory(factory.django.DjangoModelFactory): @@ -67,7 +67,7 @@ class Meta: dataset = factory.SubFactory(DatasetFactory) name = "My test model" - created_by = factory.SubFactory(OsmUserFactory) + user = factory.SubFactory(OsmUserFactory) class TrainingFactory(factory.django.DjangoModelFactory): @@ -76,7 +76,7 @@ class Meta: model = factory.SubFactory(ModelFactory) description = "My very first training" - created_by = factory.SubFactory(OsmUserFactory) + user = factory.SubFactory(OsmUserFactory) epochs = 1 zoom_level = [20, 21] batch_size = 1 diff --git a/backend/tests/test_endpoints.py b/backend/tests/test_endpoints.py index 246993c5..e6461768 100644 --- a/backend/tests/test_endpoints.py +++ b/backend/tests/test_endpoints.py @@ -2,18 +2,19 @@ import os import shutil -from django.conf import settings import validators +from django.conf import settings from rest_framework import status from rest_framework.test import APILiveServerTestCase, RequestsClient + from .factories import ( - OsmUserFactory, - TrainingFactory, - DatasetFactory, AoiFactory, + DatasetFactory, + FeedbackAoiFactory, LabelFactory, ModelFactory, - FeedbackAoiFactory, + OsmUserFactory, + TrainingFactory, ) API_BASE = "http://testserver/api/v1" @@ -30,9 +31,9 @@ def setUp(self): # Create a request factory instance self.client = RequestsClient() self.user = OsmUserFactory(osm_id=123) - self.dataset = DatasetFactory(created_by=self.user) + self.dataset = DatasetFactory(user=self.user) self.aoi = AoiFactory(dataset=self.dataset) - self.model = ModelFactory(dataset=self.dataset, created_by=self.user) + self.model = ModelFactory(dataset=self.dataset, user=self.user) self.json_type_header = headersList.copy() self.json_type_header["content-type"] = "application/json" @@ -187,11 +188,11 @@ def test_create_training(self): ) self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST) - self.training = TrainingFactory(model=self.model, created_by=self.user) + self.training = TrainingFactory(model=self.model, user=self.user) def test_create_label(self): self.label = LabelFactory(aoi=self.aoi) - self.training = TrainingFactory(model=self.model, created_by=self.user) + self.training = TrainingFactory(model=self.model, user=self.user) # create label @@ -236,7 +237,7 @@ def test_create_label(self): def test_fetch_feedbackAoi_osm_label(self): # create feedback aoi - training = TrainingFactory(model=self.model, created_by=self.user) + training = TrainingFactory(model=self.model, user=self.user) feedbackAoi = FeedbackAoiFactory(training=training, user=self.user) # download available osm data as labels for the feedback aoi @@ -249,7 +250,7 @@ def test_fetch_feedbackAoi_osm_label(self): self.assertEqual(res.status_code, status.HTTP_201_CREATED) def test_get_runStatus(self): - training = TrainingFactory(model=self.model, created_by=self.user) + training = TrainingFactory(model=self.model, user=self.user) # get running training status @@ -259,7 +260,7 @@ def test_get_runStatus(self): self.assertEqual(res.status_code, status.HTTP_200_OK) def test_submit_training_feedback(self): - training = TrainingFactory(model=self.model, created_by=self.user) + training = TrainingFactory(model=self.model, user=self.user) # apply feedback to training published checkpoints @@ -278,7 +279,7 @@ def test_submit_training_feedback(self): self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST) def test_publish_training(self): - training = TrainingFactory(model=self.model, created_by=self.user) + training = TrainingFactory(model=self.model, user=self.user) # publish an unfinished training should not pass @@ -288,7 +289,7 @@ def test_publish_training(self): self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND) def test_get_GpxView(self): - training = TrainingFactory(model=self.model, created_by=self.user) + training = TrainingFactory(model=self.model, user=self.user) feedbackAoi = FeedbackAoiFactory(training=training, user=self.user) # generate aoi GPX view - aoi_id From 7a441107afa6b4fa47a303e2f3dc9250c1a7ddf3 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 13:36:20 +0200 Subject: [PATCH 12/23] Feat : Authentication for admins and staffs --- backend/core/urls.py | 2 +- backend/core/views.py | 21 ++++---- backend/login/admin.py | 87 +++++++++++++++++++++++++++++++-- backend/login/authentication.py | 1 + backend/login/permissions.py | 4 +- 5 files changed, 98 insertions(+), 17 deletions(-) diff --git a/backend/core/urls.py b/backend/core/urls.py index f8dc8ac6..dd8bd48d 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -45,7 +45,7 @@ router.register(r"feedback", FeedbackViewset) router.register(r"feedback-aoi", FeedbackAOIViewset) router.register(r"feedback-label", FeedbackLabelViewset) -router.register(r"banners", BannerViewSet) +router.register(r"banner", BannerViewSet) urlpatterns = [ diff --git a/backend/core/views.py b/backend/core/views.py index 8ab580a2..8722ca3f 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -84,7 +84,7 @@ class DatasetViewSet( ): # This is datasetviewset , will be tightly coupled with the models authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = Dataset.objects.all() serializer_class = DatasetSerializer # connecting serializer @@ -189,7 +189,7 @@ class TrainingViewSet( ): # This is TrainingViewSet , will be tightly coupled with the models authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = Training.objects.all() http_method_names = ["get", "post", "delete"] serializer_class = TrainingSerializer # connecting serializer @@ -209,7 +209,7 @@ def retrieve(self, request, *args, **kwargs): class FeedbackViewset(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = Feedback.objects.all() http_method_names = ["get", "post", "patch", "delete"] serializer_class = FeedbackSerializer # connecting serializer @@ -219,7 +219,7 @@ class FeedbackViewset(viewsets.ModelViewSet): class FeedbackAOIViewset(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = FeedbackAOI.objects.all() http_method_names = ["get", "post", "patch", "delete"] serializer_class = FeedbackAOISerializer @@ -232,7 +232,7 @@ class FeedbackAOIViewset(viewsets.ModelViewSet): class FeedbackLabelViewset(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = FeedbackLabel.objects.all() http_method_names = ["get", "post", "patch", "delete"] serializer_class = FeedbackLabelSerializer @@ -250,7 +250,7 @@ class ModelViewSet( ): # This is ModelViewSet , will be tightly coupled with the models authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = Model.objects.all() filter_backends = ( InBBoxFilter, # it will take bbox like this api/v1/model/?in_bbox=-90,29,-89,35 , @@ -300,7 +300,7 @@ class UsersView(ListAPIView): class AOIViewSet(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = AOI.objects.all() serializer_class = AOISerializer # connecting serializer filter_backends = [DjangoFilterBackend] @@ -310,7 +310,7 @@ class AOIViewSet(viewsets.ModelViewSet): class LabelViewSet(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = Label.objects.all() serializer_class = LabelSerializer # connecting serializer bbox_filter_field = "geom" @@ -349,7 +349,7 @@ def create(self, request, *args, **kwargs): class ApprovedPredictionsViewSet(viewsets.ModelViewSet): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] - permission_allowed_methods = ["GET"] + public_methods = ["GET"] queryset = ApprovedPredictions.objects.all() serializer_class = ApprovedPredictionsSerializer bbox_filter_field = "geom" @@ -364,7 +364,7 @@ class ApprovedPredictionsViewSet(viewsets.ModelViewSet): def create(self, request, *args, **kwargs): training_id = request.data.get("training") geom = request.data.get("geom") - request.data["user"] = self.request.user.osm_id + request.data["approved_by"] = self.request.user.osm_id existing_approved_feature = ApprovedPredictions.objects.filter( training=training_id, geom=geom @@ -848,7 +848,6 @@ class BannerViewSet(viewsets.ModelViewSet): serializer_class = BannerSerializer authentication_classes = [OsmAuthentication] permission_classes = [IsStaffUser, IsAdminUser] - permission_allowed_methods = ["GET"] def get_queryset(self): now = timezone.now() diff --git a/backend/login/admin.py b/backend/login/admin.py index ef46bed6..72147c90 100644 --- a/backend/login/admin.py +++ b/backend/login/admin.py @@ -1,10 +1,91 @@ +from django import forms from django.contrib import admin +from django.contrib.auth.forms import UserChangeForm, UserCreationForm +from django.db import models from .models import OsmUser -# Register your models here. + +class OsmUserCreationForm(UserCreationForm): + class Meta: + model = OsmUser + fields = ( + "username", + "email", + "osm_id", + "img_url", + "is_staff", + "is_superuser", + "is_active", + ) + + +class OsmUserChangeForm(UserChangeForm): + class Meta: + model = OsmUser + fields = ( + "username", + "email", + "osm_id", + "img_url", + "is_staff", + "is_superuser", + "is_active", + ) @admin.register(OsmUser) -class DatasetAdmin(admin.ModelAdmin): - list_display = ["osm_id", "username"] +class OsmUserAdmin(admin.ModelAdmin): + add_form = OsmUserCreationForm + form = OsmUserChangeForm + model = OsmUser + + list_display = [ + "osm_id", + "username", + "email", + "is_staff", + "is_superuser", + "last_login", + ] + list_filter = ["is_staff", "is_superuser", "is_active"] + search_fields = ["username", "email", "osm_id"] + readonly_fields = ["last_login", "date_joined"] + + fieldsets = ( + (None, {"fields": ("username", "osm_id", "email", "img_url")}), + ( + "Permissions", + { + "fields": ( + "is_active", + "is_staff", + "is_superuser", + "groups", + "user_permissions", + ) + }, + ), + ("Important dates", {"fields": ("last_login", "date_joined")}), + ) + + add_fieldsets = ( + ( + None, + { + "classes": ("wide",), + "fields": ( + "username", + "email", + "osm_id", + "img_url", + "is_staff", + "is_superuser", + "is_active", + ), + }, + ), + ) + formfield_overrides = { + models.CharField: {"validators": []}, + } diff --git a/backend/login/authentication.py b/backend/login/authentication.py index 83e3c106..2fce1872 100644 --- a/backend/login/authentication.py +++ b/backend/login/authentication.py @@ -47,6 +47,7 @@ def authenticate(self, request): except Exception as ex: print(ex) + # raise ex raise exceptions.AuthenticationFailed( f"Osm Authentication Failed" ) # raise exception if user does not exist diff --git a/backend/login/permissions.py b/backend/login/permissions.py index fef9fd74..bc76ebad 100644 --- a/backend/login/permissions.py +++ b/backend/login/permissions.py @@ -7,8 +7,8 @@ class IsOsmAuthenticated(permissions.BasePermission): def has_permission(self, request, view): - permission_allowed_methods = getattr(view, "permission_allowed_methods", []) - if request.method in permission_allowed_methods: + public_methods = getattr(view, "public_methods", []) + if request.method in public_methods: return True # If the user is authenticated, allow access if request.user and request.user.is_authenticated: From feeaffd43247e845fb84ab427d227916067b77da Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 14:25:20 +0200 Subject: [PATCH 13/23] Fix tippecanoe command --- backend/core/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 91ca85b9..59f15293 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -313,7 +313,7 @@ def train_model( ) as f: f.write(json.dumps(aoi_serializer.data)) - tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 --named-layer={os.path.join(output_path, "aois.geojson")} --named-layer={os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" + tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 --named-layer=aois {os.path.join(output_path, "aois.geojson")} --named-layer=labels {os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" logging.info("Starting to generate vector tiles for aois and labels") try: subprocess.check_output(tippecanoe_command) From 4a107361e417fc8c85c92bd8373bbb5313aeee3f Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 14:26:52 +0200 Subject: [PATCH 14/23] Use -L instead of full named layer --- backend/core/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 59f15293..1f798fba 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -313,7 +313,7 @@ def train_model( ) as f: f.write(json.dumps(aoi_serializer.data)) - tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 --named-layer=aois {os.path.join(output_path, "aois.geojson")} --named-layer=labels {os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" + tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 -L aois:{os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" logging.info("Starting to generate vector tiles for aois and labels") try: subprocess.check_output(tippecanoe_command) From 2b3d9b3a78d71de9a06f92d065b32f3b3b9560c3 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 14:33:49 +0200 Subject: [PATCH 15/23] move tippecanoe to furhter steps --- backend/core/tasks.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 1f798fba..b0234fad 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -313,15 +313,6 @@ def train_model( ) as f: f.write(json.dumps(aoi_serializer.data)) - tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 -L aois:{os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" - logging.info("Starting to generate vector tiles for aois and labels") - try: - subprocess.check_output(tippecanoe_command) - except subprocess.CalledProcessError as ex: - logger.error(ex.output) - raise ex - logging.info("Vector tile generation done !") - # copy aois and labels to preprocess output before compressing it to tar shutil.copyfile( os.path.join(output_path, "aois.geojson"), @@ -337,6 +328,15 @@ def train_model( remove_original=True, ) + tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 -L aois:{os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" + logging.info("Starting to generate vector tiles for aois and labels") + try: + subprocess.check_output(tippecanoe_command) + except subprocess.CalledProcessError as ex: + logger.error(ex.output) + raise ex + logging.info("Vector tile generation done !") + # now remove the ramp-data all our outputs are copied to our training workspace shutil.rmtree(base_path) training_instance.accuracy = float(final_accuracy) From ba0c5ad8072074f6d18dd33aff26ad142895a047 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 15:22:52 +0200 Subject: [PATCH 16/23] Try : Fix files labels --- backend/core/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index b0234fad..db692a29 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -328,7 +328,7 @@ def train_model( remove_original=True, ) - tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 -L aois:{os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" + tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 -L aois:{ os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" logging.info("Starting to generate vector tiles for aois and labels") try: subprocess.check_output(tippecanoe_command) From 77c9889b887b9897fcf498b58548fe2c9529d267 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 15:37:21 +0200 Subject: [PATCH 17/23] Use subprocess run instead of check output --- backend/core/tasks.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index db692a29..fc9bc303 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -313,6 +313,18 @@ def train_model( ) as f: f.write(json.dumps(aoi_serializer.data)) + tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,"meta.pmtiles")} -Z7 -z18 -L aois:{ os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" + logging.info("Starting to generate vector tiles for aois and labels") + try: + result = subprocess.run( + tippecanoe_command, shell=True, check=True, capture_output=True + ) + logging.info(result.stdout.decode("utf-8")) + except subprocess.CalledProcessError as ex: + logger.error(ex.output) + raise ex + logging.info("Vector tile generation done !") + # copy aois and labels to preprocess output before compressing it to tar shutil.copyfile( os.path.join(output_path, "aois.geojson"), @@ -328,15 +340,6 @@ def train_model( remove_original=True, ) - tippecanoe_command = f"""tippecanoe -o {os.path.join(output_path,'meta.pmtiles')} -Z7 -z18 -L aois:{ os.path.join(output_path, "aois.geojson")} -L labels:{os.path.join(output_path, "labels.geojson")} --force --read-parallel -rg --drop-densest-as-needed""" - logging.info("Starting to generate vector tiles for aois and labels") - try: - subprocess.check_output(tippecanoe_command) - except subprocess.CalledProcessError as ex: - logger.error(ex.output) - raise ex - logging.info("Vector tile generation done !") - # now remove the ramp-data all our outputs are copied to our training workspace shutil.rmtree(base_path) training_instance.accuracy = float(final_accuracy) @@ -345,7 +348,7 @@ def train_model( training_instance.save() response = {} response["accuracy"] = float(final_accuracy) - # response["model_path"] = os.path.join(output_path, "checkpoint.tf") + response["tiles_path"] = os.path.join(output_path, "meta.pmtiles") response["model_path"] = os.path.join(output_path, "checkpoint.h5") response["graph_path"] = os.path.join(output_path, "graphs") sys.stdout = sys.__stdout__ From 27a32aeb368471212aec08e2cbe71d56a1abbb25 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Oct 2024 15:45:47 +0200 Subject: [PATCH 18/23] Added foundation model to Model table --- backend/core/models.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/core/models.py b/backend/core/models.py index 77dc577c..20ac0776 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -47,6 +47,11 @@ class Label(models.Model): class Model(models.Model): + FOUNDATION_MODEL_CHOICES = ( + ("RAMP", "RAMP"), + ("YOLO", "YOLO"), + ) + class ModelStatus(models.IntegerChoices): ARCHIVED = 1 PUBLISHED = 0 @@ -59,7 +64,10 @@ class ModelStatus(models.IntegerChoices): description = models.TextField(max_length=500, null=True, blank=True) user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) published_training = models.PositiveIntegerField(null=True, blank=True) - status = models.IntegerField(default=-1, choices=ModelStatus.choices) # + status = models.IntegerField(default=-1, choices=ModelStatus.choices) + foundation_model = models.CharField( + choices=FOUNDATION_MODEL_CHOICES, default="RAMP", max_length=10 + ) class Training(models.Model): From 5c6d632982bb60870ca71ef1e03461955b8a2f38 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 15 Oct 2024 13:58:48 +0200 Subject: [PATCH 19/23] Fix : Change foudnation model to base model , Remove is_active from admin area --- backend/core/admin.py | 4 ++-- backend/core/models.py | 15 ++++++--------- backend/core/serializers.py | 1 - backend/core/views.py | 17 +++++++++++++---- backend/login/admin.py | 8 +++++--- backend/login/permissions.py | 9 ++++----- 6 files changed, 30 insertions(+), 24 deletions(-) diff --git a/backend/core/admin.py b/backend/core/admin.py index 9c0f3bfa..20f4947b 100644 --- a/backend/core/admin.py +++ b/backend/core/admin.py @@ -51,8 +51,8 @@ class FeedbackAdmin(geoadmin.OSMGeoAdmin): @admin.register(Banner) class BannerAdmin(admin.ModelAdmin): - list_display = ("message", "start_date", "end_date", "is_active", "is_displayable") - list_filter = ("is_active", "start_date", "end_date") + list_display = ("message", "start_date", "end_date", "is_displayable") + list_filter = ("start_date", "end_date") search_fields = ("message",) readonly_fields = ("is_displayable",) diff --git a/backend/core/models.py b/backend/core/models.py index 20ac0776..3964587d 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -47,7 +47,7 @@ class Label(models.Model): class Model(models.Model): - FOUNDATION_MODEL_CHOICES = ( + BASE_MODEL_CHOICES = ( ("RAMP", "RAMP"), ("YOLO", "YOLO"), ) @@ -65,8 +65,8 @@ class ModelStatus(models.IntegerChoices): user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) published_training = models.PositiveIntegerField(null=True, blank=True) status = models.IntegerField(default=-1, choices=ModelStatus.choices) - foundation_model = models.CharField( - choices=FOUNDATION_MODEL_CHOICES, default="RAMP", max_length=10 + base_model = models.CharField( + choices=BASE_MODEL_CHOICES, default="RAMP", max_length=10 ) @@ -159,17 +159,14 @@ class ApprovedPredictions(models.Model): class Banner(models.Model): - message = models.CharField(max_length=255) + message = models.TextField() start_date = models.DateTimeField(default=timezone.now) end_date = models.DateTimeField(null=True, blank=True) - is_active = models.BooleanField(default=True) def is_displayable(self): now = timezone.now() - return ( - self.is_active - and (self.start_date <= now) - and (self.end_date is None or self.end_date >= now) + return (self.start_date <= now) and ( + self.end_date is None or self.end_date >= now ) def __str__(self): diff --git a/backend/core/serializers.py b/backend/core/serializers.py index ec2a22d8..bffdc1d9 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -403,7 +403,6 @@ class Meta: "message", "start_date", "end_date", - "is_active", "is_displayable", ] read_only_fields = ["is_displayable"] diff --git a/backend/core/views.py b/backend/core/views.py index 8722ca3f..6ce1960f 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -724,16 +724,24 @@ def post(self, request, *args, **kwargs): def publish_training(request, training_id: int): """Publishes training for model""" training_instance = get_object_or_404(Training, id=training_id) + if training_instance.status != "FINISHED": return Response("Training is not FINISHED", status=404) if training_instance.accuracy < 70: return Response( - "Can't publish the training since it's accuracy is below 70 %", status=404 + "Can't publish the training since its accuracy is below 70%", status=404 ) + model_instance = get_object_or_404(Model, id=training_instance.model.id) + + # Check if the current user is the owner of the model + if model_instance.user != request.user: + return Response("You are not allowed to publish this training", status=403) + model_instance.published_training = training_instance.id model_instance.status = 0 model_instance.save() + return Response("Training Published", status=status.HTTP_201_CREATED) @@ -800,8 +808,8 @@ def get(self, request, lookup_dir=None): class TrainingWorkspaceDownloadView(APIView): - # authentication_classes = [OsmAuthentication] - # permission_classes = [IsOsmAuthenticated] + authentication_classes = [OsmAuthentication] + permission_classes = [IsOsmAuthenticated] def get(self, request, lookup_dir): base_dir = os.path.join(settings.TRAINING_WORKSPACE, lookup_dir) @@ -847,7 +855,8 @@ class BannerViewSet(viewsets.ModelViewSet): queryset = Banner.objects.all() serializer_class = BannerSerializer authentication_classes = [OsmAuthentication] - permission_classes = [IsStaffUser, IsAdminUser] + permission_classes = [IsAdminUser, IsStaffUser] + public_methods = ["GET"] def get_queryset(self): now = timezone.now() diff --git a/backend/login/admin.py b/backend/login/admin.py index 72147c90..566991b1 100644 --- a/backend/login/admin.py +++ b/backend/login/admin.py @@ -86,6 +86,8 @@ class OsmUserAdmin(admin.ModelAdmin): }, ), ) - formfield_overrides = { - models.CharField: {"validators": []}, - } + + def formfield_for_dbfield(self, db_field, request, **kwargs): + if db_field.name == "username": + kwargs["validators"] = [] ## override the validation for sername + return super().formfield_for_dbfield(db_field, request, **kwargs) diff --git a/backend/login/permissions.py b/backend/login/permissions.py index bc76ebad..1ff82374 100644 --- a/backend/login/permissions.py +++ b/backend/login/permissions.py @@ -10,9 +10,9 @@ def has_permission(self, request, view): public_methods = getattr(view, "public_methods", []) if request.method in public_methods: return True - # If the user is authenticated, allow access + if request.user and request.user.is_authenticated: - # Global access for staff and admin users + # Global access if request.user.is_staff or request.user.is_superuser: return True @@ -21,7 +21,7 @@ def has_permission(self, request, view): return False def has_object_permission(self, request, view, obj): - # Allow read-only access for any authenticated user + if request.method in permissions.SAFE_METHODS: return True @@ -29,8 +29,7 @@ def has_object_permission(self, request, view, obj): if request.user.is_staff or request.user.is_superuser: return True - # Check if the object has a 'creator' field and if the user is the creator - if hasattr(obj, "creator") and obj.creator == request.user: + if hasattr(obj, "user") and obj.user == request.user: return True return False From da9eddfb4727518b5be9482c642eb3e1b274f10e Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 15 Oct 2024 14:20:13 +0200 Subject: [PATCH 20/23] Add public methods in admin and staff permission , Feat : Add KPI stats --- backend/core/urls.py | 2 ++ backend/core/views.py | 21 +++++++++++++++++++-- backend/login/permissions.py | 6 ++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/backend/core/urls.py b/backend/core/urls.py index dd8bd48d..9a236206 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -27,6 +27,7 @@ UsersView, download_training_data, geojson2osmconverter, + get_kpi_stats, publish_training, run_task_status, ) @@ -73,6 +74,7 @@ "workspace/download//", TrainingWorkspaceDownloadView.as_view() ), path("workspace//", TrainingWorkspaceView.as_view()), + path("kpi/stats/", get_kpi_stats, name="get_kpi_stats"), ] if settings.ENABLE_PREDICTION_API: urlpatterns.append(path("prediction/", PredictionView.as_view())) diff --git a/backend/core/views.py b/backend/core/views.py index 6ce1960f..54478cb3 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -860,6 +860,23 @@ class BannerViewSet(viewsets.ModelViewSet): def get_queryset(self): now = timezone.now() - return Banner.objects.filter(is_active=True, start_date__lte=now).filter( + return Banner.objects.filter(start_date__lte=now).filter( end_date__gte=now - ) | Banner.objects.filter(is_active=True, end_date__isnull=True) + ) | Banner.objects.filter(end_date__isnull=True) + + +@api_view(["GET"]) +def get_kpi_stats(request): + total_models_with_status_published = Model.objects.filter(status=0).count() + total_registered_users = OsmUser.objects.count() + total_approved_predictions = ApprovedPredictions.objects.count() + total_feedback_labels = FeedbackLabel.objects.count() + + data = { + "total_models_published": total_models_with_status_published, + "total_registered_users": total_registered_users, + "total_accepted_predictions": total_approved_predictions, + "total_feedback_labels": total_feedback_labels, + } + + return Response(data) diff --git a/backend/login/permissions.py b/backend/login/permissions.py index 1ff82374..58848120 100644 --- a/backend/login/permissions.py +++ b/backend/login/permissions.py @@ -37,6 +37,9 @@ def has_object_permission(self, request, view, obj): class IsAdminUser(permissions.BasePermission): def has_permission(self, request, view): + public_methods = getattr(view, "public_methods", []) + if request.method in public_methods: + return True return ( request.user and request.user.is_authenticated and request.user.is_superuser ) @@ -44,4 +47,7 @@ def has_permission(self, request, view): class IsStaffUser(permissions.BasePermission): def has_permission(self, request, view): + public_methods = getattr(view, "public_methods", []) + if request.method in public_methods: + return True return request.user and request.user.is_authenticated and request.user.is_staff From f10382eca2988e0a31845507096b9525652c603b Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 15 Oct 2024 14:38:42 +0200 Subject: [PATCH 21/23] FIX : Remove is_displayable to API --- backend/core/serializers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index bffdc1d9..6896dbe0 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -403,6 +403,4 @@ class Meta: "message", "start_date", "end_date", - "is_displayable", ] - read_only_fields = ["is_displayable"] From a9f14dec66457bf75e90f0168a7e931ec64dd7f3 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 15 Oct 2024 15:08:35 +0200 Subject: [PATCH 22/23] Add permission for user to be able to submit the training request but not modify / delete it --- backend/login/permissions.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/login/permissions.py b/backend/login/permissions.py index 58848120..b40c1390 100644 --- a/backend/login/permissions.py +++ b/backend/login/permissions.py @@ -28,10 +28,15 @@ def has_object_permission(self, request, view, obj): # Allow modification (PUT, DELETE) if the user is staff or admin if request.user.is_staff or request.user.is_superuser: return True - - if hasattr(obj, "user") and obj.user == request.user: - return True - + ## if the object it is trying to access has user info + if hasattr(obj, "user"): + # in order to change it it needs to be in his/her name + if obj.user == request.user: + return True + else: + if request.method == "POST": + # if object doesn't have user in it then he has permission to access the object , considered as common object + return True return False From 40b8060517087af0d775c0f5e19c5c4b630c02ba Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 15 Oct 2024 18:40:10 +0200 Subject: [PATCH 23/23] Enhance : Cache on the endpoint for key api stats --- backend/core/views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/core/views.py b/backend/core/views.py index 54478cb3..8c02394a 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -24,6 +24,9 @@ ) from django.shortcuts import get_object_or_404, redirect from django.utils import timezone +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page +from django.views.decorators.vary import vary_on_cookie, vary_on_headers from django_filters.rest_framework import DjangoFilterBackend from drf_yasg.utils import swagger_auto_schema from geojson2osm import geojson2osm @@ -865,6 +868,8 @@ def get_queryset(self): ) | Banner.objects.filter(end_date__isnull=True) +@cache_page(60 * 15) ## Cache for 15 mins +# @vary_on_cookie , if you wanna do user specific cache @api_view(["GET"]) def get_kpi_stats(request): total_models_with_status_published = Model.objects.filter(status=0).count()