Merge pull request hotosm#250 from hotosm/feature/api-worker-sep

Feature : Seperation of API and workers

backend/README.md

@@ -14,33 +14,27 @@ This project was bootstrapped with  [Geodjango Template](https://github.com/itsk
     source ./env/bin/activate
 
 ##### Setup Basemodels (Ramp Supported Currently)
+- Install git lfs
+```bash
+sudo apt-get install git-lfs
+```
+
 - Clone Ramp Basemodel 
 ```
 git clone https://github.com/radiantearth/model_ramp_baseline.git
 ```
-OR Download from google drive 
-```
-pip install gdown
-gdown --fuzzy https://drive.google.com/file/d/1wvJhkiOrSlHmmvJ0avkAdu9sslFf5_I0/view?usp=sharing
-```
 
 - Clone Ramp - Code 
 Note: This clone location will be your RAMP_HOME 
 ```
 git clone https://github.com/kshitijrajsharma/ramp-code-fAIr.git ramp-code
 ```
+
 - Copy Basemodel checkpoint to ramp-code
 ```
 cp -r model_ramp_baseline/data/input/checkpoint.tf ramp-code/ramp/checkpoint.tf
 ```
 
-Our Basemodel is available for public download [here](https://drive.google.com/file/d/1wvJhkiOrSlHmmvJ0avkAdu9sslFf5_I0/view?usp=sharing)
-
-You can unzip and  move the downloaded basemodel 
-```
-unzip checkpoint.tf.zip -d ramp-code/ramp  
-```
-
 
 - Remove basemodel repo we don't need it anymore 
 ```
@@ -136,11 +130,7 @@ pip install -r requirements.txt
     You will need more env variables (Such as Ramp home, Training Home) that can be found on ```.sample_env```  
 
 #### Now change your username, password and db name in settings.py accordingly to your database
-    python manage.py makemigrations login
-    python manage.py migrate login
-    python manage.py makemigrations core
-    python manage.py migrate core 
-    python manage.py makemigrations 
+    python manage.py makemigrations login core
     python manage.py migrate
     python manage.py runserver
 ### Now server will be available in your 8000 port on web, you can check out your localhost:8000/admin for admin panel 

backend/api-requirements.txt

@@ -0,0 +1,23 @@
+django==4.1.4
+# gdal==3.6.2
+psycopg2
+djangorestframework==3.14.0
+djangorestframework-gis==1.0
+dj-database-url==1.2.0
+django-leaflet==0.28.3
+drf-yasg==1.21.4
+django-environ==0.9.0 # used for environment
+django-filter==22.1
+django-cors-headers==3.13.0 # used for enabling cors when frontend is hosted on different server / origin
+osm-login-python==0.0.2
+celery==5.2.7
+redis==4.4.0
+django_celery_results==2.4.0
+flower==1.2.0
+validators==0.20.0
+gpxpy==1.5.0
+geojson2osm==0.0.1
+osmconflator==0.0.9
+orthogonalizer==0.0.4
+fairpredictor==0.0.26
+tflite-runtime==2.14.0

backend/core/urls.py

@@ -5,7 +5,7 @@
 # now import the views.py file into this code
 from .views import (
     AOIViewSet,
-    APIStatus,
+    # APIStatus,
     ConflateGeojson,
     DatasetViewSet,
     FeedbackAOIViewset,
@@ -52,7 +52,7 @@
     path("training/publish/<int:training_id>/", publish_training),
     path("prediction/", PredictionView.as_view()),
     path("feedback/training/submit/", FeedbackView.as_view()),
-    path("status/", APIStatus.as_view()),
+    # path("status/", APIStatus.as_view()),
     path("geojson2osm/", geojson2osmconverter, name="geojson2osmconverter"),
     path("conflate/", ConflateGeojson, name="Conflate Geojson"),
     path("aoi/gpx/<int:aoi_id>/", GenerateGpxView.as_view()),

backend/core/views.py

@@ -12,7 +12,7 @@
 from datetime import datetime
 from tempfile import NamedTemporaryFile
 
-import tensorflow as tf
+# import tensorflow as tf
 from celery import current_app
 from celery.result import AsyncResult
 from django.conf import settings
@@ -60,7 +60,8 @@
     ModelSerializer,
     PredictionParamSerializer,
 )
-from .tasks import train_model
+# from .tasks import train_model
+from celery import Celery
 from .utils import get_dir_size, gpx_generator, process_rawdata, request_rawdata
 
 
@@ -127,9 +128,11 @@ def create(self, validated_data):
         validated_data["created_by"] = user
         # create the model instance
         instance = Training.objects.create(**validated_data)
-        logging.info("Sending record to redis queue")
+
+        celery = Celery()
+
         # run your function here
-        task = train_model.delay(
+        task = celery.train_model.delay(
             dataset_id=instance.model.dataset.id,
             training_id=instance.id,
             epochs=instance.epochs,
@@ -471,8 +474,9 @@ def post(self, request, *args, **kwargs):
                 batch_size=batch_size,
                 source_imagery=training_instance.source_imagery,
             )
+            celery = Celery()
 
-            task = train_model.delay(
+            task = celery.train_model.delay(
                 dataset_id=instance.model.dataset.id,
                 training_id=instance.id,
                 epochs=instance.epochs,
@@ -614,16 +618,16 @@ def publish_training(request, training_id: int):
     return Response("Training Published", status=status.HTTP_201_CREATED)
 
 
-class APIStatus(APIView):
-    def get(self, request):
-        res = {
-            "tensorflow_version": tf.__version__,
-            "No of GPU Available": len(
-                tf.config.experimental.list_physical_devices("GPU")
-            ),
-            "API Status": "Healthy",  # static for now should be dynamic TODO
-        }
-        return Response(res, status=status.HTTP_200_OK)
+# class APIStatus(APIView):
+#     def get(self, request):
+#         res = {
+#             "tensorflow_version": tf.__version__,
+#             "No of GPU Available": len(
+#                 tf.config.experimental.list_physical_devices("GPU")
+#             ),
+#             "API Status": "Healthy",  # static for now should be dynamic TODO
+#         }
+#         return Response(res, status=status.HTTP_200_OK)
 
 
 class GenerateGpxView(APIView):

backend/requirements.txt

@@ -1,25 +1,2 @@
-django==4.1.4
-# gdal
-psycopg2
-djangorestframework==3.14.0
-djangorestframework-gis==1.0
-dj-database-url==1.2.0
-django-leaflet==0.28.3
-drf-yasg==1.21.4
-Pillow
-django-environ==0.9.0 # used for environment
-django-filter==22.1
-django-cors-headers==3.13.0 # used for enabling cors when frontend is hosted on different server / origin
-osm-login-python==0.0.2
-celery==5.2.7
-redis==4.4.0
-django_celery_results==2.4.0
-flower==1.2.0
-validators==0.20.0
-gpxpy==1.5.0
+-r api-requirements.txt
 hot-fair-utilities==1.2.3
-geojson2osm==0.0.1
-osmconflator
-orthogonalizer
-fairpredictor==0.0.26
-tflite-runtime==2.14.0

backend/sample_env

@@ -1,3 +1,4 @@
+DEBUG=True
 SECRET_KEY=yl2w)c0boi_ma-1v5)935^2#&m*r!1s9z9^*9e5co^08_ixzo6
 DATABASE_URL=postgis://admin:password@localhost:5432/ai
 EXPORT_TOOL_API_URL=MY_RAW_DATA_URL