feat: add custom logger in order to save in file

pyronear · Sep 17, 2024 · 59a5e1e · 59a5e1e
1 parent 410e9e6
commit 59a5e1e
Show file tree

Hide file tree

Showing 7 changed files with 58 additions and 44 deletions.
diff --git a/pyroengine/__init__.py b/pyroengine/__init__.py
@@ -1,3 +1,4 @@
+from .logger_config import logger  # Ensure logger is initialized first
 from .core import *
 from . import engine, sensors, utils
 from .version import __version__
diff --git a/pyroengine/core.py b/pyroengine/core.py
@@ -4,7 +4,6 @@
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
 
 import asyncio
-import logging
 import time
 from datetime import datetime
 from typing import Any, List
@@ -13,15 +12,13 @@
 import urllib3
 
 from .engine import Engine
+from .logger_config import logger
 from .sensors import ReolinkCamera
 
 __all__ = ["SystemController", "is_day_time"]
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-# Configure logging
-logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True)
-
 
 def is_day_time(cache, frame, strategy, delta=0):
     """
@@ -84,7 +81,7 @@ async def capture_camera_image(camera: ReolinkCamera, image_queue: asyncio.Queue
                 await image_queue.put((cam_id, frame))
                 await asyncio.sleep(0)  # Yield control
     except Exception as e:
-        logging.exception(f"Error during image capture from camera {cam_id}: {e}")
+        logger.exception(f"Error during image capture from camera {cam_id}: {e}")
 
 
 class SystemController:
@@ -133,7 +130,7 @@ async def analyze_stream(self, image_queue: asyncio.Queue) -> None:
             try:
                 self.engine.predict(frame, cam_id)
             except Exception as e:
-                logging.error(f"Error running prediction: {e}")
+                logger.error(f"Error running prediction: {e}")
             finally:
                 image_queue.task_done()  # Mark the task as done
 
@@ -146,7 +143,7 @@ def check_day_time(self) -> None:
             if frame is not None:
                 self.day_time = is_day_time(None, frame, "ir")
         except Exception as e:
-            logging.exception(f"Exception during initial day time check: {e}")
+            logger.exception(f"Exception during initial day time check: {e}")
 
     async def run(self, period: int = 30, send_alerts: bool = True) -> None:
         """
@@ -181,10 +178,10 @@ async def run(self, period: int = 30, send_alerts: bool = True) -> None:
                     if send_alerts:
                         self.engine._process_alerts(self.cameras)
                 except Exception as e:
-                    logging.exception(f"Error processing alerts: {e}")
+                    logger.exception(f"Error processing alerts: {e}")
 
         except Exception as e:
-            logging.warning(f"Analyze stream error: {e}")
+            logger.warning(f"Analyze stream error: {e}")
 
     async def main_loop(self, period: int, send_alerts: bool = True) -> None:
         """
@@ -200,7 +197,7 @@ async def main_loop(self, period: int, send_alerts: bool = True) -> None:
             # Sleep only once all images are processed
             loop_time = time.time() - start_ts
             sleep_time = max(period - (loop_time), 0)
-            logging.info(f"Loop run under {loop_time:.2f} seconds, sleeping for {sleep_time:.2f}")
+            logger.info(f"Loop run under {loop_time:.2f} seconds, sleeping for {sleep_time:.2f}")
             await asyncio.sleep(sleep_time)
 
     def __repr__(self) -> str:

diff --git a/pyroengine/engine.py b/pyroengine/engine.py
@@ -6,7 +6,6 @@
 import glob
 import io
 import json
-import logging
 import os
 import shutil
 import time
@@ -23,13 +22,12 @@
 
 from pyroengine.utils import box_iou, nms
 
+from .logger_config import logger
 from .sensors import ReolinkCamera
 from .vision import Classifier
 
 __all__ = ["Engine"]
 
-logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True)
-
 
 class Engine:
     """This implements an object to manage predictions and API interactions for wildfire alerts.
@@ -80,7 +78,6 @@ def __init__(
     ) -> None:
         """Init engine"""
         # Engine Setup
-
         self.model = Classifier(model_path=model_path)
         self.conf_thresh = conf_thresh
 
@@ -259,7 +256,7 @@ def predict(self, frame: Image.Image, cam_id: Optional[str] = None, pose_id: Opt
             try:
                 self.heartbeat(cam_id)
             except ConnectionError:
-                logging.exception(f"Unable to reach the pyro-api with {cam_id}")
+                logger.exception(f"Unable to reach the pyro-api with {cam_id}")
 
         cam_key = cam_id or "-1"
         # Reduce image size to save bandwidth
@@ -276,7 +273,7 @@ def predict(self, frame: Image.Image, cam_id: Optional[str] = None, pose_id: Opt
         # Log analysis result
         device_str = f"Camera '{cam_id}' - " if isinstance(cam_id, str) else ""
         pred_str = "Wildfire detected" if conf > self.conf_thresh else "No wildfire"
-        logging.info(f"{device_str}{pred_str} (confidence: {conf:.2%})")
+        logger.info(f"{device_str}{pred_str} (confidence: {conf:.2%})")
 
         # Alert
         if conf > self.conf_thresh and len(self.api_client) > 0 and isinstance(cam_id, str):
@@ -315,7 +312,7 @@ def _process_alerts(self, cameras: List[ReolinkCamera]) -> None:
             frame_info = self._alerts[0]
             cam_id = frame_info["cam_id"]
             pose_id = frame_info["pose_id"]
-            logging.info(f"Camera '{cam_id}' - Process detection from {frame_info['ts']}...")
+            logger.info(f"Camera '{cam_id}' - Process detection from {frame_info['ts']}...")
 
             # Save alert on device
             self._local_backup(frame_info["frame"], cam_id, pose_id)
@@ -328,7 +325,7 @@ def _process_alerts(self, cameras: List[ReolinkCamera]) -> None:
                     if camera.ip_address == cam_id:
                         azimuth = camera.cam_azimuths[pose_id - 1] if pose_id is not None else camera.cam_azimuths[0]
                         bboxes = self._alerts[0]["bboxes"]
-                        logging.info(f"Azimuth : {azimuth} , bboxes : {bboxes}")
+                        logger.info(f"Azimuth : {azimuth} , bboxes : {bboxes}")
                         if len(bboxes) != 0:
                             response = self.api_client[cam_id].create_detection(stream.getvalue(), azimuth, bboxes)
                             # Force a KeyError if the request failed
@@ -337,18 +334,18 @@ def _process_alerts(self, cameras: List[ReolinkCamera]) -> None:
                                 print(response.json())
                                 raise KeyError(f"Missing 'id' in response from camera '{cam_id}'")  # Clear
                             else:
-                                logging.info(f"Camera '{cam_id}' - detection created")
+                                logger.info(f"Camera '{cam_id}' - detection created")
                         break
 
                 self._alerts.popleft()
                 stream.seek(0)  # "Rewind" the stream to the beginning so we can read its content
             except (KeyError, ConnectionError) as e:
-                logging.exception(f"Camera '{cam_id}' - unable to upload cache")
-                logging.exception(e)
+                logger.exception(f"Camera '{cam_id}' - unable to upload cache")
+                logger.exception(e)
                 break
             except Exception as e:
-                logging.exception(f"Camera '{cam_id}' - unable to create detection")
-                logging.exception(e)
+                logger.exception(f"Camera '{cam_id}' - unable to create detection")
+                logger.exception(e)
                 break
 
     def _local_backup(

diff --git a/pyroengine/logger_config.py b/pyroengine/logger_config.py
@@ -0,0 +1,26 @@
+import logging
+import os
+from logging.handlers import TimedRotatingFileHandler
+
+# Define the logging format
+log_format = "%(asctime)s | %(levelname)s: %(message)s"
+os.makedirs(os.path.dirname("/var/log/engine.log"), exist_ok=True)
+
+# Create a StreamHandler (for stdout)
+stream_handler = logging.StreamHandler()
+stream_handler.setFormatter(logging.Formatter(log_format))
+
+# Create a TimedRotatingFileHandler (for writing logs to a new file every day)
+file_handler = TimedRotatingFileHandler("/var/log/engine.log", when="midnight", interval=1, backupCount=5)
+file_handler.setFormatter(logging.Formatter(log_format))
+file_handler.suffix = "%Y-%m-%d"  # Adds a date suffix to the log file name
+
+# Export the logger instance
+logger = logging.getLogger()  # Get the root logger
+
+# Ensure we only add handlers once
+logger.addHandler(stream_handler)
+logger.addHandler(file_handler)
+
+logger.setLevel(logging.INFO)
+logger.info("Logger is set up correctly.")
diff --git a/pyroengine/sensors.py b/pyroengine/sensors.py
@@ -3,7 +3,6 @@
 # This program is licensed under the Apache License 2.0.
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
 
-import logging
 import time
 from io import BytesIO
 from typing import List, Optional
@@ -16,7 +15,7 @@
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 # Configure logging
-logging.basicConfig(level=logging.DEBUG)
+from .logger_config import logger
 
 
 class ReolinkCamera:
@@ -73,12 +72,12 @@ def _handle_response(self, response, success_message: str):
         if response.status_code == 200:
             response_data = response.json()
             if response_data[0]["code"] == 0:
-                logging.debug(success_message)
+                logger.debug(success_message)
             else:
-                logging.error(f"Error in camera call: {response_data}")
+                logger.error(f"Error in camera call: {response_data}")
             return response_data
         else:
-            logging.error(f"Failed operation during camera control: {response.status_code}, {response.text}")
+            logger.error(f"Failed operation during camera control: {response.status_code}, {response.text}")
             return None
 
     def capture(self, pos_id: Optional[int] = None, timeout: int = 2) -> Optional[Image.Image]:
@@ -96,7 +95,7 @@ def capture(self, pos_id: Optional[int] = None, timeout: int = 2) -> Optional[Im
             self.move_camera("ToPos", idx=int(pos_id), speed=50)
             time.sleep(1)
         url = self._build_url("Snap")
-        logging.debug("Start capture")
+        logger.debug("Start capture")
 
         try:
             response = requests.get(url, verify=False, timeout=timeout)  # nosec: B501
@@ -105,9 +104,9 @@ def capture(self, pos_id: Optional[int] = None, timeout: int = 2) -> Optional[Im
                 image = Image.open(image_data).convert("RGB")
                 return image
             else:
-                logging.error(f"Failed to capture image: {response.status_code}, {response.text}")
+                logger.error(f"Failed to capture image: {response.status_code}, {response.text}")
         except requests.RequestException as e:
-            logging.error(f"Request failed: {e}")
+            logger.error(f"Request failed: {e}")
         return None
 
     def move_camera(self, operation: str, speed: int = 20, idx: int = 0):

diff --git a/pyroengine/vision.py b/pyroengine/vision.py
@@ -4,7 +4,6 @@
 # See LICENSE or go to <https://opensource.org/licenses/Apache-2.0> for full license details.
 
 import json
-import logging
 import os
 import platform
 import shutil
@@ -16,6 +15,7 @@
 from PIL import Image
 from ultralytics import YOLO  # type: ignore[import-untyped]
 
+from .logger_config import logger
 from .utils import DownloadProgressBar
 
 __all__ = ["Classifier"]
@@ -26,9 +26,6 @@
 METADATA_NAME = "model_metadata.json"
 
 
-logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True)
-
-
 # Utility function to save metadata
 def save_metadata(metadata_path, metadata):
     with open(metadata_path, "w") as f:
@@ -52,7 +49,7 @@ def __init__(self, model_folder="data", imgsz=1024, conf=0.15, iou=0.05, format=
                 if self.is_arm_architecture():
                     model = "yolov8s_ncnn_model.zip"
                 else:
-                    logging.info("NCNN format is optimized for arm architecture only, switching to onnx")
+                    logger.info("NCNN format is optimized for arm architecture only, switching to onnx")
                     model = "yolov8s.onnx"
             elif format in ["onnx", "pt"]:
                 model = f"yolov8s.{format}"
@@ -74,9 +71,9 @@ def __init__(self, model_folder="data", imgsz=1024, conf=0.15, iou=0.05, format=
                 # Load existing metadata
                 metadata = self.load_metadata(metadata_path)
                 if metadata and metadata.get("sha256") == expected_sha256:
-                    logging.info("Model already exists and the SHA256 hash matches. No download needed.")
+                    logger.info("Model already exists and the SHA256 hash matches. No download needed.")
                 else:
-                    logging.info("Model exists but the SHA256 hash does not match or the file doesn't exist.")
+                    logger.info("Model exists but the SHA256 hash does not match or the file doesn't exist.")
                     os.remove(model_path)
                     self.download_model(model_url, model_path, expected_sha256, metadata_path)
             else:
@@ -110,15 +107,15 @@ def download_model(self, model_url, model_path, expected_sha256, metadata_path):
         os.makedirs(os.path.split(model_path)[0], exist_ok=True)
 
         # Download the model
-        logging.info(f"Downloading model from {model_url} ...")
+        logger.info(f"Downloading model from {model_url} ...")
         with DownloadProgressBar(unit="B", unit_scale=True, miniters=1, desc=model_path) as t:
             urlretrieve(model_url, model_path, reporthook=t.update_to)
-        logging.info("Model downloaded!")
+        logger.info("Model downloaded!")
 
         # Save the metadata
         metadata = {"sha256": expected_sha256}
         save_metadata(metadata_path, metadata)
-        logging.info("Metadata saved!")
+        logger.info("Metadata saved!")
 
     # Utility function to load metadata
     def load_metadata(self, metadata_path):

diff --git a/src/run.py b/src/run.py
@@ -6,7 +6,6 @@
 import argparse
 import asyncio
 import json
-import logging
 import os
 
 import urllib3
@@ -18,8 +17,6 @@
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
-logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True)
-
 
 def main(args):
     print(args)