From 59a5e1e2da9154fd50389592f1fa3457fb2e56ce Mon Sep 17 00:00:00 2001 From: Ronan Date: Tue, 17 Sep 2024 15:41:04 +0200 Subject: [PATCH] feat: add custom logger in order to save in file --- pyroengine/__init__.py | 1 + pyroengine/core.py | 17 +++++++---------- pyroengine/engine.py | 23 ++++++++++------------- pyroengine/logger_config.py | 26 ++++++++++++++++++++++++++ pyroengine/sensors.py | 15 +++++++-------- pyroengine/vision.py | 17 +++++++---------- src/run.py | 3 --- 7 files changed, 58 insertions(+), 44 deletions(-) create mode 100644 pyroengine/logger_config.py diff --git a/pyroengine/__init__.py b/pyroengine/__init__.py index 56f77dcd..847e4b6f 100644 --- a/pyroengine/__init__.py +++ b/pyroengine/__init__.py @@ -1,3 +1,4 @@ +from .logger_config import logger # Ensure logger is initialized first from .core import * from . import engine, sensors, utils from .version import __version__ diff --git a/pyroengine/core.py b/pyroengine/core.py index a41fec7c..b853f2b6 100644 --- a/pyroengine/core.py +++ b/pyroengine/core.py @@ -4,7 +4,6 @@ # See LICENSE or go to for full license details. import asyncio -import logging import time from datetime import datetime from typing import Any, List @@ -13,15 +12,13 @@ import urllib3 from .engine import Engine +from .logger_config import logger from .sensors import ReolinkCamera __all__ = ["SystemController", "is_day_time"] urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -# Configure logging -logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True) - def is_day_time(cache, frame, strategy, delta=0): """ @@ -84,7 +81,7 @@ async def capture_camera_image(camera: ReolinkCamera, image_queue: asyncio.Queue await image_queue.put((cam_id, frame)) await asyncio.sleep(0) # Yield control except Exception as e: - logging.exception(f"Error during image capture from camera {cam_id}: {e}") + logger.exception(f"Error during image capture from camera {cam_id}: {e}") class SystemController: @@ -133,7 +130,7 @@ async def analyze_stream(self, image_queue: asyncio.Queue) -> None: try: self.engine.predict(frame, cam_id) except Exception as e: - logging.error(f"Error running prediction: {e}") + logger.error(f"Error running prediction: {e}") finally: image_queue.task_done() # Mark the task as done @@ -146,7 +143,7 @@ def check_day_time(self) -> None: if frame is not None: self.day_time = is_day_time(None, frame, "ir") except Exception as e: - logging.exception(f"Exception during initial day time check: {e}") + logger.exception(f"Exception during initial day time check: {e}") async def run(self, period: int = 30, send_alerts: bool = True) -> None: """ @@ -181,10 +178,10 @@ async def run(self, period: int = 30, send_alerts: bool = True) -> None: if send_alerts: self.engine._process_alerts(self.cameras) except Exception as e: - logging.exception(f"Error processing alerts: {e}") + logger.exception(f"Error processing alerts: {e}") except Exception as e: - logging.warning(f"Analyze stream error: {e}") + logger.warning(f"Analyze stream error: {e}") async def main_loop(self, period: int, send_alerts: bool = True) -> None: """ @@ -200,7 +197,7 @@ async def main_loop(self, period: int, send_alerts: bool = True) -> None: # Sleep only once all images are processed loop_time = time.time() - start_ts sleep_time = max(period - (loop_time), 0) - logging.info(f"Loop run under {loop_time:.2f} seconds, sleeping for {sleep_time:.2f}") + logger.info(f"Loop run under {loop_time:.2f} seconds, sleeping for {sleep_time:.2f}") await asyncio.sleep(sleep_time) def __repr__(self) -> str: diff --git a/pyroengine/engine.py b/pyroengine/engine.py index b396db3e..34d4fd0e 100644 --- a/pyroengine/engine.py +++ b/pyroengine/engine.py @@ -6,7 +6,6 @@ import glob import io import json -import logging import os import shutil import time @@ -23,13 +22,12 @@ from pyroengine.utils import box_iou, nms +from .logger_config import logger from .sensors import ReolinkCamera from .vision import Classifier __all__ = ["Engine"] -logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True) - class Engine: """This implements an object to manage predictions and API interactions for wildfire alerts. @@ -80,7 +78,6 @@ def __init__( ) -> None: """Init engine""" # Engine Setup - self.model = Classifier(model_path=model_path) self.conf_thresh = conf_thresh @@ -259,7 +256,7 @@ def predict(self, frame: Image.Image, cam_id: Optional[str] = None, pose_id: Opt try: self.heartbeat(cam_id) except ConnectionError: - logging.exception(f"Unable to reach the pyro-api with {cam_id}") + logger.exception(f"Unable to reach the pyro-api with {cam_id}") cam_key = cam_id or "-1" # Reduce image size to save bandwidth @@ -276,7 +273,7 @@ def predict(self, frame: Image.Image, cam_id: Optional[str] = None, pose_id: Opt # Log analysis result device_str = f"Camera '{cam_id}' - " if isinstance(cam_id, str) else "" pred_str = "Wildfire detected" if conf > self.conf_thresh else "No wildfire" - logging.info(f"{device_str}{pred_str} (confidence: {conf:.2%})") + logger.info(f"{device_str}{pred_str} (confidence: {conf:.2%})") # Alert if conf > self.conf_thresh and len(self.api_client) > 0 and isinstance(cam_id, str): @@ -315,7 +312,7 @@ def _process_alerts(self, cameras: List[ReolinkCamera]) -> None: frame_info = self._alerts[0] cam_id = frame_info["cam_id"] pose_id = frame_info["pose_id"] - logging.info(f"Camera '{cam_id}' - Process detection from {frame_info['ts']}...") + logger.info(f"Camera '{cam_id}' - Process detection from {frame_info['ts']}...") # Save alert on device self._local_backup(frame_info["frame"], cam_id, pose_id) @@ -328,7 +325,7 @@ def _process_alerts(self, cameras: List[ReolinkCamera]) -> None: if camera.ip_address == cam_id: azimuth = camera.cam_azimuths[pose_id - 1] if pose_id is not None else camera.cam_azimuths[0] bboxes = self._alerts[0]["bboxes"] - logging.info(f"Azimuth : {azimuth} , bboxes : {bboxes}") + logger.info(f"Azimuth : {azimuth} , bboxes : {bboxes}") if len(bboxes) != 0: response = self.api_client[cam_id].create_detection(stream.getvalue(), azimuth, bboxes) # Force a KeyError if the request failed @@ -337,18 +334,18 @@ def _process_alerts(self, cameras: List[ReolinkCamera]) -> None: print(response.json()) raise KeyError(f"Missing 'id' in response from camera '{cam_id}'") # Clear else: - logging.info(f"Camera '{cam_id}' - detection created") + logger.info(f"Camera '{cam_id}' - detection created") break self._alerts.popleft() stream.seek(0) # "Rewind" the stream to the beginning so we can read its content except (KeyError, ConnectionError) as e: - logging.exception(f"Camera '{cam_id}' - unable to upload cache") - logging.exception(e) + logger.exception(f"Camera '{cam_id}' - unable to upload cache") + logger.exception(e) break except Exception as e: - logging.exception(f"Camera '{cam_id}' - unable to create detection") - logging.exception(e) + logger.exception(f"Camera '{cam_id}' - unable to create detection") + logger.exception(e) break def _local_backup( diff --git a/pyroengine/logger_config.py b/pyroengine/logger_config.py new file mode 100644 index 00000000..c42a4f9d --- /dev/null +++ b/pyroengine/logger_config.py @@ -0,0 +1,26 @@ +import logging +import os +from logging.handlers import TimedRotatingFileHandler + +# Define the logging format +log_format = "%(asctime)s | %(levelname)s: %(message)s" +os.makedirs(os.path.dirname("/var/log/engine.log"), exist_ok=True) + +# Create a StreamHandler (for stdout) +stream_handler = logging.StreamHandler() +stream_handler.setFormatter(logging.Formatter(log_format)) + +# Create a TimedRotatingFileHandler (for writing logs to a new file every day) +file_handler = TimedRotatingFileHandler("/var/log/engine.log", when="midnight", interval=1, backupCount=5) +file_handler.setFormatter(logging.Formatter(log_format)) +file_handler.suffix = "%Y-%m-%d" # Adds a date suffix to the log file name + +# Export the logger instance +logger = logging.getLogger() # Get the root logger + +# Ensure we only add handlers once +logger.addHandler(stream_handler) +logger.addHandler(file_handler) + +logger.setLevel(logging.INFO) +logger.info("Logger is set up correctly.") diff --git a/pyroengine/sensors.py b/pyroengine/sensors.py index 9af14752..04d70399 100644 --- a/pyroengine/sensors.py +++ b/pyroengine/sensors.py @@ -3,7 +3,6 @@ # This program is licensed under the Apache License 2.0. # See LICENSE or go to for full license details. -import logging import time from io import BytesIO from typing import List, Optional @@ -16,7 +15,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # Configure logging -logging.basicConfig(level=logging.DEBUG) +from .logger_config import logger class ReolinkCamera: @@ -73,12 +72,12 @@ def _handle_response(self, response, success_message: str): if response.status_code == 200: response_data = response.json() if response_data[0]["code"] == 0: - logging.debug(success_message) + logger.debug(success_message) else: - logging.error(f"Error in camera call: {response_data}") + logger.error(f"Error in camera call: {response_data}") return response_data else: - logging.error(f"Failed operation during camera control: {response.status_code}, {response.text}") + logger.error(f"Failed operation during camera control: {response.status_code}, {response.text}") return None def capture(self, pos_id: Optional[int] = None, timeout: int = 2) -> Optional[Image.Image]: @@ -96,7 +95,7 @@ def capture(self, pos_id: Optional[int] = None, timeout: int = 2) -> Optional[Im self.move_camera("ToPos", idx=int(pos_id), speed=50) time.sleep(1) url = self._build_url("Snap") - logging.debug("Start capture") + logger.debug("Start capture") try: response = requests.get(url, verify=False, timeout=timeout) # nosec: B501 @@ -105,9 +104,9 @@ def capture(self, pos_id: Optional[int] = None, timeout: int = 2) -> Optional[Im image = Image.open(image_data).convert("RGB") return image else: - logging.error(f"Failed to capture image: {response.status_code}, {response.text}") + logger.error(f"Failed to capture image: {response.status_code}, {response.text}") except requests.RequestException as e: - logging.error(f"Request failed: {e}") + logger.error(f"Request failed: {e}") return None def move_camera(self, operation: str, speed: int = 20, idx: int = 0): diff --git a/pyroengine/vision.py b/pyroengine/vision.py index e8f405c8..e9b75a17 100644 --- a/pyroengine/vision.py +++ b/pyroengine/vision.py @@ -4,7 +4,6 @@ # See LICENSE or go to for full license details. import json -import logging import os import platform import shutil @@ -16,6 +15,7 @@ from PIL import Image from ultralytics import YOLO # type: ignore[import-untyped] +from .logger_config import logger from .utils import DownloadProgressBar __all__ = ["Classifier"] @@ -26,9 +26,6 @@ METADATA_NAME = "model_metadata.json" -logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True) - - # Utility function to save metadata def save_metadata(metadata_path, metadata): with open(metadata_path, "w") as f: @@ -52,7 +49,7 @@ def __init__(self, model_folder="data", imgsz=1024, conf=0.15, iou=0.05, format= if self.is_arm_architecture(): model = "yolov8s_ncnn_model.zip" else: - logging.info("NCNN format is optimized for arm architecture only, switching to onnx") + logger.info("NCNN format is optimized for arm architecture only, switching to onnx") model = "yolov8s.onnx" elif format in ["onnx", "pt"]: model = f"yolov8s.{format}" @@ -74,9 +71,9 @@ def __init__(self, model_folder="data", imgsz=1024, conf=0.15, iou=0.05, format= # Load existing metadata metadata = self.load_metadata(metadata_path) if metadata and metadata.get("sha256") == expected_sha256: - logging.info("Model already exists and the SHA256 hash matches. No download needed.") + logger.info("Model already exists and the SHA256 hash matches. No download needed.") else: - logging.info("Model exists but the SHA256 hash does not match or the file doesn't exist.") + logger.info("Model exists but the SHA256 hash does not match or the file doesn't exist.") os.remove(model_path) self.download_model(model_url, model_path, expected_sha256, metadata_path) else: @@ -110,15 +107,15 @@ def download_model(self, model_url, model_path, expected_sha256, metadata_path): os.makedirs(os.path.split(model_path)[0], exist_ok=True) # Download the model - logging.info(f"Downloading model from {model_url} ...") + logger.info(f"Downloading model from {model_url} ...") with DownloadProgressBar(unit="B", unit_scale=True, miniters=1, desc=model_path) as t: urlretrieve(model_url, model_path, reporthook=t.update_to) - logging.info("Model downloaded!") + logger.info("Model downloaded!") # Save the metadata metadata = {"sha256": expected_sha256} save_metadata(metadata_path, metadata) - logging.info("Metadata saved!") + logger.info("Metadata saved!") # Utility function to load metadata def load_metadata(self, metadata_path): diff --git a/src/run.py b/src/run.py index d2a5aca6..08cf9e82 100644 --- a/src/run.py +++ b/src/run.py @@ -6,7 +6,6 @@ import argparse import asyncio import json -import logging import os import urllib3 @@ -18,8 +17,6 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True) - def main(args): print(args)