Merge pull request #52 from thiagosalles/MES-1603-HealthcheckMonitore…

…dModules

MES-1603 - Healthcheck Monitored Modules

.tool-versions

@@ -0,0 +1 @@
+python 3.7.13

README.md

@@ -167,6 +167,52 @@ async def consumer_access_storage(msg):
     data = baterdude["my_variable"]
 ```
 
+### Monitoring extra modules on Healthcheck
+
+If you run extra modules on your application, like workers or services, you can include them in the healthcheck.
+
+First, you need to update your module to implement the interface `HealthcheckMonitored`:
+```python
+from barterdude.hooks.healthcheck import HealthcheckMonitored
+
+class ExtraService(HealthcheckMonitored):
+```
+
+Implementing that interface will require the definition of the method `healthcheck` in your module. It should return a boolean value indicating if your module is healhty or not:
+```python
+    def healthcheck(self):
+        return self._thread.is_alive()
+```
+
+Finally, you need to make the BarterDude and Healthcheck module be aware of your module. To do so, you'll use the Data Sharing feature:
+```python
+from barterdude import BarterDude
+from app.extra_service import ExtraService
+
+barterdude = BarterDude()
+barterdude["extra_service"] = ExtraService()
+```
+
+If you are already running your extra modules on BartedDude startup using the data sharing model, it's all done:
+```python
+@app.run_on_startup
+async def startup(app):
+    app["client_session"] = ClientSession()
+    app["extra_service"] = ExtraService()
+```
+
+The healthcheck module will identify all shared modules that implement the interface `HealthcheckMonitored` and run its healthcheck method automatically.
+The result of all monitored modules will be included in the result body of the healthcheck endpoint and if any of the modules fail, the healthcheck endpoint will indicate that:
+```json
+{
+    "extra_service": "ok",
+    "message": "Success rate: 1.0 (expected: 0.9)",
+    "fail": 0,
+    "success": 1,
+    "status": "ok"
+}
+```
+
 ### Schema Validation
 
 Consumed messages can be validated by json schema:

barterdude/hooks/healthcheck.py

@@ -1,3 +1,4 @@
+from abc import ABC, abstractmethod
 import json
 
 from barterdude import BarterDude
@@ -21,6 +22,12 @@ def _response(status, body):
     return web.Response(status=status, body=json.dumps(body))
 
 
+class HealthcheckMonitored(ABC):
+    @abstractmethod
+    def healthcheck(self):
+        pass
+
+
 class Healthcheck(HttpHook):
     def __init__(
         self,
@@ -30,6 +37,7 @@ def __init__(
         health_window: float = 60.0,  # seconds
         max_connection_fails: int = 3
     ):
+        self.__barterdude = barterdude
         self.__success_rate = success_rate
         self.__health_window = health_window
         self.__success = deque()
@@ -68,18 +76,39 @@ async def __call__(self, req: web.Request):
                 )
             })
 
+        response = {}
+        status = 200
+
+        all_monitored_modules_passed = True
+        for module in self.__barterdude:
+            if isinstance(self.__barterdude[module], HealthcheckMonitored):
+                passed = self.__barterdude[module].healthcheck()
+                response[module] = "ok" if passed else "fail"
+                all_monitored_modules_passed &= passed
+
+        if not all_monitored_modules_passed:
+            status = 500
+
         old_timestamp = time() - self.__health_window
         success = _remove_old(self.__success, old_timestamp)
         fail = _remove_old(self.__fail, old_timestamp)
+
         if success == 0 and fail == 0:
-            return _response(200, {
-                "message": f"No messages in last {self.__health_window}s"
-            })
+            response["message"] = (
+                f"No messages in last {self.__health_window}s"
+            )
+            return _response(status, response)
 
         rate = success / (success + fail)
-        return _response(200 if rate >= self.__success_rate else 500, {
+
+        if rate < self.__success_rate:
+            status = 500
+
+        response.update({
             "message":
                 f"Success rate: {rate} (expected: {self.__success_rate})",
             "fail": fail,
             "success": success
         })
+
+        return _response(status, response)

tests_unit/test_hooks/test_healthcheck.py

@@ -1,6 +1,14 @@
-from asynctest import TestCase, Mock
+from asynctest import TestCase, MagicMock
 from freezegun import freeze_time
-from barterdude.hooks.healthcheck import Healthcheck
+from barterdude.hooks.healthcheck import Healthcheck, HealthcheckMonitored
+
+
+class HealthcheckMonitoredMock(HealthcheckMonitored):
+    def __init__(self, healthy=True):
+        self.healthy = healthy
+
+    def healthcheck(self):
+        return self.healthy
 
 
 @freeze_time()
@@ -10,8 +18,14 @@ class TestHealthcheck(TestCase):
     def setUp(self):
         self.success_rate = 0.9
         self.health_window = 60.0
+        self.app = MagicMock()
+        self.monitoredModules = {}
+        self.app.__iter__.side_effect = lambda: iter(self.monitoredModules)
+        self.app.__getitem__.side_effect = (
+            lambda module: self.monitoredModules[module]
+        )
         self.healthcheck = Healthcheck(
-            Mock(),
+            self.app,
             "/healthcheck",
             self.success_rate,
             self.health_window
@@ -21,7 +35,7 @@ async def test_should_call_before_consume(self):
         await self.healthcheck.before_consume(None)
 
     async def test_should_pass_healthcheck_when_no_messages(self):
-        response = await self.healthcheck(Mock())
+        response = await self.healthcheck(self.app)
         self.assertEqual(response.status, 200)
         self.assertEqual(response.content_type, "text/plain")
         self.assertEqual(
@@ -31,7 +45,7 @@ async def test_should_pass_healthcheck_when_no_messages(self):
 
     async def test_should_pass_healthcheck_when_only_sucess(self):
         await self.healthcheck.on_success(None)
-        response = await self.healthcheck(Mock())
+        response = await self.healthcheck(self.app)
         self.assertEqual(response.status, 200)
         self.assertEqual(response.content_type, "text/plain")
         self.assertEqual(
@@ -44,7 +58,7 @@ async def test_should_pass_healthcheck_when_success_rate_is_high(self):
         await self.healthcheck.on_fail(None, None)
         for i in range(0, 9):
             await self.healthcheck.on_success(None)
-        response = await self.healthcheck(Mock())
+        response = await self.healthcheck(self.app)
         self.assertEqual(response.status, 200)
         self.assertEqual(response.content_type, "text/plain")
         self.assertEqual(
@@ -55,7 +69,7 @@ async def test_should_pass_healthcheck_when_success_rate_is_high(self):
 
     async def test_should_fail_healthcheck_when_only_fail(self):
         await self.healthcheck.on_fail(None, None)
-        response = await self.healthcheck(Mock())
+        response = await self.healthcheck(self.app)
         self.assertEqual(response.status, 500)
         self.assertEqual(response.content_type, "text/plain")
         self.assertEqual(
@@ -67,7 +81,7 @@ async def test_should_fail_healthcheck_when_only_fail(self):
     async def test_should_fail_healthcheck_when_success_rate_is_low(self):
         await self.healthcheck.on_success(None)
         await self.healthcheck.on_fail(None, None)
-        response = await self.healthcheck(Mock())
+        response = await self.healthcheck(self.app)
         self.assertEqual(response.status, 500)
         self.assertEqual(response.content_type, "text/plain")
         self.assertEqual(
@@ -79,7 +93,7 @@ async def test_should_fail_healthcheck_when_success_rate_is_low(self):
     async def test_should_fail_when_force_fail_is_called(self):
         self.healthcheck.force_fail()
         await self.healthcheck.on_success(None)
-        response = await self.healthcheck(Mock())
+        response = await self.healthcheck(self.app)
         self.assertEqual(response.status, 500)
         self.assertEqual(response.content_type, "text/plain")
         self.assertEqual(
@@ -93,7 +107,7 @@ async def test_should_erase_old_messages(self):
             for i in range(0, 10):
                 await self.healthcheck.on_fail(None, None)
             await self.healthcheck.on_success(None)
-            response = await self.healthcheck(Mock())
+            response = await self.healthcheck(self.app)
             self.assertEqual(
                 response.body._value.decode('utf-8'),
                 '{"message": "Success rate: 0.125 (expected: 0.9)", '
@@ -102,8 +116,120 @@ async def test_should_erase_old_messages(self):
 
     async def test_should_fail_healthcheck_when_fail_to_connect(self):
         await self.healthcheck.on_connection_fail(None, 3)
-        response = await self.healthcheck(Mock())
+        response = await self.healthcheck(self.app)
         self.assertEqual(
             response.body._value.decode('utf-8'),
             '{"message": "Reached max connection fails (3)", "status": "fail"}'
         )
+
+    async def test_should_pass_when_has_one_healthy_monitored_module(self):
+        self.monitoredModules = {
+            "testModule": HealthcheckMonitoredMock(True)
+        }
+        response = await self.healthcheck(self.app)
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(
+            response.body._value.decode('utf-8'),
+            '{"testModule": "ok", '
+            '"message": "No messages in last 60.0s", "status": "ok"}'
+        )
+
+    async def test_should_pass_when_has_two_healthy_monitored_module(self):
+        self.monitoredModules = {
+            "testModule1": HealthcheckMonitoredMock(True),
+            "testModule2": HealthcheckMonitoredMock(True)
+        }
+        response = await self.healthcheck(self.app)
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(
+            response.body._value.decode('utf-8'),
+            '{"testModule1": "ok", "testModule2": "ok", '
+            '"message": "No messages in last 60.0s", "status": "ok"}'
+        )
+
+    async def test_should_fail_when_has_one_failing_monitored_module(self):
+        self.monitoredModules = {
+            "testModule": HealthcheckMonitoredMock(False)
+        }
+        response = await self.healthcheck(self.app)
+        self.assertEqual(response.status, 500)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(
+            response.body._value.decode('utf-8'),
+            '{"testModule": "fail", '
+            '"message": "No messages in last 60.0s", "status": "fail"}'
+        )
+
+    async def test_should_fail_when_has_two_failing_monitored_module(self):
+        self.monitoredModules = {
+            "testModule1": HealthcheckMonitoredMock(False),
+            "testModule2": HealthcheckMonitoredMock(False)
+        }
+        response = await self.healthcheck(self.app)
+        self.assertEqual(response.status, 500)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(
+            response.body._value.decode('utf-8'),
+            '{"testModule1": "fail", "testModule2": "fail", '
+            '"message": "No messages in last 60.0s", "status": "fail"}'
+        )
+
+    async def test_should_fail_when_has_failing_and_healthy_modules(self):
+        self.monitoredModules = {
+            "testModule1": HealthcheckMonitoredMock(False),
+            "testModule2": HealthcheckMonitoredMock(True)
+        }
+        response = await self.healthcheck(self.app)
+        self.assertEqual(response.status, 500)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(
+            response.body._value.decode('utf-8'),
+            '{"testModule1": "fail", "testModule2": "ok", '
+            '"message": "No messages in last 60.0s", "status": "fail"}'
+        )
+
+    async def test_should_pass_when_has_one_healthy_module_and_messages(self):
+        self.monitoredModules = {
+            "testModule": HealthcheckMonitoredMock(True)
+        }
+        await self.healthcheck.on_success(None)
+        response = await self.healthcheck(self.app)
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(
+            response.body._value.decode('utf-8'),
+            '{"testModule": "ok", '
+            '"message": "Success rate: 1.0 (expected: 0.9)", '
+            '"fail": 0, "success": 1, "status": "ok"}'
+        )
+
+    async def test_should_pass_when_has_one_failing_module_and_messages(self):
+        self.monitoredModules = {
+            "testModule": HealthcheckMonitoredMock(False)
+        }
+        await self.healthcheck.on_success(None)
+        response = await self.healthcheck(self.app)
+        self.assertEqual(response.status, 500)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(
+            response.body._value.decode('utf-8'),
+            '{"testModule": "fail", '
+            '"message": "Success rate: 1.0 (expected: 0.9)", '
+            '"fail": 0, "success": 1, "status": "fail"}'
+        )
+
+    async def test_should_pass_healthcheck_when_has_simple_module(self):
+        self.monitoredModules = {
+            "testModule": MagicMock()
+        }
+        await self.healthcheck.on_success(None)
+        response = await self.healthcheck(self.app)
+        self.assertEqual(response.status, 200)
+        self.assertEqual(response.content_type, "text/plain")
+        self.assertEqual(
+            response.body._value.decode('utf-8'),
+            '{"message": "Success rate: 1.0 (expected: 0.9)", '
+            '"fail": 0, "success": 1, "status": "ok"}'
+        )