Rework the histogram activity chart

temba/flows/tests.py

@@ -56,7 +56,6 @@
     trim_flow_sessions,
     update_session_wait_expires,
 )
-from .views import FlowCRUDL
 
 
 class FlowTest(TembaTest, CRUDLTestMixin):
@@ -2888,9 +2887,6 @@ def test_results(self):
         self.assertEqual("Color", counts[0]["name"])
         self.assertEqual(2, counts[0]["total"])
 
-        FlowCRUDL.ActivityChart.HISTOGRAM_MIN = 0
-        FlowCRUDL.ActivityChart.PERIOD_MIN = 0
-
         # and some charts
         response = self.client.get(reverse("flows.flow_activity_data", args=[flow.id]))
         data = response.json()

temba/flows/views.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 from datetime import datetime, timedelta
 from urllib.parse import urlencode
 
@@ -19,10 +20,11 @@
 from django.conf import settings
 from django.contrib.humanize.templatetags import humanize
 from django.core.exceptions import ValidationError
-from django.db.models import Max, Min, Sum
+from django.db.models import Sum
 from django.db.models.functions import Lower
 from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse
 from django.urls import reverse
+from django.utils import timezone
 from django.utils.encoding import force_str
 from django.utils.functional import cached_property
 from django.utils.translation import gettext_lazy as _, ngettext_lazy as _p
@@ -47,6 +49,7 @@
 from temba.orgs.views.mixins import BulkActionMixin, OrgObjPermsMixin, OrgPermsMixin
 from temba.triggers.models import Trigger
 from temba.utils import analytics, gettext, json, languages, on_transaction_commit
+from temba.utils.dates import date_trunc
 from temba.utils.fields import (
     CheckboxWidget,
     ContactSearchWidget,
@@ -1146,12 +1149,6 @@ def create_export(self, org, user, form):
             )
 
     class ActivityData(BaseReadView):
-        # the min number of responses to show a histogram
-        HISTOGRAM_MIN = 0
-
-        # the min number of responses to show the period charts
-        PERIOD_MIN = 0
-
         permission = "flows.flow_results"
 
         day_names = (
@@ -1164,7 +1161,7 @@ class ActivityData(BaseReadView):
             _("Saturday"),
         )
 
-        def get_day_of_week_counts(self, exit_uuids: list) -> dict:
+        def get_day_of_week_counts(self, exit_uuids: list) -> dict[int, int]:
             dow = (
                 self.object.path_counts.filter(from_uuid__in=exit_uuids)
                 .extra({"day": "extract(dow from period::timestamp)"})
@@ -1174,7 +1171,7 @@ def get_day_of_week_counts(self, exit_uuids: list) -> dict:
 
             return {int(d.get("day")): d.get("count") for d in dow}
 
-        def get_hour_of_day_counts(self, exit_uuids: list) -> dict:
+        def get_hour_of_day_counts(self, exit_uuids: list) -> dict[int, int]:
             hod = (
                 self.object.path_counts.filter(from_uuid__in=exit_uuids)
                 .extra({"hour": "extract(hour from period::timestamp)"})
@@ -1185,14 +1182,21 @@ def get_hour_of_day_counts(self, exit_uuids: list) -> dict:
 
             return {int(h.get("hour")): h.get("count") for h in hod}
 
+        def get_date_counts(self, exit_uuids: list) -> list[tuple]:
+            dates = (
+                self.object.path_counts.filter(from_uuid__in=exit_uuids)
+                .extra({"date": "period::date"})
+                .values("date")
+                .annotate(count=Sum("count"))
+                .order_by("date")
+            )
+
+            return [(d.get("date"), d.get("count")) for d in dates]
+
         def render_to_response(self, context, **response_kwargs):
             total_responses = 0
             flow = self.object
-
             exit_uuids = flow.metadata["waiting_exit_uuids"]
-            dates = self.object.path_counts.filter(from_uuid__in=exit_uuids).aggregate(Max("period"), Min("period"))
-            start_date = dates.get("period__min")
-            end_date = dates.get("period__max")
 
             hour_dict = self.get_hour_of_day_counts(exit_uuids)
             hours = []
@@ -1206,34 +1210,32 @@ def render_to_response(self, context, **response_kwargs):
                 dow.append({"name": x, "msgs": day_count})
                 total_responses += day_count
 
-            if total_responses > self.PERIOD_MIN:
-                dow = sorted(dow, key=lambda k: k["name"])
-                dow = [
-                    {
-                        "name": self.day_names[d["name"]],
-                        "msgs": d["msgs"],
-                        "y": 100 * float(d["msgs"]) / float(total_responses),
-                    }
-                    for d in dow
-                ]
+            dow = [
+                {
+                    "name": self.day_names[d["name"]],
+                    "msgs": d["msgs"],
+                    "y": 100 * float(d["msgs"]) / float(total_responses),
+                }
+                for d in dow
+            ]
 
-            min_date = None
-            histogram = []
+            dates = self.get_date_counts(exit_uuids)
+            min_date = dates[0][0] if dates else timezone.now().date() - timedelta(days=30)
 
-            if total_responses > self.HISTOGRAM_MIN:
-                # our main histogram
-                date_range = end_date - start_date
-                histogram = self.object.path_counts.filter(from_uuid__in=exit_uuids)
+            # bucket dates into months or weeks depending on the range
+            if min_date < timezone.now().date() - timedelta(days=365 * 3):
+                buckets = defaultdict(int)
+                for day, count in dates:
+                    buckets[date_trunc("month", day)] += count
 
-                if date_range < timedelta(days=500):
-                    histogram = histogram.extra({"bucket": "date_trunc('day', period)"})
-                    min_date = end_date - timedelta(days=100)
-                else:
-                    histogram = histogram.extra({"bucket": "date_trunc('week', period)"})
-                    min_date = end_date - timedelta(days=500)
+                dates = sorted(buckets.items(), key=lambda x: x[0])
+
+            elif min_date < timezone.now().date() - timedelta(days=365):
+                buckets = defaultdict(int)
+                for day, count in dates:
+                    buckets[date_trunc("week", day)] += count
 
-                histogram = histogram.values("bucket").annotate(count=Sum("count")).order_by("bucket")
-                histogram = [[_["bucket"], _["count"]] for _ in histogram]
+                dates = sorted(buckets.items(), key=lambda x: x[0])
 
             summary = {
                 "responses": total_responses,
@@ -1279,13 +1281,11 @@ def render_to_response(self, context, **response_kwargs):
 
             return JsonResponse(
                 {
-                    "start_date": start_date,
-                    "end_date": end_date,
                     "min_date": min_date,
                     "summary": summary,
                     "dow": dow,
                     "hod": hours,
-                    "histogram": histogram,
+                    "histogram": dates,
                     "completion": completion,
                 },
                 json_dumps_params={"indent": 2},

temba/utils/dates.py

@@ -46,3 +46,18 @@ def date_range(start: date, stop: date):
     """
     for n in range(int((stop - start).days)):
         yield start + timedelta(n)
+
+
+def date_trunc(field, source: date) -> date:
+    """
+    Some limited mimicry of the SQL date_trunc function
+    """
+
+    if field == "week":
+        return source - timedelta(days=source.weekday() % 7)
+    elif field == "month":
+        return source.replace(day=1)
+    elif field == "year":
+        return source.replace(month=1, day=1)
+    else:
+        raise ValueError(f"Unsupported truncation field: {field}")

temba/utils/tests.py

@@ -22,7 +22,7 @@
 from . import countries, format_number, get_nested_key, languages, percentage, redact, set_nested_key, str_to_bool
 from .checks import storage
 from .crons import clear_cron_stats, cron_task
-from .dates import date_range, datetime_to_str, datetime_to_timestamp, timestamp_to_datetime
+from .dates import date_range, date_trunc, datetime_to_str, datetime_to_timestamp, timestamp_to_datetime
 from .fields import ExternalURLField, NameValidator
 from .text import clean_string, generate_secret, generate_token, slugify_with, truncate, unsnakify
 from .timezones import TimeZoneFormField, timezone_to_country_code
@@ -133,6 +133,22 @@ def test_date_range(self):
         )
         self.assertEqual([], list(date_range(date(2015, 1, 29), date(2015, 1, 29))))
 
+    def test_date_trunc(self):
+        tests = [
+            (date(2024, 11, 22), "week", date(2024, 11, 18)),
+            (date(2024, 11, 22), "month", date(2024, 11, 1)),
+            (date(2024, 11, 22), "year", date(2024, 1, 1)),
+            (date(2024, 1, 2), "week", date(2024, 1, 1)),
+            (date(2023, 1, 2), "week", date(2023, 1, 2)),
+            (date(2022, 1, 2), "week", date(2021, 12, 27)),
+            (date(2021, 1, 2), "week", date(2020, 12, 28)),
+        ]
+        for d, field, expected in tests:
+            self.assertEqual(expected, date_trunc(field, d), f"date_trunc mismatch for input {d}, {field}")
+
+        with self.assertRaises(ValueError):
+            date_trunc("hour", date(2024, 11, 22))
+
 
 class CountriesTest(TembaTest):
     def test_from_tel(self):