Ability to click on the affected URLs of a pattern and see which URLs have been affected

sde_collections/models/candidate_url.py

@@ -6,21 +6,24 @@
 
 from .collection import Collection
 from .collection_choice_fields import Divisions, DocumentTypes
-from .pattern import ExcludePattern, TitlePattern
+from .pattern import ExcludePattern, IncludePattern, TitlePattern
 
 
 class CandidateURLQuerySet(models.QuerySet):
-    def with_exclusion_status(self):
+    def with_exclusion_and_inclusion_status(self):
         return self.annotate(
             excluded=models.Exists(
                 ExcludePattern.candidate_urls.through.objects.filter(candidateurl=models.OuterRef("pk"))
-            )
+            ),
+            included=models.Exists(
+                IncludePattern.candidate_urls.through.objects.filter(candidateurl=models.OuterRef("pk"))
+            ),
         )
 
 
 class CandidateURLManager(models.Manager):
     def get_queryset(self):
-        return CandidateURLQuerySet(self.model, using=self._db).with_exclusion_status()
+        return CandidateURLQuerySet(self.model, using=self._db).with_exclusion_and_inclusion_status()
 
 
 class CandidateURL(models.Model):

sde_collections/serializers.py

@@ -1,3 +1,4 @@
+from django.db import models
 from rest_framework import serializers
 
 from .models.candidate_url import CandidateURL
@@ -63,6 +64,59 @@ class CandidateURLSerializer(serializers.ModelSerializer):
     match_pattern_type = serializers.SerializerMethodField(read_only=True)
     candidate_urls_count = serializers.SerializerMethodField(read_only=True)
 
+    # New fields for annotated parameters
+    included = serializers.BooleanField(read_only=True)
+
+    def get_candidate_urls_count(self, obj):
+        titlepattern = obj.titlepattern_urls.last()
+        return titlepattern.candidate_urls.count() if titlepattern else 0
+
+    def get_generated_title_id(self, obj):
+        titlepattern = obj.titlepattern_urls.last()
+        return titlepattern.id if titlepattern else None
+
+    def get_match_pattern_type(self, obj):
+        titlepattern = obj.titlepattern_urls.last()
+        return titlepattern.match_pattern_type if titlepattern else None
+
+    class Meta:
+        model = CandidateURL
+        fields = (
+            "id",
+            "excluded",
+            "included",
+            "url",
+            "scraped_title",
+            "generated_title",
+            "generated_title_id",
+            "match_pattern_type",
+            "candidate_urls_count",
+            "document_type",
+            "document_type_display",
+            "division",
+            "division_display",
+            "visited",
+            "test_title",
+            "production_title",
+            "present_on_test",
+            "present_on_prod",
+        )
+
+
+class AffectedURLSerializer(serializers.ModelSerializer):
+    excluded = serializers.BooleanField(required=False)
+    document_type_display = serializers.CharField(source="get_document_type_display", read_only=True)
+    division_display = serializers.CharField(source="get_division_display", read_only=True)
+    url = serializers.CharField(required=False)
+    generated_title_id = serializers.SerializerMethodField(read_only=True)
+    match_pattern_type = serializers.SerializerMethodField(read_only=True)
+    candidate_urls_count = serializers.SerializerMethodField(read_only=True)
+
+    # New fields for annotated parameters
+    included = serializers.BooleanField(read_only=True)
+    included_by_pattern = serializers.CharField(read_only=True)
+    match_pattern_id = serializers.IntegerField(read_only=True)
+
     def get_candidate_urls_count(self, obj):
         titlepattern = obj.titlepattern_urls.last()
         return titlepattern.candidate_urls.count() if titlepattern else 0
@@ -95,6 +149,9 @@ class Meta:
             "production_title",
             "present_on_test",
             "present_on_prod",
+            "included",  # New field
+            "included_by_pattern",  # New field
+            "match_pattern_id",  # New field
         )
 
 
@@ -172,6 +229,27 @@ class Meta:
         model = ExcludePattern
         fields = BasePatternSerializer.Meta.fields + ("reason",)
 
+    def get_candidate_urls_count(self, instance):
+        # Count URLs matched by the excluded pattern
+        matched_urls = instance.candidate_urls
+        matched_urls_count = matched_urls.count()
+
+        # Get the IDs of the matched URLs
+        matched_url_ids = matched_urls.values_list("id", flat=True)
+
+        # Count URLs included by other patterns in the same collection
+        included_urls_count = (
+            IncludePattern.objects.filter(collection=instance.collection, candidate_urls__in=matched_url_ids)
+            .annotate(included_count=models.Count("candidate_urls"))
+            .aggregate(total=models.Sum("included_count"))["total"]
+            or 0
+        )
+
+        # Calculate effective URLs count
+        effective_urls_count = matched_urls_count - included_urls_count
+
+        return effective_urls_count
+
 
 class IncludePatternSerializer(BasePatternSerializer, serializers.ModelSerializer):
     class Meta:
@@ -245,3 +323,22 @@ def validate_match_pattern(self, value):
         except DivisionPattern.DoesNotExist:
             pass
         return value
+
+
+class BaseAffectedURLSerializer(serializers.ModelSerializer):
+    match_pattern_type_display = serializers.CharField(source="get_match_pattern_type_display", read_only=True)
+    candidate_urls_count = serializers.SerializerMethodField(read_only=True)
+
+    def get_candidate_urls_count(self, instance):
+        return instance.candidate_urls.count()
+
+    class Meta:
+        fields = (
+            "id",
+            "collection",
+            "match_pattern",
+            "match_pattern_type",
+            "match_pattern_type_display",
+            "candidate_urls_count",
+        )
+        abstract = True

sde_collections/urls.py

@@ -9,6 +9,20 @@
 router.register(r"collections", views.CollectionViewSet, basename="collection")
 router.register(r"collections-read", views.CollectionReadViewSet, basename="collection-read")
 router.register(r"candidate-urls", views.CandidateURLViewSet)
+router.register(
+    r"include-pattern-affected-urls", views.IncludePatternAffectedURLsViewSet, basename="include-pattern-affected-urls"
+)
+router.register(
+    r"exclude-pattern-affected-urls", views.ExcludePatternAffectedURLsViewSet, basename="exclude-pattern-affected-urls"
+)
+router.register(
+    r"title-pattern-affected-urls", views.TitlePatternAffectedURLsViewSet, basename="title-pattern-affected-urls"
+)
+router.register(
+    r"documenttype-pattern-affected-urls",
+    views.DocumentTypePatternAffectedURLsViewSet,
+    basename="documenttype-pattern-affected-urls",
+)
 router.register(r"exclude-patterns", views.ExcludePatternViewSet)
 router.register(r"include-patterns", views.IncludePatternViewSet)
 router.register(r"title-patterns", views.TitlePatternViewSet)
@@ -43,6 +57,26 @@
         view=views.CandidateURLsListView.as_view(),
         name="candidate_urls",
     ),
+    path(
+        "exclude-pattern/<int:id>/",
+        view=views.ExcludePatternAffectedURLsListView.as_view(),
+        name="affected_urls",
+    ),
+    path(
+        "include-pattern/<int:id>/",
+        view=views.IncludePatternAffectedURLsListView.as_view(),
+        name="affected_urls",
+    ),
+    path(
+        "title-pattern/<int:id>/",
+        view=views.TitlePatternAffectedURLsListView.as_view(),
+        name="affected_urls",
+    ),
+    path(
+        "document-type-pattern/<int:id>/",
+        view=views.DocumentTypePatternAffectedURLsListView.as_view(),
+        name="affected_urls",
+    ),
     path(
         "consolidate/",
         view=views.WebappGitHubConsolidationView.as_view(),

sde_collections/views.py

@@ -35,6 +35,7 @@
     TitlePattern,
 )
 from .serializers import (
+    AffectedURLSerializer,
     CandidateURLAPISerializer,
     CandidateURLBulkCreateSerializer,
     CandidateURLSerializer,
@@ -226,6 +227,72 @@ def get_context_data(self, **kwargs):
         return context
 
 
+class BaseAffectedURLsListView(LoginRequiredMixin, ListView):
+    """
+    Base view for displaying a list of URLs affected by a match pattern
+    """
+
+    model = CandidateURL
+    template_name = "sde_collections/affected_urls.html"
+    context_object_name = "affected_urls"
+    pattern_model = None
+    pattern_type = None
+
+    def get_queryset(self):
+        self.pattern = self.pattern_model.objects.get(id=self.kwargs["id"])
+        queryset = self.pattern.matched_urls()
+        return queryset
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context["pattern"] = self.pattern
+        context["pattern_id"] = self.kwargs["id"]
+        context["url_count"] = self.get_queryset().count()
+        context["collection"] = self.pattern.collection
+        context["pattern_type"] = self.pattern_type
+        return context
+
+
+class ExcludePatternAffectedURLsListView(BaseAffectedURLsListView):
+    pattern_model = ExcludePattern
+    pattern_type = "Exclude"
+
+    def get_queryset(self):
+        self.pattern = self.pattern_model.objects.get(id=self.kwargs["id"])
+        queryset = self.pattern.matched_urls()
+
+        # Subquery to get the match_pattern and id of the IncludePattern
+        include_pattern_subquery = IncludePattern.objects.filter(candidate_urls=models.OuterRef("pk")).values(
+            "match_pattern", "id"
+        )[:1]
+
+        # Annotate with inclusion status, match_pattern, and id of the IncludePattern
+        queryset = queryset.annotate(
+            included=models.Exists(include_pattern_subquery),
+            included_by_pattern=models.Subquery(
+                include_pattern_subquery.values("match_pattern"), output_field=models.CharField()
+            ),
+            match_pattern_id=models.Subquery(include_pattern_subquery.values("id"), output_field=models.IntegerField()),
+        )
+
+        return queryset
+
+
+class IncludePatternAffectedURLsListView(BaseAffectedURLsListView):
+    pattern_model = IncludePattern
+    pattern_type = "Include"
+
+
+class TitlePatternAffectedURLsListView(BaseAffectedURLsListView):
+    pattern_model = TitlePattern
+    pattern_type = "Title"
+
+
+class DocumentTypePatternAffectedURLsListView(BaseAffectedURLsListView):
+    pattern_model = DocumentTypePattern
+    pattern_type = "Document Type"
+
+
 class SdeDashboardView(LoginRequiredMixin, ListView):
     model = Collection
     template_name = "sde_collections/sde_dashboard.html"
@@ -272,6 +339,18 @@ def get_queryset(self):
             is_excluded = self.request.GET.get("is_excluded")
             if is_excluded:
                 queryset = self._filter_by_is_excluded(queryset, is_excluded)
+
+            collection_id = self.request.GET.get("collection_id")
+            if collection_id:
+                queryset = queryset.annotate(
+                    included=models.Exists(
+                        IncludePattern.candidate_urls.through.objects.filter(
+                            candidateurl=models.OuterRef("pk"),
+                            includepattern__collection_id=collection_id,  # Filter by the specific collection
+                        )
+                    )
+                )
+
         return queryset.order_by("url")
 
     def update_division(self, request, pk=None):
@@ -318,8 +397,8 @@ def get(self, request, *args, **kwargs):
     def get_queryset(self):
         queryset = (
             CandidateURL.objects.filter(collection__config_folder=self.config_folder)
-            .with_exclusion_status()
-            .filter(excluded=False)
+            .with_exclusion_and_inclusion_status()
+            .filter(models.Q(excluded=False) | models.Q(included=True))
         )
         return queryset
 
@@ -554,3 +633,57 @@ def get(self, request, *args, **kwargs):
             "resolved_title_errors": resolved_title_errors,
         }
         return render(request, "sde_collections/titles_and_errors_list.html", context)
+
+
+class BaseAffectedURLsViewSet(CollectionFilterMixin, viewsets.ModelViewSet):
+    queryset = CandidateURL.objects.all()
+    serializer_class = AffectedURLSerializer
+    pattern_model = None
+    pattern_type = None
+
+    def get_queryset(self):
+        pattern_id = self.request.GET.get("pattern_id")
+        self.pattern = self.pattern_model.objects.get(id=pattern_id)
+        queryset = self.pattern.matched_urls()
+        return queryset
+
+
+class IncludePatternAffectedURLsViewSet(BaseAffectedURLsViewSet):
+    pattern_model = IncludePattern
+    pattern_type = "Include"
+
+
+class ExcludePatternAffectedURLsViewSet(BaseAffectedURLsViewSet):
+    pattern_model = ExcludePattern
+    pattern_type = "Exclude"
+
+    def get_queryset(self):
+        pattern_id = self.request.GET.get("pattern_id")
+        self.pattern = self.pattern_model.objects.get(id=pattern_id)
+        queryset = self.pattern.matched_urls()
+
+        # Subquery to get the match_pattern and id of the IncludePattern
+        include_pattern_subquery = IncludePattern.objects.filter(candidate_urls=models.OuterRef("pk")).values(
+            "match_pattern", "id"
+        )[:1]
+
+        # Annotate with inclusion status, match_pattern, and id of the IncludePattern
+        queryset = queryset.annotate(
+            included=models.Exists(include_pattern_subquery),
+            included_by_pattern=models.Subquery(
+                include_pattern_subquery.values("match_pattern"), output_field=models.CharField()
+            ),
+            match_pattern_id=models.Subquery(include_pattern_subquery.values("id"), output_field=models.IntegerField()),
+        )
+
+        return queryset
+
+
+class TitlePatternAffectedURLsViewSet(BaseAffectedURLsViewSet):
+    pattern_model = TitlePattern
+    pattern_type = "Title"
+
+
+class DocumentTypePatternAffectedURLsViewSet(BaseAffectedURLsViewSet):
+    pattern_model = DocumentTypePattern
+    pattern_type = "Document Type"