Tests: run tests with the ext-theme (#11383)

* Tests: run tests with the ext-theme

* Fix tests

* Run tests in another job

* Fix proxito tests

* Comment

* Update common

* Use custom 401 for proxito only

* Move 401 to errors/proxito

* Change working to match ext-theme

* One more template

* One more

* Use log in instead of sign in

This matches ext-theme

* Comment

.circleci/config.yml

@@ -9,6 +9,7 @@ jobs:
     docker:
       - image: 'cimg/python:3.10'
         environment:
+          # Don't skip search tests.
           TOX_POSARGS: ''
           PYTEST_COVERAGE: --cov-report=xml --cov-config .coveragerc --cov=. --cov-append
       - image: 'docker.elastic.co/elasticsearch/elasticsearch:8.10.2'
@@ -25,10 +26,33 @@ jobs:
       - run: git submodule update --init
       - run: sudo apt update
       - run: sudo apt install -y rclone
-      - run: pip install --user 'tox<5'
+      - run: pip install --user tox
       - run: tox -e py310
       - codecov/upload
 
+  tests-ext-theme:
+    docker:
+      - image: 'cimg/python:3.10'
+        environment:
+          # Don't skip search tests.
+          TOX_POSARGS: ''
+      - image: 'docker.elastic.co/elasticsearch/elasticsearch:8.10.2'
+        name: search
+        environment:
+          discovery.type: single-node
+          ES_JAVA_OPTS: -Xms750m -Xmx750m
+          ELASTIC_PASSWORD: password
+          # Disabled SSL for testing.
+          xpack.security.transport.ssl.enabled: 'false'
+    steps:
+      - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
+      - run: sudo apt update
+      - run: sudo apt install -y rclone
+      - run: pip install --user tox
+      - run: tox -e ext-theme
+
   tests-embedapi:
     docker:
       - image: 'cimg/python:3.10'
@@ -75,6 +99,7 @@ workflows:
     jobs:
       - checks
       - tests
+      - tests-ext-theme
       - tests-embedapi:
           requires:
             - checks

readthedocs/core/views/__init__.py

@@ -7,7 +7,7 @@
 
 import structlog
 from django.conf import settings
-from django.http import JsonResponse
+from django.http import Http404, JsonResponse
 from django.shortcuts import redirect
 from django.urls import reverse
 from django.views.generic import TemplateView, View
@@ -148,6 +148,14 @@ def get(self, request, *args, **kwargs):
         return self.render_to_response(context, status=418)
 
 
+class PageNotFoundView(View):
+
+    """Just a 404 view that ignores all URL parameters."""
+
+    def get(self, request, *args, **kwargs):
+        raise Http404()
+
+
 def do_not_track(request):
     dnt_header = request.headers.get("Dnt")
 

readthedocs/organizations/tests/test_privacy_urls.py

@@ -66,6 +66,8 @@ class AuthUserOrganizationsTest(OrganizationMixin, TestCase):
         "/organizations/{slug}/teams/{team}/members/{member}/revoke/": {
             "status_code": 405
         },
+        # Placeholder URL.
+        "/organizations/{slug}/authorization/": {"status_code": 404},
     }
 
     def login(self):

readthedocs/organizations/urls/private.py

@@ -1,6 +1,7 @@
 """URLs that require login."""
 from django.urls import path, re_path
 
+from readthedocs.core.views import PageNotFoundView
 from readthedocs.organizations.views import private as views
 
 urlpatterns = [
@@ -81,4 +82,12 @@
         views.DeleteOrganizationTeamMember.as_view(),
         name="organization_team_member_delete",
     ),
+    # Placeholder URL, so that we can test the new templates
+    # with organizations enabled from our community codebase.
+    # TODO: migrate this functionality from corporate to community.
+    re_path(
+        r"^(?P<slug>[\w.-]+)/authorization/$",
+        PageNotFoundView.as_view(),
+        name="organization_sso",
+    ),
 ]

readthedocs/projects/urls/private.py

@@ -1,11 +1,11 @@
 """Project URLs for authenticated users."""
-
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.urls import path, re_path
 from django.views.generic.base import RedirectView
 
 from readthedocs.constants import pattern_opts
+from readthedocs.core.views import PageNotFoundView
 from readthedocs.projects.backends.views import ImportWizardView
 from readthedocs.projects.views import private
 from readthedocs.projects.views.private import (
@@ -191,6 +191,19 @@
         TrafficAnalyticsView.as_view(),
         name="projects_traffic_analytics",
     ),
+    # Placeholder URLs, so that we can test the new templates
+    # with organizations enabled from our community codebase.
+    # TODO: migrate these functionalities from corporate to community.
+    re_path(
+        r"^(?P<project_slug>{project_slug})/sharing/$".format(**pattern_opts),
+        PageNotFoundView.as_view(),
+        name="projects_temporary_access_list",
+    ),
+    re_path(
+        (r"^(?P<project_slug>{project_slug})/keys/$".format(**pattern_opts)),
+        PageNotFoundView.as_view(),
+        name="projects_keys",
+    ),
 ]
 
 # TODO move this up to the list above when it's not a conditional URL.

readthedocs/proxito/tests/handler_404_urls.py

@@ -12,7 +12,10 @@
 """
 from functools import wraps
 
+from django.http import Http404
+
 from readthedocs.proxito.urls import *  # noqa
+from readthedocs.proxito.urls import handler404 as proxito_handler404
 from readthedocs.proxito.views.serve import ServeError404
 
 
@@ -21,18 +24,17 @@
 def map_proxito_path(view_func):
     @wraps(view_func)
     def inner_view(request, *args, **kwargs):
-        return view_func(
-            request,
-            proxito_path=request.path,
-        )
+        try:
+            return view_func(
+                request,
+                proxito_path=request.path,
+            )
+        except Http404 as e:
+            # Fall back to our custom 404 handler,
+            # if a 404 was raised by proxito's 404 view.
+            return proxito_handler404(request, exception=e)
 
     return inner_view
 
 
 handler404 = map_proxito_path(ServeError404.as_view())
-
-# Patch URLs that call the `fast_404` view directly.
-for pattern in urlpatterns:
-    if getattr(pattern, "name", None) == "docs_detail_directory_indexing":
-        pattern.callback = handler404
-        break

readthedocs/proxito/tests/test_old_redirects.py

@@ -9,8 +9,6 @@
 """
 
 import django_dynamic_fixture as fixture
-import pytest
-from django.http import Http404
 from django.test.utils import override_settings
 
 from readthedocs.builds.constants import EXTERNAL
@@ -196,8 +194,8 @@ def test_disabled_redirect(self):
         redirect.enabled = False
         redirect.save()
 
-        with self.assertRaises(Http404):
-            self.client.get(url, headers={"host": "project.dev.readthedocs.io"})
+        r = self.client.get(url, headers={"host": "project.dev.readthedocs.io"})
+        self.assertEqual(r.status_code, 404)
 
     def test_redirect_order(self):
         redirect_a = fixture.get(
@@ -267,8 +265,8 @@ def test_redirect_ignored_on_external_domain(self):
             slug="22",
             type=EXTERNAL,
         )
-        with self.assertRaises(Http404):
-            self.client.get(url, headers={"host": "project--22.readthedocs.build"})
+        r = self.client.get(url, headers={"host": "project--22.readthedocs.build"})
+        self.assertEqual(r.status_code, 404)
 
     def test_infinite_redirect(self):
         host = "project.dev.readthedocs.io"
@@ -279,11 +277,11 @@ def test_infinite_redirect(self):
             from_url="/en/latest/install.html",
             to_url="/en/latest/install.html",
         )
-        with pytest.raises(Http404):
-            self.client.get("/en/latest/install.html", headers={"host": host})
+        r = self.client.get("/en/latest/install.html", headers={"host": host})
+        self.assertEqual(r.status_code, 404)
 
-        with pytest.raises(Http404):
-            self.client.get("/en/latest/install.html?foo=bar", headers={"host": host})
+        r = self.client.get("/en/latest/install.html?foo=bar", headers={"host": host})
+        self.assertEqual(r.status_code, 404)
 
     def test_infinite_redirect_changing_protocol(self):
         host = "project.dev.readthedocs.io"
@@ -294,11 +292,12 @@ def test_infinite_redirect_changing_protocol(self):
             from_url="/en/latest/install.html",
             to_url=f"https://{host}/en/latest/install.html",
         )
-        with pytest.raises(Http404):
-            self.client.get("/en/latest/install.html", headers={"host": host})
 
-        with pytest.raises(Http404):
-            self.client.get("/en/latest/install.html?foo=bar", headers={"host": host})
+        r = self.client.get("/en/latest/install.html", headers={"host": host})
+        self.assertEqual(r.status_code, 404)
+
+        r = self.client.get("/en/latest/install.html?foo=bar", headers={"host": host})
+        self.assertEqual(r.status_code, 404)
 
     def test_exact_redirect_avoid_infinite_redirect(self):
         """
@@ -335,10 +334,10 @@ def test_exact_redirect_avoid_infinite_redirect(self):
             "http://project.dev.readthedocs.io/en/latest/redirect/",
         )
 
-        with self.assertRaises(Http404):
-            self.client.get(
-                "/en/latest/redirect/", headers={"host": "project.dev.readthedocs.io"}
-            )
+        r = self.client.get(
+            "/en/latest/redirect/", headers={"host": "project.dev.readthedocs.io"}
+        )
+        self.assertEqual(r.status_code, 404)
 
         fixture.get(
             Redirect,
@@ -356,11 +355,11 @@ def test_exact_redirect_avoid_infinite_redirect(self):
             "http://project.dev.readthedocs.io/en/latest/subdir/redirect.html",
         )
 
-        with self.assertRaises(Http404):
-            self.client.get(
-                "/en/latest/subdir/redirect.html",
-                headers={"host": "project.dev.readthedocs.io"},
-            )
+        r = self.client.get(
+            "/en/latest/subdir/redirect.html",
+            headers={"host": "project.dev.readthedocs.io"},
+        )
+        self.assertEqual(r.status_code, 404)
 
     def test_page_redirect_avoid_infinite_redirect(self):
         fixture.get(
@@ -379,11 +378,11 @@ def test_page_redirect_avoid_infinite_redirect(self):
             "http://project.dev.readthedocs.io/en/latest/subdir/redirect.html",
         )
 
-        with self.assertRaises(Http404):
-            self.client.get(
-                "/en/latest/subdir/redirect.html",
-                headers={"host": "project.dev.readthedocs.io"},
-            )
+        r = self.client.get(
+            "/en/latest/subdir/redirect.html",
+            headers={"host": "project.dev.readthedocs.io"},
+        )
+        self.assertEqual(r.status_code, 404)
 
         fixture.get(
             Redirect,
@@ -402,11 +401,11 @@ def test_page_redirect_avoid_infinite_redirect(self):
             "http://project.dev.readthedocs.io/en/latest/dir/subdir/redirect.html",
         )
 
-        with self.assertRaises(Http404):
-            self.client.get(
-                "/en/latest/dir/subdir/redirect.html",
-                headers={"host": "project.dev.readthedocs.io"},
-            )
+        r = self.client.get(
+            "/en/latest/dir/subdir/redirect.html",
+            headers={"host": "project.dev.readthedocs.io"},
+        )
+        self.assertEqual(r.status_code, 404)
 
     def test_exact_redirect_to_parent_path(self):
         self.project.versioning_scheme = SINGLE_VERSION_WITHOUT_TRANSLATIONS
@@ -497,11 +496,11 @@ def test_redirect_root(self):
         )
 
         # Prefix redirects should match the whole path.
-        with self.assertRaises(Http404):
-            self.client.get(
-                "/en/latest/woot/faq.html",
-                headers={"host": "project.dev.readthedocs.io"},
-            )
+        r = self.client.get(
+            "/en/latest/woot/faq.html",
+            headers={"host": "project.dev.readthedocs.io"},
+        )
+        self.assertEqual(r.status_code, 404)
 
     def test_redirect_page(self):
         Redirect.objects.create(
@@ -860,11 +859,11 @@ def test_redirect_html_root_index(self):
             )
 
         with override_settings(PYTHON_MEDIA=True):
-            with self.assertRaises(Http404):
-                # File does not exist in storage media
-                r = self.client.get(
-                    "/en/latest/", headers={"host": "project.dev.readthedocs.io"}
-                )
+            # File does not exist in storage media
+            r = self.client.get(
+                "/en/latest/", headers={"host": "project.dev.readthedocs.io"}
+            )
+            self.assertEqual(r.status_code, 404)
 
     def test_redirect_html_index(self):
         fixture.get(
@@ -924,12 +923,12 @@ def test_not_found_page_without_trailing_slash(self):
             to_url="/en/latest/:splat",
         )
 
-        with self.assertRaises(Http404):
-            # Avoid infinite redirect
-            r = self.client.get(
-                "/en/latest/section/file-not-found",
-                headers={"host": "project.dev.readthedocs.io"},
-            )
+        # Avoid infinite redirect
+        r = self.client.get(
+            "/en/latest/section/file-not-found",
+            headers={"host": "project.dev.readthedocs.io"},
+        )
+        self.assertEqual(r.status_code, 404)
 
     def test_page_redirect_with_and_without_trailing_slash(self):
         fixture.get(

readthedocs/proxito/views/mixins.py

@@ -270,7 +270,7 @@ def _serve_file_from_python(self, request, path, storage):
         return serve(request, path, root_path)
 
     def _serve_401(self, request, project):
-        res = render(request, "401.html")
+        res = render(request, "errors/proxito/401.html")
         res.status_code = 401
         log.debug("Unauthorized access to documentation.", project_slug=project.slug)
         return res