Add GitHub Actions, linting and formatting

.circleci/report_nightly_build_failure.py

@@ -7,13 +7,18 @@
 
 import requests
 
-if 'SLACK_WEBHOOK_URL' in os.environ:
+if "SLACK_WEBHOOK_URL" in os.environ:
     print("Reporting to #nightly-build-failures slack channel")
-    response = requests.post(os.environ['SLACK_WEBHOOK_URL'], json={
-        "text": "A Nightly build failed. See " + os.environ['CIRCLE_BUILD_URL'],
-    })
+    response = requests.post(
+        os.environ["SLACK_WEBHOOK_URL"],
+        json={
+            "text": "A Nightly build failed. See " + os.environ["CIRCLE_BUILD_URL"],
+        },
+    )
 
     print("Slack responded with:", response)
 
 else:
-    print("Unable to report to #nightly-build-failures slack channel because SLACK_WEBHOOK_URL is not set")
+    print(
+        "Unable to report to #nightly-build-failures slack channel because SLACK_WEBHOOK_URL is not set"
+    )

.git-blame-ignore-revs

@@ -0,0 +1,6 @@
+# Set up ruff and pre-commit, apply fixes
+84f991d844dcbfc29e4ce9a0b0afb3846c7dcd06
+c6cf87660d0e685121d8c9e62729c30c4641df07
+b83aab8c4442b3e1cf95950a5b1fcbae0d7ad5b1
+e166089173b2337803ba17cdff051d24b4b0515e
+ca3b4000068d19e736149a038b0d479e5e78b8e9

.github/workflows/test.yml

@@ -0,0 +1,40 @@
+name: wagtail-transfer CI
+
+on:
+  push:
+    branches:
+      - main
+      - master
+      - 'stable/**'
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{env.PYTHON_LATEST}}
+      - uses: pre-commit/[email protected]
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh-actions
+      - name: Test with tox
+        run: tox

.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+exclude: |
+  (?x)(
+    wagtail_transfer.js
+    |document.txt
+    |.babelrc
+  )
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-yaml
+        args: ['--unsafe']
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: 1.16.0
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black==23.10.0]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.1.2'
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
+        args: [--check]

docs/management_commands.md

@@ -30,4 +30,3 @@ Suppose a site has been developed and populated with content on a staging enviro
  * On both instances, run: `./manage.py preseed_transfer_table wagtailcore.page --range=1-199`
 
  The `preseed_transfer_table` command generates consistent UUIDs between the two site instances, so any transfers involving this ID range will recognise the pages as matching, and handle them as updates rather than creations.
-

docs/settings.md

@@ -5,24 +5,24 @@
 ### `WAGTAILTRANSFER_SECRET_KEY`
 
 ```python
-WAGTAILTRANSFER_SECRET_KEY = '7cd5de8229be75e1e0c2af8abc2ada7e'
+WAGTAILTRANSFER_SECRET_KEY = "7cd5de8229be75e1e0c2af8abc2ada7e"
 ```
 
-The secret key used to authenticate requests to import content from this site to another. The secret key in the 
-matching part of the importing site's `WAGTAILTRANSFER_SOURCES` must be identical, or the transfer will be rejected - 
-this prevents unauthorised import of sensitive data. 
+The secret key used to authenticate requests to import content from this site to another. The secret key in the
+matching part of the importing site's `WAGTAILTRANSFER_SOURCES` must be identical, or the transfer will be rejected -
+this prevents unauthorised import of sensitive data.
 
 ### `WAGTAILTRANSFER_SOURCES`
 
 ```python
 WAGTAILTRANSFER_SOURCES = {
-    'staging': {
-        'BASE_URL': 'https://staging.example.com/wagtail-transfer/',
-        'SECRET_KEY': '4ac4822149691395773b2a8942e1a472',
+    "staging": {
+        "BASE_URL": "https://staging.example.com/wagtail-transfer/",
+        "SECRET_KEY": "4ac4822149691395773b2a8942e1a472",
     },
-    'production': {
-        'BASE_URL': 'https://www.example.com/wagtail-transfer/',
-        'SECRET_KEY': 'a36476ffc6af34dc935570d97369eca0',
+    "production": {
+        "BASE_URL": "https://www.example.com/wagtail-transfer/",
+        "SECRET_KEY": "a36476ffc6af34dc935570d97369eca0",
     },
 }
 ```
@@ -32,44 +32,44 @@ A dictionary defining the sites available to import from, and their secret keys.
 ### `WAGTAILTRANSFER_UPDATE_RELATED_MODELS`
 
 ```python
-WAGTAILTRANSFER_UPDATE_RELATED_MODELS = ['wagtailimages.image', 'adverts.advert']
+WAGTAILTRANSFER_UPDATE_RELATED_MODELS = ["wagtailimages.image", "adverts.advert"]
 ```
 
-Specifies a list of models that, whenever we encounter references to them in imported content, should be updated to the 
+Specifies a list of models that, whenever we encounter references to them in imported content, should be updated to the
 latest version from the source site as part of the import.
 
-Whenever an object being imported contains a reference to a related object (through a ForeignKey, RichTextField or 
-StreamField), the 'importance' of that related object will tend to vary according to its type. For example, a reference 
-to an Image object within a page usually means that the image will be shown on that page; in this case, the Image model 
-is sufficiently important to the imported page that we want the importer to not only ensure that image exists at the 
-destination, but is updated to its newest version as well. Contrast this with the example of an 'author' snippet 
-attached to blog posts, containing various fields of data about that person (e.g. bio, social media links); in this 
-case, the author information is not really part of the blog post, and it's not expected that we would update it when 
+Whenever an object being imported contains a reference to a related object (through a ForeignKey, RichTextField or
+StreamField), the 'importance' of that related object will tend to vary according to its type. For example, a reference
+to an Image object within a page usually means that the image will be shown on that page; in this case, the Image model
+is sufficiently important to the imported page that we want the importer to not only ensure that image exists at the
+destination, but is updated to its newest version as well. Contrast this with the example of an 'author' snippet
+attached to blog posts, containing various fields of data about that person (e.g. bio, social media links); in this
+case, the author information is not really part of the blog post, and it's not expected that we would update it when
 running an import of blog posts.
 
 ### `WAGTAILTRANSFER_LOOKUP_FIELDS`
 
 ```python
-WAGTAILTRANSFER_LOOKUP_FIELDS = {'blog.author': ['first_name', 'surname']}
+WAGTAILTRANSFER_LOOKUP_FIELDS = {"blog.author": ["first_name", "surname"]}
 ```
 
 Specifies a list of fields to use for object lookups on the given models.
 
-Normally, imported objects will be assigned a random UUID known across all sites, so that those objects can be 
-recognised on subsequent imports and be updated rather than creating a duplicate. This behaviour is less useful for 
-models that already have a uniquely identifying field, or set of fields, such as an author identified by first name 
-and surname - if the same author exists on the source and destination site, but this was not the result of a previous 
-import, then the UUID-based matching will consider them distinct, and attempt to create a duplicate author record at the 
-destination. Adding an entry in WAGTAILTRANSFER_LOOKUP_FIELDS will mean that any imported instances of the given model 
+Normally, imported objects will be assigned a random UUID known across all sites, so that those objects can be
+recognised on subsequent imports and be updated rather than creating a duplicate. This behaviour is less useful for
+models that already have a uniquely identifying field, or set of fields, such as an author identified by first name
+and surname - if the same author exists on the source and destination site, but this was not the result of a previous
+import, then the UUID-based matching will consider them distinct, and attempt to create a duplicate author record at the
+destination. Adding an entry in WAGTAILTRANSFER_LOOKUP_FIELDS will mean that any imported instances of the given model
 will be looked up based on the specified fields, rather than by UUID.
 
 The default value for `WAGTAILTRANSFER_LOOKUP_FIELDS` is:
 
 ```python
 {
-    'taggit.tag': ['slug'],
-    'wagtailcore.locale': ["language_code"],
-    'contenttypes.contenttype': ['app_label', 'model'],
+    "taggit.tag": ["slug"],
+    "wagtailcore.locale": ["language_code"],
+    "contenttypes.contenttype": ["app_label", "model"],
 }
 ```
 
@@ -78,31 +78,33 @@ Overriding these values may result in issues as described above, particularly in
 ### `WAGTAILTRANSFER_NO_FOLLOW_MODELS`
 
 ```python
-WAGTAILTRANSFER_NO_FOLLOW_MODELS = ['wagtailcore.page', 'organisations.Company']
+WAGTAILTRANSFER_NO_FOLLOW_MODELS = ["wagtailcore.page", "organisations.Company"]
 ```
 
-Specifies a list of models that should not be imported by association when they are referenced from imported content. 
+Specifies a list of models that should not be imported by association when they are referenced from imported content.
 Defaults to `['wagtailcore.page', 'contenttypes.contenttype']`.
 
-By default, objects referenced within imported content will be recursively imported to ensure that those references are 
-still valid on the destination site. However, this is not always desirable - for example, if this happened for the Page 
-model, this would imply that any pages linked from an imported page would get imported as well, along with any pages 
-linked from those pages, and so on, leading to an unpredictable number of extra pages being added anywhere in the page 
-tree as a side-effect of the import. Models listed in WAGTAILTRANSFER_NO_FOLLOW_MODELS will thus be skipped in this 
-process, leaving the reference unresolved. The effect this has on the referencing page will vary according to the kind 
-of relation: nullable foreign keys, one-to-many and many-to-many relations will simply omit the missing object; 
-references in rich text and StreamField will become broken links (just as linking a page and then deleting it would); 
-while non-nullable foreign keys will prevent the object from being created at all (meaning that any objects referencing 
+By default, objects referenced within imported content will be recursively imported to ensure that those references are
+still valid on the destination site. However, this is not always desirable - for example, if this happened for the Page
+model, this would imply that any pages linked from an imported page would get imported as well, along with any pages
+linked from those pages, and so on, leading to an unpredictable number of extra pages being added anywhere in the page
+tree as a side-effect of the import. Models listed in WAGTAILTRANSFER_NO_FOLLOW_MODELS will thus be skipped in this
+process, leaving the reference unresolved. The effect this has on the referencing page will vary according to the kind
+of relation: nullable foreign keys, one-to-many and many-to-many relations will simply omit the missing object;
+references in rich text and StreamField will become broken links (just as linking a page and then deleting it would);
+while non-nullable foreign keys will prevent the object from being created at all (meaning that any objects referencing
 that object will end up with unresolved references, to be handled by the same set of rules).
 
-Note that these settings do not accept models that are defined as subclasses through multi-table inheritance - in 
+Note that these settings do not accept models that are defined as subclasses through multi-table inheritance - in
 particular, they cannot be used to define behaviour that only applies to specific subclasses of Page.
 
 
 ### `WAGTAILTRANSFER_FOLLOWED_REVERSE_RELATIONS`
 
 ```python
-WAGTAILTRANSFER_FOLLOWED_REVERSE_RELATIONS = [('wagtailimages.image', 'tagged_items', True)]
+WAGTAILTRANSFER_FOLLOWED_REVERSE_RELATIONS = [
+    ("wagtailimages.image", "tagged_items", True)
+]
 ```
 
 Specifies a list of models, their reverse relations to follow, and whether deletions should be synced, when identifying object references that should be imported to the destination site. Defaults to `[('wagtailimages.image', 'tagged_items', True)]`.
@@ -145,10 +147,9 @@ class MyCustomAdapter(FieldAdapter):
     pass
 
 
-@hooks.register('register_field_adapters')
+@hooks.register("register_field_adapters")
 def register_my_custom_adapter():
     return {models.Field: MyCustomAdapter}
-
 ```
 
 
@@ -168,18 +169,17 @@ from wagtail_transfer.serializers import PageSerializer
 
 from myapp.models import MyModel
 
-class MyModelCustomSerializer(PageSerializer):
 
+class MyModelCustomSerializer(PageSerializer):
     ignored_fields = PageSerializer.ignored_fields + [
-        'secret_field_1',
-        'environment_specific_data_field_123',
-        ...
+        "secret_field_1",
+        "environment_specific_data_field_123",
+        ...,
     ]
     pass
 
 
-@hooks.register('register_custom_serializers')
+@hooks.register("register_custom_serializers")
 def register_my_custom_serializer():
     return {MyModel: MyModelCustomSerializer}
-
 ```

docs/setup.md

@@ -7,13 +7,13 @@
 3. In your project's top-level urls.py, add:
 
         from wagtail_transfer import urls as wagtailtransfer_urls
-    
+
     and add:
 
         url(r'^wagtail-transfer/', include(wagtailtransfer_urls)),
-    
+
     to the `urlpatterns` list above `include(wagtail_urls)`.
-    
+
 4. Add the settings `WAGTAILTRANSFER_SOURCES` and `WAGTAILTRANSFER_SECRET_KEY` to your project settings.
     These are formatted as:
 
@@ -29,20 +29,19 @@
         }
 
         WAGTAILTRANSFER_SECRET_KEY = '7cd5de8229be75e1e0c2af8abc2ada7e'
-        
+
     However, it is best to store the `SECRET_KEY`s themselves in local environment variables for security.
-        
+
     `WAGTAILTRANSFER_SOURCES` is a dictionary defining the sites available to import from, and their secret keys.
 
-    `WAGTAILTRANSFER_SECRET_KEY` and the per-source `SECRET_KEY` settings are used to authenticate the communication between the 
-    source and destination instances; this prevents unauthorised users from using this API to retrieve sensitive data such 
-    as password hashes. The `SECRET_KEY` for each entry in `WAGTAILTRANSFER_SOURCES` must match that instance's 
+    `WAGTAILTRANSFER_SECRET_KEY` and the per-source `SECRET_KEY` settings are used to authenticate the communication between the
+    source and destination instances; this prevents unauthorised users from using this API to retrieve sensitive data such
+    as password hashes. The `SECRET_KEY` for each entry in `WAGTAILTRANSFER_SOURCES` must match that instance's
     `WAGTAILTRANSFER_SECRET_KEY`.
-
-Once you've followed these instructions for all your source and destination sites, you can start 
-[importing content](basic_usage.md). 
 
-If you need additional configuration - you want to configure which referenced models are updated, how models are identified 
+Once you've followed these instructions for all your source and destination sites, you can start
+[importing content](basic_usage.md).
+
+If you need additional configuration - you want to configure which referenced models are updated, how models are identified
 between Wagtail instances, or which models are pulled in and imported from references on an imported page, you can
 check out [how mappings and references work](how_it_works.md) and the [settings reference](settings.md).
-

ruff.toml

@@ -0,0 +1,54 @@
+target-version = "py38"
+
+exclude = [
+  "vendor",
+  "dist",
+  "build",
+  "venv",
+  ".venv",
+  ".tox",
+  ".git",
+  "__pycache__",
+  "node_modules",
+  "LC_MESSAGES",
+  "locale",
+  "migrations",
+]
+
+select = [
+  "B",   # flake8-bugbear
+  "BLE", # flake8-blind-except
+  "C4",  # flake8-comprehensions
+  "DJ",  # flake8-django
+  "E",   # pycodestyle errors
+  "F",   # pyflakes
+  "I",   # isort
+  "PGH", # pygrep-hooks
+  "RUF100",  # unused noqa
+  "S",   # flake8-bandit
+  "T20",  # flake8-print
+  "UP",   # pyupgrade
+  "W",    # pycodestyle warnings
+  "YTT",   # flake8-2020
+]
+
+fixable = ["C4", "E", "F", "I", "UP"]
+
+ignore = [
+  "E501",    # line-too-long (conflicts with formatter)
+  "W191",    # tab-indentation (conflicts with formatter)
+  "DJ008",   # model without __str__ method
+  "B019",    # functools.cache/lru_cache can lead to memory leaks
+  "S113",    # use of requests without timeout
+]
+
+[lint.per-file-ignores]
+"**/tests/**/*.py" = [
+  "S105",  # possible hardcoded password
+  "S106",  # possible hardcoded password
+  "DJ001", # use of null=True on CharField
+]
+".circleci/report_nightly_build_failure.py" = ["T201"]  # use of print()
+
+[isort]
+known-first-party = ["wagtail_transfer"]