Skip to content

Commit

Permalink
Improve rdl2jkan error handling and logging
Browse files Browse the repository at this point in the history
Import script now outputs errors into a log file
  • Loading branch information
lydiascarf committed Jun 12, 2024
1 parent 05b898c commit 2e874a3
Showing 1 changed file with 23 additions and 4 deletions.
27 changes: 23 additions & 4 deletions import/rdl2jkan.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,20 @@
import os
import re
import unicodedata
import logging
from pathlib import Path

import yaml

generated_dir = "generated"
datasets_output_dir = f"{generated_dir}/_datasets"
logname = f"{generated_dir}/error.log"

logging.basicConfig(filename=logname,
filemode='a',
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.DEBUG)

# Copied Django's slugify from https://github.com/django/django/blob/main/django/utils/text.py
# It's somewhat overkill for our case (which is just generating valid filenames), but it's relatively
Expand Down Expand Up @@ -336,8 +344,19 @@ def write_frontmatter(metadata, output_path):
datasets_json = json.load(input_file)
for dataset_json in datasets_json["datasets"]:
# Generate output
dataset_frontmatter = make_dataset_frontmatter(dataset_json)
# Write output
write_frontmatter(dataset_frontmatter, datasets_output_dir)
try:
# Write output
dataset_frontmatter = make_dataset_frontmatter(dataset_json)
write_frontmatter(dataset_frontmatter, datasets_output_dir)
except Exception as e:
logging.error(f"While writing {dataset_json.get("title", "a dataset with a missing title")} (id: {dataset_json.get("id", "missing")})",exc_info=e)


print("Done! Look in the import README to see what to do with these files")
print("\nAll done! Please enjoy your datasets :)\n",
"Datasets have been generated in: `import/generated/_datasets`",
"To include them in your JKAN site, run the following from `import`",
"\nmv generated/_datasets/* ../_datasets\n",
"This may overwrite the existing contents of `_datasets`.\n",
f"Issues with your input files have been logged to: `import/{logname}`",
"More info is availabile at `import/README.md`\n",
sep=os.linesep)

0 comments on commit 2e874a3

Please sign in to comment.