Skip to content

Commit

Permalink
package: adding support to InvenioRDM 9 packages (Closes #2)
Browse files Browse the repository at this point in the history
  • Loading branch information
@M3nin0
M3nin0 committed Aug 10, 2022
1 parent c975d0e commit ca57fcc
Show file tree
Hide file tree
Showing 8 changed files with 114 additions and 102 deletions.
109 changes: 58 additions & 51 deletions docs/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,37 +20,37 @@
# needs_sphinx = '1.0'

# Do not warn on external images.
suppress_warnings = ['image.nonlocal_uri']
suppress_warnings = ["image.nonlocal_uri"]

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.coverage',
'sphinx.ext.doctest',
'sphinx.ext.intersphinx',
'sphinx.ext.viewcode',
"sphinx.ext.autodoc",
"sphinx.ext.coverage",
"sphinx.ext.doctest",
"sphinx.ext.intersphinx",
"sphinx.ext.viewcode",
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
source_suffix = ".rst"

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'
master_doc = "index"

# General information about the project.
project = u'invenio-geographic-identifiers'
copyright = u'2022, GEO Secretariat'
author = u'GEO Secretariat'
project = "invenio-geographic-identifiers"
copyright = "2022, GEO Secretariat"
author = "GEO Secretariat"

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
Expand Down Expand Up @@ -94,7 +94,7 @@
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []
Expand All @@ -106,23 +106,21 @@
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------
html_theme = 'alabaster'
html_theme = "alabaster"

html_theme_options = {
'description': 'Geographic identifiers vocabulary for the InvenioRDM',
'github_user': 'inveniosoftware',
'github_repo': 'invenio-geographic-identifiers',
'github_button': False,
'github_banner': True,
'show_powered_by': False,
'extra_nav_links': {
'invenio-geographic-identifiers@GitHub':
'https://github.com/geo-knowledge-hub/'
'invenio-geographic-identifiers',
'invenio-geographic-identifiers@PyPI':
'https://pypi.python.org/pypi/'
'invenio-geographic-identifiers/',
}
"description": "Geographic identifiers vocabulary for the InvenioRDM",
"github_user": "inveniosoftware",
"github_repo": "invenio-geographic-identifiers",
"github_button": False,
"github_banner": True,
"show_powered_by": False,
"extra_nav_links": {
"invenio-geographic-identifiers@GitHub": "https://github.com/geo-knowledge-hub/"
"invenio-geographic-identifiers",
"invenio-geographic-identifiers@PyPI": "https://pypi.python.org/pypi/"
"invenio-geographic-identifiers/",
},
}

# The theme to use for HTML and HTML Help pages. See the documentation for
Expand Down Expand Up @@ -172,12 +170,12 @@

# Custom sidebar templates, maps document names to template names.
html_sidebars = {
'**': [
'about.html',
'navigation.html',
'relations.html',
'searchbox.html',
'donate.html',
"**": [
"about.html",
"navigation.html",
"relations.html",
"searchbox.html",
"donate.html",
]
}

Expand Down Expand Up @@ -226,20 +224,17 @@
# html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'invenio-geographic-identifiers_namedoc'
htmlhelp_basename = "invenio-geographic-identifiers_namedoc"

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
# 'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
# 'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
# 'preamble': '',

# Latex figure (float) alignment
# 'figure_align': 'htbp',
}
Expand All @@ -248,9 +243,13 @@
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'invenio-geographic-identifiers.tex',
u'invenio-geographic-identifiers Documentation',
u'GEO Secretariat', 'manual'),
(
master_doc,
"invenio-geographic-identifiers.tex",
"invenio-geographic-identifiers Documentation",
"GEO Secretariat",
"manual",
),
]

# The name of an image file (relative to this directory) to place at the top of
Expand Down Expand Up @@ -279,9 +278,13 @@
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'invenio-geographic-identifiers',
u'invenio-geographic-identifiers Documentation',
[author], 1)
(
master_doc,
"invenio-geographic-identifiers",
"invenio-geographic-identifiers Documentation",
[author],
1,
)
]

# If true, show URL addresses after external links.
Expand All @@ -295,11 +298,15 @@
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'invenio-geographic-identifiers',
u'invenio-geographic-identifiers Documentation',
author, 'invenio-geographic-identifiers',
'Geographic identifiers vocabulary for the InvenioRDM',
'Miscellaneous'),
(
master_doc,
"invenio-geographic-identifiers",
"invenio-geographic-identifiers Documentation",
author,
"invenio-geographic-identifiers",
"Geographic identifiers vocabulary for the InvenioRDM",
"Miscellaneous",
),
]

# Documents to append as an appendix to all manuals.
Expand All @@ -317,10 +324,10 @@

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
'python': ('https://docs.python.org/', None),
"python": ("https://docs.python.org/", None),
# TODO: Configure external documentation references, eg:
# 'Flask-Admin': ('https://flask-admin.readthedocs.io/en/latest/', None),
}

# Autodoc configuraton.
autoclass_content = 'both'
autoclass_content = "both"
24 changes: 14 additions & 10 deletions invenio_geographic_identifiers/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def get_config_for_ds(vocabulary, filepath=None, origin=None):
with open(filepath) as f:
config = yaml.safe_load(f).get(vocabulary)
if origin:
config["reader"]["args"]["origin"] = origin
config["readers"][0]["args"]["origin"] = origin

return config

Expand All @@ -52,15 +52,17 @@ def geoidentifiers():
def _process_vocab(config, num_samples=None):
"""Import a vocabulary."""
ds = DataStreamFactory.create(
reader_config=config["reader"],
readers_config=config["readers"],
transformers_config=config.get("transformers"),
writers_config=config["writers"],
)

success, errored = 0, 0
success, errored, filtered = 0, 0, 0
left = num_samples or -1
for result in ds.process():
left = left - 1
if result.filtered:
filtered += 1
if result.errors:
for err in result.errors:
click.secho(err, fg="red")
Expand All @@ -70,10 +72,10 @@ def _process_vocab(config, num_samples=None):
if left == 0:
click.secho(f"Number of samples reached {num_samples}", fg="green")
break
return success, errored
return success, errored, filtered


def _output_process(vocabulary, op, success, errored):
def _output_process(vocabulary, op, success, errored, filtered):
"""Outputs the result of an operation."""
total = success + errored

Expand All @@ -83,7 +85,9 @@ def _output_process(vocabulary, op, success, errored):

click.secho(
f"Vocabulary {vocabulary} {op}. Total items {total}. \n"
f"{success} items succeeded, {errored} contained errors.",
f"{success} items succeeded\n"
f"{errored} contained errors\n"
f"{filtered} were filtered.",
fg=color,
)

Expand All @@ -101,9 +105,9 @@ def import_vocab(vocabulary, filepath=None, origin=None, num_samples=None):
exit(1)

config = get_config_for_ds(vocabulary, filepath, origin)
success, errored = _process_vocab(config, num_samples)
success, errored, filtered = _process_vocab(config, num_samples)

_output_process(vocabulary, "imported", success, errored)
_output_process(vocabulary, "imported", success, errored, filtered)


@geoidentifiers.command()
Expand All @@ -122,9 +126,9 @@ def update(vocabulary, filepath=None, origin=None):
for w_conf in config["writers"]:
w_conf["args"]["update"] = True

success, errored = _process_vocab(config)
success, errored, filtered = _process_vocab(config)

_output_process(vocabulary, "updated", success, errored)
_output_process(vocabulary, "updated", success, errored, filtered)


@geoidentifiers.command()
Expand Down
60 changes: 31 additions & 29 deletions invenio_geographic_identifiers/contrib/geonames/datastreams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,35 +66,37 @@ def _entry_id(self, entry):
VOCABULARIES_DATASTREAM_READERS = {"geonames-reader": ZippedCSVReader}

DATASTREAM_CONFIG = {
"reader": {
"type": "geonames-reader",
"args": {
"csv_options": {
"fieldnames": [
"geonameid",
"name",
"asciiname",
"alternatenames",
"latitude",
"longitude",
"feature_class",
"feature_code",
"country_code",
"cc2",
"admin1_code",
"admin2_code",
"admin3_code",
"admin4_code",
"population",
"elevation",
"dem",
"timezone",
"modification_date",
],
"delimiter": "\t",
}
},
},
"readers": [
{
"type": "geonames-reader",
"args": {
"csv_options": {
"fieldnames": [
"geonameid",
"name",
"asciiname",
"alternatenames",
"latitude",
"longitude",
"feature_class",
"feature_code",
"country_code",
"cc2",
"admin1_code",
"admin2_code",
"admin3_code",
"admin4_code",
"population",
"elevation",
"dem",
"timezone",
"modification_date",
],
"delimiter": "\t",
}
},
}
],
"transformers": [{"type": "geonames-transformer"}],
"writers": [
{
Expand Down
3 changes: 1 addition & 2 deletions invenio_geographic_identifiers/datastreams/readers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import io
import zipfile

from invenio_vocabularies.datastreams.datastreams import StreamEntry
from invenio_vocabularies.datastreams.readers import BaseReader


Expand Down Expand Up @@ -41,7 +40,7 @@ def _iter(self, fp, *args, **kwargs):
else:
reader = csv.reader(csvfile, **self.csv_options)
for row in reader:
yield StreamEntry(row)
yield row

def read(self, item=None, *args, **kwargs):
"""Opens a `zip` archive or uses the given file pointer."""
Expand Down
1 change: 1 addition & 0 deletions invenio_geographic_identifiers/geoidentifiers/geoidentifiers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
]
),
# Service Layer
service_id="geoidentifier",
service_schema=GeographicIdentifiersSchema,
search_options=GeographicIdentifiersSearchOptions,
service_components=service_components,
Expand Down
6 changes: 3 additions & 3 deletions setup.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ python_requires = >=3.7
zip_safe = False
install_requires =
invenio-i18n>=1.2.0
invenio-vocabularies>=0.10.3,<0.11.0
invenio-vocabularies>=0.11.4,<0.12.0

[options.extras_require]
docs =
Expand All @@ -45,8 +45,8 @@ tests =
pytest-black>=0.3.0,<0.3.10
pytest-mock>=1.6.0
pytest-invenio>=1.4.0
invenio-db[postgresql,mysql,versioning]>=1.0.11,<2.0.0
invenio-app>=1.3.1,<2.0.0
invenio-db[postgresql,mysql,versioning]>=1.0.14,<2.0
invenio-app>=1.3.4
sphinx>=4.2.0,<5
elasticsearch7 =
invenio-search[elasticsearch7]>=1.4.2,<2.0
Expand Down
Loading

0 comments on commit ca57fcc

Please sign in to comment.