diff --git a/.gitignore b/.gitignore index a8186829..755229f5 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ GeoHealthCheck.conf # Data GeoHealthCheck/data.db + +# IDE +.idea diff --git a/.travis.yml b/.travis.yml index 3b43f7be..c0ee9d76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,17 +6,18 @@ python: sudo: false install: - - pip install -r requirements.txt + - pip install -e . - pip install -r requirements-dev.txt - - paver setup + - geohc create-instance script: - - echo -e "admin\ntest\ntest\nyou@example.com\nyou@example.com" | python GeoHealthCheck/models.py create + - geohc db create + - geohc db adduser -u admin -p test -e you@example.com -r admin - flake8 - - python GeoHealthCheck/models.py load tests/data/fixtures.json y - - python GeoHealthCheck/healthcheck.py - - python tests/run_tests.py + - geohc db load -f tests/data/fixtures.json -y + - geohc run-healthchecks + - geohc run-tests - cd docs && make html after-script: - - python GeoHealthCheck/models.py drop + - geohc db drop diff --git a/Dockerfile b/Dockerfile index 04ed6203..1ca2f69d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,22 +45,22 @@ ENV LC_ALL="en_US.UTF-8" \ GHC_SMTP_PASSWORD=None \ GHC_METADATA_CACHE_SECS=900 \ \ -# WSGI server settings, assumed is gunicorn \ -HOST=0.0.0.0 \ -PORT=80 \ -WSGI_WORKERS=4 \ -WSGI_WORKER_TIMEOUT=6000 \ -WSGI_WORKER_CLASS='eventlet' \ -\ -# GHC Core Plugins modules and/or classes, seldom needed to set: \ -# if not specified here or in Container environment \ -# all GHC built-in Plugins will be active. \ -#ENV GHC_PLUGINS 'GeoHealthCheck.plugins.probe.owsgetcaps,\ -# GeoHealthCheck.plugins.probe.wms, ...., ...\ -# GeoHealthCheck.plugins.check.checks' \ -\ -# GHC User Plugins, best be overridden via Container environment \ -GHC_USER_PLUGINS='' + # WSGI server settings, assumed is gunicorn \ + HOST=0.0.0.0 \ + PORT=80 \ + WSGI_WORKERS=4 \ + WSGI_WORKER_TIMEOUT=6000 \ + WSGI_WORKER_CLASS='eventlet' \ + \ + # GHC Core Plugins modules and/or classes, seldom needed to set: \ + # if not specified here or in Container environment \ + # all GHC built-in Plugins will be active. \ + #ENV GHC_PLUGINS 'GeoHealthCheck.plugins.probe.owsgetcaps,\ + # GeoHealthCheck.plugins.probe.wms, ...., ...\ + # GeoHealthCheck.plugins.check.checks' \ + \ + # GHC User Plugins, best be overridden via Container environment \ + GHC_USER_PLUGINS='' RUN apk add --no-cache --virtual .build-deps gcc build-base libxslt-dev libxml2-dev linux-headers postgresql-dev \ && apk add --no-cache bash postgresql-client libxslt libxml2 tzdata openntpd python3 python3-dev \ diff --git a/GeoHealthCheck/__init__.py b/GeoHealthCheck/__init__.py index e1614da1..f8ac92ff 100644 --- a/GeoHealthCheck/__init__.py +++ b/GeoHealthCheck/__init__.py @@ -27,12 +27,4 @@ # # ================================================================= -from util import read - - -def get_package_version(file_): - """get version from top-level package init""" - return read(file_) - - -__version__ = get_package_version('../VERSION') +__version__ = '0.8.dev0' diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index f7feba37..66a5440b 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -42,14 +42,15 @@ from flask_migrate import Migrate from itertools import chain -import views -from __init__ import __version__ -from enums import RESOURCE_TYPES -from factory import Factory -from init import App -from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient -from resourceauth import ResourceAuth -from util import send_email, geocode, format_checked_datetime, \ +from GeoHealthCheck import views +from GeoHealthCheck.__init__ import __version__ +from GeoHealthCheck.enums import RESOURCE_TYPES +from GeoHealthCheck.factory import Factory +from GeoHealthCheck.init import App +from GeoHealthCheck.models import Resource, Run, ProbeVars, CheckVars, Tag, \ + User, Recipient +from GeoHealthCheck.resourceauth import ResourceAuth +from GeoHealthCheck.util import send_email, geocode, format_checked_datetime, \ format_run_status, format_obj_value # Module globals for convenience @@ -76,7 +77,7 @@ # Should GHC Runner be run within GHC webapp? if CONFIG['GHC_RUNNER_IN_WEBAPP'] is True: LOGGER.info('Running GHC Scheduler in WebApp') - from scheduler import start_schedule + from GeoHealthCheck.scheduler import start_schedule # Start scheduler start_schedule() @@ -531,7 +532,8 @@ def add(): url = request.form['url'].strip() resources_to_add = [] - from healthcheck import sniff_test_resource, run_test_resource + from GeoHealthCheck.healthcheck import sniff_test_resource, \ + run_test_resource sniffed_resources = sniff_test_resource(CONFIG, resource_type, url) if not sniffed_resources: @@ -734,7 +736,7 @@ def test(resource_identifier): flash(gettext('Resource not found'), 'danger') return redirect(request.referrer) - from healthcheck import run_test_resource + from GeoHealthCheck.healthcheck import run_test_resource result = run_test_resource( resource) diff --git a/GeoHealthCheck/check.py b/GeoHealthCheck/check.py index e08770f5..6b9874a8 100644 --- a/GeoHealthCheck/check.py +++ b/GeoHealthCheck/check.py @@ -1,5 +1,5 @@ -from plugin import Plugin -from result import CheckResult +from GeoHealthCheck.plugin import Plugin +from GeoHealthCheck.result import CheckResult class Check(Plugin): diff --git a/GeoHealthCheck/cli.py b/GeoHealthCheck/cli.py new file mode 100644 index 00000000..1d26a97e --- /dev/null +++ b/GeoHealthCheck/cli.py @@ -0,0 +1,585 @@ +# ============================================================================ +# +# Authors: Rob van Loon +# +# Copyright (c) 2019 Rob van Loon +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= +import click + +from GeoHealthCheck.__init__ import __version__ + + +def verbose_echo(ctx, verbose_text): + if ctx.obj['VERBOSE']: + click.echo(verbose_text) + + +def abort_if_false(ctx, _, value): + if not value: + ctx.abort() + + +def sphinx_make(): + """return what command Sphinx is using for make""" + from os import name + if name == 'nt': + return 'make.bat' + return 'make' + + +@click.group() +@click.pass_context +@click.option('--verbose', '-v', is_flag=True, help='Verbose') +def cli(ctx, verbose): + ctx.ensure_object(dict) + ctx.obj['VERBOSE'] = verbose + + +@cli.command() +@click.pass_context +def version(ctx): + """Show the current version of GHC + """ + verbose_echo(ctx, 'GeoHC: get version') + click.echo(__version__) + + +@cli.command() +@click.pass_context +@click.option('--basepath', '-p', type=str, help='path to the directory to' + 'install the instance data and help files.') +def create_instance(ctx, basepath): + """Create an instance of GeoHealthCheck App + """ + verbose_echo(ctx, 'GeoHC: create instance') + import glob + import os + import shutil + from io import BytesIO + from pathlib import Path + from urllib.request import urlopen + import zipfile + basedir = os.path.abspath(os.path.dirname(__file__)) + if (basepath is not None): + basedir = os.path.abspath(basepath) + verbose_echo(ctx, 'Setting base install directory to %s' % basedir) + basedir_parent = os.path.normpath(str(Path(basedir).parent)) + config_file = os.path.normpath('%s/config_main.py' % basedir) + config_site = os.path.normpath(basedir_parent + '/instance/config_site.py') + + # setup dirs + if not os.path.exists(os.path.normpath('%s/static/lib' % basedir)): + os.mkdir(os.path.normpath('%s/static/lib' % basedir)) + if not os.path.exists(os.path.normpath('%s/instance' % basedir_parent)): + os.mkdir(os.path.normpath('%s/instance' % basedir_parent)) + # setup config + shutil.copy(config_file, config_site) + if not os.path.exists(os.path.normpath('%s/instance/data' + % basedir_parent)): + data_dir = os.path.normpath('%s/instance/data' % basedir_parent) + os.mkdir(data_dir, mode=0o777) + + skin = 'http://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v3.3.7+1.zip' # noqa + + skin_dirs = ['dist', 'vendor'] + need_to_fetch = False + + for skin_dir in skin_dirs: + skin_dir_path = os.sep.join( + ['startbootstrap-sb-admin-2-3.3.7-1', skin_dir]) + if not os.path.exists(skin_dir_path): + need_to_fetch = True + + if need_to_fetch: + zipstr = BytesIO(urlopen(skin).read()) + zipfile_obj = zipfile.ZipFile(zipstr) + zipfile_obj.extractall(os.path.normpath('%s/static/lib' % basedir)) + + for zf_mem in skin_dirs: + src_loc = os.path.normpath('%s/static/lib/' + 'startbootstrap-sb-admin-2-3.3.7-1/%s' + % (basedir, zf_mem)) + dest_loc = os.path.normpath('%s/static/lib/%s' % (basedir, zf_mem)) + if not os.path.exists(dest_loc): + shutil.move(src_loc, dest_loc) + else: + click.echo('directory already exists. Skipping') + + shutil.rmtree(os.path.normpath('%s/static/lib/' + 'startbootstrap-sb-admin-2-3.3.7-1' + % basedir)) + + # install sparklines to static/site/js + with open(os.path.normpath('%s/static/lib/' + 'jspark.js' % basedir), 'w') as f: + content = urlopen('http://ejohn.org/files/jspark.js').read().decode() + content.replace('red', 'green') + f.write(content) + + # install bootstrap-tagsinput to static/lib + click.echo('Getting select2') + select2 = 'https://github.com/select2/select2/archive/4.0.3.zip' + + zipstr = BytesIO(urlopen(select2).read()) + zipfile_obj = zipfile.ZipFile(zipstr) + zipfile_obj.extractall(os.path.normpath('%s/static/lib' % basedir)) + dirname = glob.glob(os.path.normpath('%s/static/lib/select2-*' + % basedir))[0] + dstdir = ''.join(dirname.rsplit('-', 1)[:-1]) + try: + shutil.move(dirname, dstdir) + except OSError: + shutil.rmtree(dstdir) + shutil.move(dirname, dstdir) + + # install leafletjs to static/lib + click.echo('Getting leaflet') + leafletjs = 'http://cdn.leafletjs.com/downloads/leaflet-0.7.5.zip' + + zipstr = BytesIO(urlopen(leafletjs).read()) + zipfile_obj = zipfile.ZipFile(zipstr) + zipfile_obj.extractall(os.path.normpath('%s/static/lib/leaflet' % basedir)) + + # install html5shiv to static/lib + with open(os.path.normpath('%s/static/lib/html5shiv.min.js' % basedir), + 'w') as f: + url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js' + content = urlopen(url).read().decode() + f.write(content) + + # install respond to static/lib + with open(os.path.normpath('%s/static/lib/respond.min.js' % basedir), + 'w') as f: + url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js' + content = urlopen(url).read().decode() + f.write(content) + + # build i18n .mo files + compile_translations(ctx, path=basedir) + + # build local docs + update_documentation(ctx, basepath=basedir_parent) + + # message user + click.echo('GeoHealthCheck is now built. Edit settings like the secret-key' + ' and the database connection string in %s' % config_site) + click.echo('before deploying the application. Alternatively, you can') + click.echo('start a development instance with ') + click.echo('"python GeoHealthCheck/app.py"') + verbose_echo(ctx, 'GeoHC: finished creating the instance.') + + +@cli.command() +@click.pass_context +@click.option('--host', '-h', default='0.0.0.0', help='IP to host the app') +@click.option('--port', '-p', default=8000, help='port number to host the app') +def serve(ctx, host, port): + """ + Run the app. Press 'ctrl-c' to exit again. + + This function is a wrapper around `python GeoHealthCheck/app.py` + """ + verbose_echo(ctx, 'GeoHC: serve') + click.echo('Press ctrl-c to exit.') + from os import system, chdir + chdir('GeoHealthCheck') + system(f"python app.py {host}:{port}") + + +@cli.command() +@click.pass_context +def create_secret_key(ctx): + """ + Create a secret key for the application. + """ + from codecs import encode + from os import urandom + click.echo('Secret key: \'%s\'' % encode(urandom(24), 'hex').decode()) + click.echo('Copy/paste this key to set the SECRET_KEY value in ' + 'instance/config_site.py') + + +@cli.command() +@click.option('-p', '--password', prompt=True, hide_input=True, + help='password') +@click.pass_context +def create_hash(ctx, password): + """Create a password hash + """ + from GeoHealthCheck.util import create_hash + token = create_hash(password) + click.echo('Copy/paste the entire token below for example to set password') + click.echo(token) + + +@cli.command() +@click.pass_context +def create_wsgi(ctx): + """Create an apache wsgi and conf file""" + verbose_echo(ctx, 'GeoHC: creating apache wsgi and conf files.') + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + instance = '%s%sinstance' % (basedir, os.sep) + verbose_echo(ctx, 'GeoHC: Files will be created in: %s' % instance) + + wsgi_script = '%s%sGeoHealthCheck.wsgi' % (instance, os.sep) + with open(wsgi_script, 'w') as ff: + ff.write('import sys\n') + ff.write('sys.path.insert(0, \'%s\')\n' % basedir) + ff.write('from GeoHealthCheck.app import APP as application') + verbose_echo(ctx, 'GeoHC: finished wsgi script.') + + wsgi_conf = '%s%sGeoHealthCheck.conf' % (instance, os.sep) + with open(wsgi_conf, 'w') as ff: + ff.write('WSGIScriptAlias / %s%sGeoHealthCheck.wsgi\n' % (instance, + os.sep)) + ff.write('\n' % (basedir, os.sep)) + ff.write('Order deny,allow\n') + ff.write('Allow from all\n') + ff.write('') + verbose_echo(ctx, 'GeoHC: finished conf file.') + + +@cli.command() +@click.pass_context +def update_docs(ctx): + """Update the spinx build of the documentation.""" + update_documentation(ctx) + + +def update_documentation(ctx, basepath=None): + verbose_echo(ctx, 'GeoHC: start building documentation.') + + import os + import shutil + from pathlib import Path + + basedir = str(Path(os.path.abspath(os.path.dirname(__file__))).parent) + if basepath: + basedir = os.path.abspath(basepath) + static_docs = os.path.normpath('%s/GeoHealthCheck/static/docs' % basedir) + docs = os.path.normpath('%s/docs' % basedir) + + if os.path.exists(static_docs): + verbose_echo(ctx, 'GeoHC: deleting previous doc directory.') + shutil.rmtree(static_docs) + + os.chdir(docs) + make = sphinx_make() + verbose_echo(ctx, 'GeoHC: cleaning old documentation.') + os.system('%s clean' % make) + verbose_echo(ctx, 'GeoHC: building new documentation.') + os.system('%s html' % make) + os.chdir(basedir) + + verbose_echo(ctx, 'GeoHC: copying documentation to build folder.') + source_html_dir = os.path.normpath('%s/docs/_build/html' % basedir) + shutil.copytree(source_html_dir, static_docs) + click.echo('GeoHC: finished refreshing documentation.') + + +@cli.command() +@click.pass_context +def clean(ctx): + """Clean Environment + """ + import os + import shutil + import tempfile + + basedir = os.path.abspath(os.path.dirname(__file__)) + static_docs = os.path.normpath('%s/GeoHealthCheck/static/docs' % basedir) + static_lib = os.path.normpath('%s/GeoHealthCheck/static/lib' % basedir) + tmp = os.path.normpath(tempfile.mkdtemp()) + + try: + shutil.rmtree(static_lib) + verbose_echo(ctx, 'GeoHC: removed %s' % static_lib) + except FileNotFoundError: + pass + try: + shutil.rmtree(tmp) + verbose_echo(ctx, 'GeoHC: removed temp directory: %s' % tmp) + except FileNotFoundError: + pass + try: + shutil.rmtree(static_docs) + verbose_echo(ctx, 'GeoHC: removed %s' % static_docs) + except FileNotFoundError: + pass + + click.echo('GeoHC: finished cleaning environment.') + + +@cli.command() +@click.pass_context +def lang_extract_translations(ctx): + """extract translations wrapped in _() or gettext()""" + verbose_echo(ctx, 'GeoHC: extracting translations') + import os + + pot_dir = os.path.normpath('GeoHealthCheck/translations/en/LC_MESSAGES') + verbose_echo(ctx, 'GeoHC: Translations directory: %s' % pot_dir) + if not os.path.exists(pot_dir): + pot_dir.makedirs() + + basedir = os.path.abspath(os.path.dirname(__file__)) + base_pot = os.path.normpath('%s/GeoHealthCheck/translations/en/' + 'LC_MESSAGES/messages.po' % basedir) + verbose_echo(ctx, 'GeoHC: starting translation') + os.system('pybabel extract -F babel.cfg -o %s GeoHealthCheck' % base_pot) + click.echo('GeoHC: finished extracting translations.') + + +@cli.command() +@click.option('-l', '--lang', required=True, help='2-letter language code') +@click.pass_context +def lang_add_language_catalogue(ctx, lang): + """adds new language profile""" + verbose_echo(ctx, 'GeoHC: Adding language catalogue.') + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + base_pot = os.path.normpath( + '%s/translations/en/LC_MESSAGES/messages.po' % basedir) + translations = os.path.normpath('%s/translations' % basedir) + verbose_echo(ctx, 'GeoHC: Base translation set: %s' % base_pot) + os.system('pybabel init -i %s -d %s -l %s' % ( + base_pot, translations, lang)) + click.echo('GeoHC: Finished translating: %s' % lang) + + +@cli.command() +@click.pass_context +def lang_compile_translations(ctx): + compile_translations(ctx) + + +def compile_translations(ctx, path=None): + """build language files""" + verbose_echo(ctx, 'GeoHC: start building language files.') + import os + basedir = path if path else os.path.abspath(os.path.dirname(__file__)) + translations = os.path.normpath('%s/translations' % basedir) + os.system('pybabel compile -d %s' % translations) + click.echo('GeoHC: Finished building language files.') + + +@cli.command() +@click.pass_context +def lang_update_translations(ctx): + """update language strings""" + verbose_echo(ctx, 'GeoHC: update translations.') + lang_extract_translations(ctx) + + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + base_pot = os.path.normpath( + '%s/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po' % basedir) + translations = os.path.normpath('%s/GeoHealthCheck/translations' % basedir) + os.system('pybabel update -i %s -d %s' % (base_pot, translations)) + click.echo('GeoHC: Finished updating translations.') + + +@cli.command() +@click.pass_context +def runner_daemon(ctx): + """Run the HealthCheck runner daemon scheduler""" + verbose_echo(ctx, 'GeoHC: going to run the scheduler daemon. Press ctrl-c ' + 'to stop.') + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + scheduler = basedir + '/scheduler.py' + os.system('python %s' % os.path.normpath(scheduler)) + + +@cli.command() +@click.pass_context +def run_healthchecks(ctx): + """Run all HealthChecks directly""" + verbose_echo(ctx, 'GeoHC: going to run all the healthchecks once.') + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + healtcheck = basedir + '/healtcheck.py' + os.system('python %s' % os.path.normpath(healtcheck)) + click.echo('GeoHC: Finished running the healthchecks.') + + +@cli.command() +@click.pass_context +def run_tests(ctx): + """Run all tests""" + import os + basedir = os.path.abspath(os.path.dirname(__file__)) + tests = basedir + '/run_tests.py' + os.system('python %s' % os.path.normpath(tests)) + + +@cli.command() +@click.pass_context +def build_wheel(ctx): + """Build a python wheel""" + verbose_echo(ctx, 'GeoHC: Build a python wheel') + from os import system + system('python setup.py bdist_wheel') + click.echo('GeoHC: Finished building a wheel.') + + +# Database sub commands +@cli.group() +@click.pass_context +@click.option('--verbose', '-v', is_flag=True, help='Verbose') +def db(ctx, verbose): + """Database subcommands""" + ctx.ensure_object(dict) + ctx.obj['VERBOSE'] = verbose + + +@db.command() +@click.pass_context +def create(ctx): + """Create the GHC database + + Note: you still need to add a user + """ + verbose_echo(ctx, 'GeoHC: create db') + from GeoHealthCheck.init import App + from GeoHealthCheck.models import db_commit + verbose_echo(ctx, 'GeoHC: get database') + DB = App.get_db() + verbose_echo(ctx, 'GeoHC: create all tables') + DB.create_all() + db_commit() + DB.session.remove() + click.echo('Database is created. Use `geohc db-adduser` to add users to ' + 'the database.') + + +@db.command() +@click.pass_context +@click.option('-u', '--user', type=str, help='username', prompt=True) +@click.option('-e', '--email', type=str, help='email address', prompt=True) +@click.option('-p', '--password', type=str, help='password', prompt=True, + hide_input=True, confirmation_prompt=True) +@click.option('-r', '--role', type=click.Choice(['admin', 'user']), + prompt=True, help='role for this user') +def adduser(ctx, user, email, password, role): + """Add an user to the database + """ + verbose_echo(ctx, 'GeoHC: add user to database') + from GeoHealthCheck.init import App + from GeoHealthCheck.models import User, db_commit + verbose_echo(ctx, 'GeoHC: get database') + DB = App.get_db() + verbose_echo(ctx, 'GeoHC: create user') + user_to_add = User(user, password, email, role=role) + verbose_echo(ctx, 'GeoHC: adding user to database') + DB.session.add(user_to_add) + db_commit() + DB.session.remove() + click.echo(f'User {user} is added.') + + +@db.command() +@click.pass_context +@click.option('-y', '--yes', is_flag=True, callback=abort_if_false, + expose_value=False, help='Confirm dropping tables', + prompt='This will drop the tables in the database. Are you ' + 'sure?') +def drop(ctx): + """Drop the current database + """ + verbose_echo(ctx, 'GeoHC: drop db') + click.confirm("This will drop the tables in the database. Are you sure?", + abort=True) + verbose_echo(ctx, 'User confirmed dropping tables') + from GeoHealthCheck.init import App + from GeoHealthCheck.models import db_commit + verbose_echo(ctx, 'GeoHC: get database') + DB = App.get_db() + verbose_echo(ctx, 'GeoHC: dropping all tables') + DB.drop_all() + db_commit() + DB.session.remove() + click.echo('Database dropped all tables.') + + +@db.command() +@click.pass_context +@click.option('-f', '--file', type=click.Path(), multiple=False, required=True, + help='Path to the file to load into the database. MUST be JSON.') +@click.option('-y', '--yes', is_flag=True, callback=abort_if_false, + expose_value=False, help='Confirm dropping old content.', + prompt='WARNING: all database data will be lost. Proceed?') +def load(ctx, file): + """Load JSON into the database + + e.g. test/data/fixtures.json + """ + verbose_echo(ctx, 'GeoHC: load data into db') + verbose_echo(ctx, 'User confirmed loading new data and losing old data.') + from GeoHealthCheck.init import App + from GeoHealthCheck.models import load_data + DB = App.get_db() + if file[-5:].lower() != '.json': + click.echo("File must have '.json' file extension. Aborting import.") + ctx.exit() + verbose_echo(ctx, 'Start loading file.') + load_data(file) + verbose_echo(ctx, 'Data loaded!') + DB.session.remove() + click.echo('Finished loading data.') + + +@db.command() +@click.pass_context +def flush(ctx): + """Flush runs: remove old data over retention date. + """ + verbose_echo(ctx, 'GeoHC: flush old runs from database.') + from GeoHealthCheck.models import flush_runs + flush_runs() + click.echo('Finished flushing old runs from database.') + + +@db.command() +@click.pass_context +def upgrade(ctx): + """Upgrade the database + """ + verbose_echo(ctx, 'GeoHC: upgrade db') + from os import system, chdir + chdir('GeoHealthCheck') + system('python manage.py db upgrade') + click.echo('Upgrade DB finished.') + + +@db.command() +@click.pass_context +def export(ctx): + click.exho('GeoHC: todo- write export.') + + +if __name__ == '__main__': + cli() diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index 8897fa56..1b2dac59 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -44,12 +44,12 @@ from owslib.csw import CatalogueServiceWeb from owslib.sos import SensorObservationService -from init import App -from enums import RESOURCE_TYPES -from models import Resource, Run -from probe import Probe -from result import ResourceResult -from notifications import notify +from GeoHealthCheck.init import App +from GeoHealthCheck.enums import RESOURCE_TYPES +from GeoHealthCheck.models import Resource, Run +from GeoHealthCheck.probe import Probe +from GeoHealthCheck.result import ResourceResult +from GeoHealthCheck.notifications import notify LOGGER = logging.getLogger(__name__) APP = App.get_app() diff --git a/GeoHealthCheck/init.py b/GeoHealthCheck/init.py index c46466a5..43003e88 100644 --- a/GeoHealthCheck/init.py +++ b/GeoHealthCheck/init.py @@ -67,7 +67,11 @@ def init(): # Read and override configs app.config.from_pyfile('config_main.py') - app.config.from_pyfile('../instance/config_site.py') + try: + app.config.from_pyfile('../instance/config_site.py') + except FileNotFoundError: + # for Docker build + app.config.from_pyfile('/GeoHealthCheck/instance/config_site.py') app.config.from_envvar('GHC_SETTINGS', silent=True) # Global Logging config diff --git a/GeoHealthCheck/manage.py b/GeoHealthCheck/manage.py index 82c029df..1e14569f 100644 --- a/GeoHealthCheck/manage.py +++ b/GeoHealthCheck/manage.py @@ -39,7 +39,7 @@ from flask_script import Manager from flask_migrate import Migrate, MigrateCommand -from init import App +from GeoHealthCheck.init import App DB = App.get_db() APP = App.get_app() diff --git a/GeoHealthCheck/migrations/README.md b/GeoHealthCheck/migrations/README.md index 81db6740..5f0d6527 100755 --- a/GeoHealthCheck/migrations/README.md +++ b/GeoHealthCheck/migrations/README.md @@ -6,7 +6,7 @@ and Flask-Script. Users should be able to upgrade existing installs via: # In top dir of installation - paver upgrade + geohc db upgrade The `versions` dir contains the various upgrades. These were initially created using the Alembic `autogenerate` facility @@ -39,7 +39,7 @@ Subsequently the upgrade can be performed using: python manage.py db upgrade # or the equivalent (for users) - paver upgrade + geohc db upgrade ## Revisions diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 1ec7d7e5..33f65e51 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -38,11 +38,11 @@ from sqlalchemy.orm import deferred from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound -import util -from enums import RESOURCE_TYPES -from factory import Factory -from init import App -from resourceauth import ResourceAuth +from GeoHealthCheck import util +from GeoHealthCheck.enums import RESOURCE_TYPES +from GeoHealthCheck.factory import Factory +from GeoHealthCheck.init import App +from GeoHealthCheck.resourceauth import ResourceAuth from wtforms.validators import Email, ValidationError from owslib.util import bind_url @@ -893,6 +893,10 @@ def db_commit(): if len(sys.argv) > 1: if sys.argv[1] == 'create': + LOGGER.warning('models.py create is deprecated since 0.8.0. Please' + ' use cli: `geohc db create` and ' + '`geohc db adduser`', + DeprecationWarning) print('Creating database objects') DB.create_all() @@ -915,11 +919,17 @@ def db_commit(): user_to_add = User(username, password1, email1, role='admin') DB.session.add(user_to_add) db_commit() + elif sys.argv[1] == 'drop': + LOGGER.warning('models.py drop is deprecated since 0.8.0. Please ' + 'use cli: `geohc db drop`', DeprecationWarning) print('Dropping database objects') DB.drop_all() db_commit() + elif sys.argv[1] == 'load': + LOGGER.warning('models.py load is deprecated since 0.8.0. Please ' + 'use cli: `geohc db load`', DeprecationWarning) print('Load database from JSON file (e.g. tests/fixtures.json)') if len(sys.argv) > 2: file_path = sys.argv[2] @@ -943,9 +953,12 @@ def db_commit(): print('Provide path to JSON file, e.g. tests/fixtures.json') elif sys.argv[1] == 'run': - print('NOTICE: models.py no longer here.') - print('Use: python healthcheck.py or upcoming cli.py') + LOGGER.warning('models.py run is deprecated. Please use `python ' + 'healthcheck.py`', DeprecationWarning) + elif sys.argv[1] == 'flush': + LOGGER.warning('models.py flush is deprecated since 0.8.0. Please ' + 'use cli: `geohc db flush`', DeprecationWarning) flush_runs() DB.session.remove() diff --git a/GeoHealthCheck/notifications.py b/GeoHealthCheck/notifications.py index 8b22af4a..bf6e4e9f 100644 --- a/GeoHealthCheck/notifications.py +++ b/GeoHealthCheck/notifications.py @@ -35,7 +35,7 @@ import requests from flask_babel import gettext -from util import render_template2 +from GeoHealthCheck.util import render_template2 LOGGER = logging.getLogger(__name__) diff --git a/GeoHealthCheck/plugin.py b/GeoHealthCheck/plugin.py index 3534c4b3..da73529a 100644 --- a/GeoHealthCheck/plugin.py +++ b/GeoHealthCheck/plugin.py @@ -1,9 +1,12 @@ -from factory import Factory +from GeoHealthCheck.factory import Factory import logging import inspect from collections.abc import Mapping import copy -from init import App +try: + from init import App +except ImportError: + from GeoHealthCheck.init import App LOGGER = logging.getLogger(__name__) diff --git a/GeoHealthCheck/probe.py b/GeoHealthCheck/probe.py index 2501b593..eaa53781 100644 --- a/GeoHealthCheck/probe.py +++ b/GeoHealthCheck/probe.py @@ -4,10 +4,10 @@ import datetime import requests -from factory import Factory -from init import App -from plugin import Plugin -from result import ProbeResult +from GeoHealthCheck.factory import Factory +from GeoHealthCheck.init import App +from GeoHealthCheck.plugin import Plugin +from GeoHealthCheck.result import ProbeResult LOGGER = logging.getLogger(__name__) diff --git a/GeoHealthCheck/resourceauth.py b/GeoHealthCheck/resourceauth.py index ba1982eb..16b07379 100644 --- a/GeoHealthCheck/resourceauth.py +++ b/GeoHealthCheck/resourceauth.py @@ -1,9 +1,10 @@ import json import logging -from plugin import Plugin -from factory import Factory -from util import encode, decode -from init import App +from GeoHealthCheck.plugin import Plugin +from GeoHealthCheck.factory import Factory +from GeoHealthCheck.util import encode, decode +from GeoHealthCheck.init import App + APP = App.get_app() LOGGER = logging.getLogger(__name__) diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index 62fa9c2c..83846786 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -33,14 +33,14 @@ import random import string from datetime import datetime, timedelta -from models import Resource, ResourceLock, flush_runs -from healthcheck import run_resource +from GeoHealthCheck.models import Resource, ResourceLock, flush_runs +from GeoHealthCheck.healthcheck import run_resource from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.jobstores.base import JobLookupError from apscheduler.events import \ EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, \ EVENT_JOB_MISSED, EVENT_JOB_ERROR -from init import App +from GeoHealthCheck.init import App LOGGER = logging.getLogger(__name__) DB = App.get_db() diff --git a/GeoHealthCheck/views.py b/GeoHealthCheck/views.py index d5102e7c..0f3db82c 100644 --- a/GeoHealthCheck/views.py +++ b/GeoHealthCheck/views.py @@ -28,12 +28,12 @@ # ================================================================= import logging -import models -import util +from GeoHealthCheck import models +from GeoHealthCheck import util from sqlalchemy import text -from plugin import Plugin -from factory import Factory -from init import App +from GeoHealthCheck.plugin import Plugin +from GeoHealthCheck.factory import Factory +from GeoHealthCheck.init import App APP = App.get_app() LOGGER = logging.getLogger(__name__) diff --git a/README.md b/README.md index 8891d525..0719b947 100644 --- a/README.md +++ b/README.md @@ -13,19 +13,19 @@ Easiest is [to run GHC using Docker](https://github.com/geopython/GeoHealthCheck Below a quick overview of a manual install on Unix-based systems like Apple Mac and Linux. ```bash -virtualenv GeoHealthCheck && cd $_ -. bin/activate +python3 -m venv ghc_venv +. ghc_venv/bin/activate git clone https://github.com/geopython/GeoHealthCheck.git cd GeoHealthCheck -pip install Paver +pip install . # setup installation -paver setup +geohc create-instance # generate secret key -paver create_secret_key +geohc create-secret-key # setup local configuration (overrides GeoHealthCheck/config_main.py) vi instance/config_site.py # edit at least secret key: -# - SECRET_KEY # copy/paste result string from paver create_secret_key +# - SECRET_KEY # copy/paste result string from geohc create-secret-key # Optional: edit other settings or leave defaults # - SQLALCHEMY_DATABASE_URI @@ -36,34 +36,46 @@ vi instance/config_site.py # - GHC_SITE_TITLE # - GHC_MAP (or use default settings) -# setup database and superuser account interactively -paver create +# setup database +geohc db create + +# create superuser account interactively +geohc db adduser # start webserver with healthcheck runner daemon inside # (default is 0.0.0.0:8000) -python GeoHealthCheck/app.py +geohc serve # or start webserver on another port -python GeoHealthCheck/app.py 0.0.0.0:8881 +geohc serve -p 8881 # or start webserver on another IP -python GeoHealthCheck/app.py 192.168.0.105:8001 +geohc serve -h 192.168.0.105 -p 8001 # OR start webserver and separate runner daemon (scheduler) process vi instance/config_site.py # GHC_RUNNER_IN_WEBAPP = False -python GeoHealthCheck/scheduler.py & -python GeoHealthCheck/app.py +geohc runner-daemon & +geohc serve # next: use a real webserver or preferably Docker for production # other commands # # drop database -python GeoHealthCheck/models.py drop +geohc db drop # load data in database (WARN: deletes existing data!) # See example data .json files in tests/data -python GeoHealthCheck/models.py load <.json data file> [y/n] +geohc db load -f <.json data file> -y + +# More help on the `geohc` cli command: +geohc --help + +# More help on a specific command +geohc db load --help ``` -More in the [full GHC documentation](http://docs.geohealthcheck.org/). \ No newline at end of file +**Note for developers:** instead of `pip install .`, you might want to install with the *editable* +option specified: `pip install -e .` + +More in the [full GHC documentation](http://docs.geohealthcheck.org/). diff --git a/VERSION b/VERSION deleted file mode 100644 index 3128c152..00000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.8.dev0 diff --git a/docker/README.md b/docker/README.md index a1b0977f..8c2ddbd4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -97,6 +97,9 @@ docker-compose -f docker-compose.yml up [-d] # go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host) ``` +This setup maps the sqlite database to a docker volume (`ghc_sqlitedb`) that is stored on the +machine (outside the docker container). To get more information about the volume, run +`docker volume inspect ghc_sqlitedb`. ### Using PostGIS DB @@ -235,11 +238,11 @@ docker exec -it docker_geohealthcheck_1 bash source /venv/bin/activate . cd /GeoHealthCheck/ -# next can use Paver commands e.g. DB upgrade -paver upgrade +# next can use cli commands e.g. DB upgrade +geohc db-upgrade etc ``` -NB: database upgrades (`paver upgrade`) +NB: database upgrades (`geohc db upgrade`) are always performed automatically when running GHC via Docker. diff --git a/docker/scripts/configure.sh b/docker/scripts/configure.sh index 8256ff23..54d33c02 100755 --- a/docker/scripts/configure.sh +++ b/docker/scripts/configure.sh @@ -15,7 +15,8 @@ echo "Using DB_TYPE=${DB_TYPE}" # Create DB shorthand function create_db() { pushd /GeoHealthCheck/ - paver create -u ${ADMIN_NAME} -p ${ADMIN_PWD} -e ${ADMIN_EMAIL} + geohc db create + geohc db adduser -u ${ADMIN_NAME} -p ${ADMIN_PWD} -e ${ADMIN_EMAIL} -r admin popd } diff --git a/docker/scripts/install.sh b/docker/scripts/install.sh index 16532c83..07c77ec3 100755 --- a/docker/scripts/install.sh +++ b/docker/scripts/install.sh @@ -11,9 +11,10 @@ source /venv/bin/activate pip install -I -r /GeoHealthCheck/docker/scripts/requirements.txt cd /GeoHealthCheck +pip install . # Sets up GHC itself -paver setup +geohc create-instance --basepath /GeoHealthCheck/GeoHealthCheck mv /config_site.py /GeoHealthCheck/instance/config_site.py # Copy possible Plugins into app tree diff --git a/docker/scripts/requirements.txt b/docker/scripts/requirements.txt index fd68b637..30ed0374 100644 --- a/docker/scripts/requirements.txt +++ b/docker/scripts/requirements.txt @@ -1,5 +1,3 @@ -Paver==1.3.4 -Sphinx==1.8.1 psycopg2==2.7.5 eventlet==0.24.1 gunicorn==19.9.0 diff --git a/docker/scripts/run-runner.sh b/docker/scripts/run-runner.sh index bfcaa8af..f34ee4fe 100755 --- a/docker/scripts/run-runner.sh +++ b/docker/scripts/run-runner.sh @@ -12,10 +12,6 @@ echo "START /run-runner.sh" # that schedules healthcheck jobs source /venv/bin/activate . -# Make sure PYTHONPATH includes GeoHealthCheck -export PYTHONPATH=/GeoHealthCheck/GeoHealthCheck:$PYTHONPATH - -cd /GeoHealthCheck -paver runner_daemon +geohc runner-daemon echo "END /run-runner.sh" diff --git a/docker/scripts/run-web.sh b/docker/scripts/run-web.sh index 9e25ed23..ca83f70e 100755 --- a/docker/scripts/run-web.sh +++ b/docker/scripts/run-web.sh @@ -12,11 +12,9 @@ echo "START /run-web.sh" source /venv/bin/activate . # Make sure PYTHONPATH includes GeoHealthCheck -export PYTHONPATH=/GeoHealthCheck/GeoHealthCheck:$PYTHONPATH +export PYTHONPATH=/GeoHealthCheck:$PYTHONPATH -cd /GeoHealthCheck - -paver upgrade +geohc db upgrade # SCRIPT_NAME should not have value '/' [ "${SCRIPT_NAME}" = '/' ] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" diff --git a/docs/admin.rst b/docs/admin.rst index 2ee3b142..d53ddac3 100644 --- a/docs/admin.rst +++ b/docs/admin.rst @@ -7,12 +7,23 @@ This chapter describes maintenance tasks for the administrator of a GHC instance There is a separate :ref:`userguide` that provides guidance to the end-user to configure the actual Resource healthchecks. -Each of the sections below is geared at a specific administrative task area. +Each of the sections below is geared at a specific administrative task area. We +assume you have installed GeoHealthCheck with `pip install`, and that you have +acces to the command line interface `geohc`. You can get help from the cli via :: + + geohc --help + +This will list all available commands. To get help with a specific command, run +`geohc <> --help`, e.g. :: + + geohc db adduser --help + +This will show you all the (required) options to add a user to the database. Database -------- -For database administration the following commands are available. +For database administration the following commands are available: create db ......... @@ -21,7 +32,18 @@ To create the database execute the following: Open a command line, (if needed activate your virtualenv), and do :: - python GeoHealthCheck/models.py create + geohc db create + +add user +........ + +To add a new user to the database: + +Open a command line, (if needed activate your virtualenv), and do :: + + geohc db adduser + # or + geohc db adduser -u username -p password -e email@address.com -r admin drop db ....... @@ -30,7 +52,7 @@ To delete the database execute the following, however you will loose all your in Open a command line, (if needed activate your virtualenv), and do :: - python GeoHealthCheck/models.py drop + geohc db drop Note: you need to create a Database again before you can start GHC again. @@ -39,10 +61,27 @@ load data To load a JSON data file, do (WARN: deletes existing data!) :: - python GeoHealthCheck/models.py load [y/n] + geohc db load --file my_data.json +You will be asked for confirmation. You can also specify the `-y` or `--yes` +flag to indicate you are sure and to skip the confirmation question. Hint: see `tests/data` for example JSON data files. + +flush +..... + +To flush the old records from the system, you can flush the records :: + + geohc db flush + +upgrade +....... + +To upgrade the database structure (might be necessary with a new release of GeoHealthCheck) :: + + geohc db upgrade + export data ........... @@ -92,10 +131,9 @@ Or more compact within the root dir of your GHC installation: :: >>> create_hash('mynewpassword') '$pbkdf2-sha256$29000$8X4PAUAIAcC4V2rNea9Vqg$XnMx1SfEiBzBAMOQOOC7uxCcyzVuKaHENLj3IfXvfu0' -Or even more compact within the root dir of your GHC installation via Paver: :: +Or even more compact within the root dir of your GHC installation via the cli: :: - $ paver create_hash -p mypass - ---> pavement.create_hash + $ geohc create_hash -p mypass Copy/paste the entire token below for example to set password $pbkdf2-sha256$29000$FkJoTYnxPqc0pjQG4HxP6Q$C3SZb8jqtM7zKS1DSLcouc/CL9XMI9cL5xT6DRTOEd4 @@ -110,4 +148,5 @@ Build Documentation ------------------- Open a command line, (if needed activate your virtualenv) and move into the directory ``GeoHealthCheck/doc/``. -In there, type ``make html`` plus ENTER and the documentation should be built locally. +In there, type ``make html`` plus ENTER and the documentation should be built locally. Or you can +use the cli: `geohc refresh-docs`. diff --git a/docs/conf.py b/docs/conf.py index b51008ec..dd91ba54 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,6 +14,7 @@ import sys import os +import re # indicate Sphinx is building (to replace @Config decorators) os.environ['SPHINX_BUILD'] = '1' @@ -55,8 +56,10 @@ # built documents. # # The short X.Y version. -with open('../VERSION') as ff: - version = ff.read().strip() +with open('../GeoHealthCheck/__init__.py') as ff: + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + ff.read(), re.M) + version = version_match.group(1) # The full version, including alpha/beta/rc tags. release = version diff --git a/docs/config.rst b/docs/config.rst index 14e2c0e7..c4395ec6 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -14,7 +14,7 @@ a configuration file in the environment settings that will override settings in The configuration options are: - **SQLALCHEMY_DATABASE_URI**: the database configuration. See the SQLAlchemy documentation for more info -- **SECRET_KEY**: secret key to set when enabling authentication. Use the output of ``paver create_secret_key`` to set this value +- **SECRET_KEY**: secret key to set when enabling authentication. Use the output of ``geohc create-secret-key`` to set this value - **GHC_RETENTION_DAYS**: the number of days to keep Run history - **GHC_PROBE_HTTP_TIMEOUT_SECS**: stop waiting for the first byte of a Probe response after the given number of seconds - **GHC_MINIMAL_RUN_FREQUENCY_MINS**: minimal run frequency for Resource that can be set in web UI @@ -46,7 +46,7 @@ The configuration options are: Example on overriding the configuration with an environment variable: :: export GHC_SETTINGS=/tmp/my_GHC_settings.py - paver run_tests + geohc run-tests As an example: the `my_GHC_settings.py` file can contain a single line to define a test database: :: @@ -106,7 +106,7 @@ This is the preferred mode as each `Resource` can have its own schedule (configu via Dashboard) and `cron` has dependencies on local environment. Later versions may phase out cron-scheduling completely. -The **GHC Runner** can be run via the command `paver runner_daemon` or can run internally within +The **GHC Runner** can be run via the command `geohc runner-daemon` or can run internally within the **GHC Webapp** by setting the config variable **GHC_RUNNER_IN_WEBAPP** to `True` (the default). NB it is still possible to run GHC as in the pre-v0.5.0 mode using cron-jobs: just run the **GHC Webapp** with **GHC_RUNNER_IN_WEBAPP** set to `False` and have your cron-jobs scheduled. @@ -148,7 +148,7 @@ Compiling Language Files ........................ At runtime compiled versions, `*.mo` files, of the language-files are used. -Easiest to compile is via: `paver compile_translations` in the project root dir. +Easiest to compile is via: `geohc compile-translations` in the project root dir. This basically calls ``pybabel compile` with the proper options. Now you can e.g. test your new translations by starting GHC. @@ -170,7 +170,7 @@ Missing translations will have `msgstr ""` like in this excerpt: :: Next all empty `msgstr`s can be filled. -Updating is easiest using the command `paver update_translations` within the root dir of the project. +Updating is easiest using the command `geohc update-translations` within the root dir of the project. This will basically call `pybabel extract` followed by `pybabel update` with the proper parameters. Customizing the Score Matrix diff --git a/docs/install.rst b/docs/install.rst index f93aad90..d2c85615 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -27,9 +27,8 @@ GeoHealthCheck is built on the awesome Flask micro-framework and uses `APScheduler` is used to run scheduled healthchecks. -These dependencies are automatically installed (see below). ``Paver`` is used -for installation and management. ``Cron`` was used for scheduling the actual -healthchecks before v0.5.0. +These dependencies are automatically installed (see below). A command line interface is provided + with `click`. ``Cron`` was used for scheduling the actual healthchecks before v0.5.0. Starting from version v0.8.0.0 GeoHealthCheck requires **python 3**. Previous versions require **python 2**. @@ -55,20 +54,20 @@ Install git clone https://github.com/geopython/GeoHealthCheck.git cd GeoHealthCheck - # install paver dependency for admin tool - pip install Paver + # install + pip install . # setup app - paver setup + geohc create-instance # create secret key to use for auth - paver create_secret_key + geohc create-secret-key # almost there! Customize config vi instance/config_site.py # edit: # - SQLALCHEMY_DATABASE_URI - # - SECRET_KEY # from paver create_secret_key + # - SECRET_KEY # from geohc create-secret-key # - GHC_RETENTION_DAYS # - GHC_SELF_REGISTER # - GHC_NOTIFICATIONS @@ -83,16 +82,25 @@ Install # - GHC_MAP # or use default settings # init database - python GeoHealthCheck/models.py create + geohc db-create + + # create an (admin) user account + geohc db-adduser -u my_user_name -p my_pass -e e@mail.com -r admin # start web-app - python GeoHealthCheck/app.py # http://localhost:8000/ + geohc serve # when you are done, you can exit the virtualenv deactivate NB GHC supports internal scheduling, no cronjobs required. +Developing +.......... +If you plan to develop with the GeoHealthCheck code base, you might want to install with the +*editable* option specified: `pip install -e`. This gives you access to the command line interface, +and you can still edit and run the code. + .. _upgrade: Upgrade @@ -103,15 +111,15 @@ An existing GHC database installation can be upgraded with: .. code-block:: bash # In the top directory (e.g. the topdir cloned from github) - paver upgrade + geohc db-upgrade # Notice any output, in particular errors Notes: * **Always backup your database first!!** -* make sure Flask-Migrate is installed (see requirements.txt), else: `pip install Flask-Migrate==2.5.2`, but best is to run `paver setup` also for other dependencies -* upgrading is "smart": you can always run `paver upgrade`, it has no effect when DB is already up to date +* make sure Flask-Migrate is installed (see requirements.txt), else: `pip install Flask-Migrate==2.5.2`, but best is to run `geohc create-instance` also for other dependencies +* upgrading is "smart": you can always run `geohc db upgrade`, it has no effect when DB is already up to date * when upgrading from earlier versions without Plugin-support: - adapt your `config_site.py` to Plugin settings from `config_main.py` @@ -119,7 +127,7 @@ Notes: When running with Docker see the `Docker Readme `_ -how to run `paver upgrade` within your Docker Container. +how to run `geohc upgrade` within your Docker Container. Upgrade notes v0.5.0 .................... @@ -136,6 +144,13 @@ the `paver upgrade` command. Also password recovery was changed: a user can crea a unique, personal URL that GHC sends by email. This requires a working email configuration and a reachable `SITE_URL` config value. See :ref:`admin_user_mgt` for solving password problems. +Upgrade notes v0.8.0 +.................... + +In previous version GHC used `paver` to control the setup and administration of +the application. With this version GHC switched `click` and the cli commands +changed to `geohc`. Type `geohc --help` to get more information. + Running ------- @@ -144,9 +159,9 @@ Start using Flask's built-in WSGI server: .. code-block:: bash - python GeoHealthCheck/app.py # http://localhost:8000 - python GeoHealthCheck/app.py 0.0.0.0:8881 # http://localhost:8881 - python GeoHealthCheck/app.py 192.168.0.105:8957 # http://192.168.0.105:8957 + geohc serve # http://localhost:8000 + geohc serve --port 8881 # http://localhost:8881 + geohc serve --host 192.168.0.105 --port 8957 # http://192.168.0.105:8957 This runs the (Flask) **GHC Webapp**, by default with the **GHC Runner** (scheduled healthchecker) internally. @@ -157,10 +172,10 @@ From the command-line run both processes, e.g. in background or different termin .. code-block:: bash # run GHC Runner, here in background - python GeoHealthCheck/scheduler.py & + geohc runner-daemon & # run GHC Webapp for http://localhost:8000 - python GeoHealthCheck/app.py + geohc serve To enable in Apache, use ``GeoHealthCheck.wsgi`` and configure in Apache diff --git a/pavement.py b/pavement.py deleted file mode 100644 index daeb802e..00000000 --- a/pavement.py +++ /dev/null @@ -1,329 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# -# Copyright (c) 2014 Tom Kralidis -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= - -import codecs -import glob -import os -import shutil -import tempfile -from io import BytesIO -from urllib.request import urlopen -import zipfile - -from paver.easy import (Bunch, call_task, cmdopts, info, options, - path, pushd, sh, task) - -BASEDIR = os.path.abspath(os.path.dirname(__file__)) - -options( - base=Bunch( - home=path(BASEDIR), - docs=path('%s/docs' % BASEDIR), - instance=path('%s/instance' % BASEDIR), - pot=path('%s/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po' % - BASEDIR), - static_docs=path('%s/GeoHealthCheck/static/docs' % BASEDIR), - static_lib=path('%s/GeoHealthCheck/static/lib' % BASEDIR), - tmp=path(tempfile.mkdtemp()), - translations=path('%s/GeoHealthCheck/translations' % BASEDIR) - ), -) - - -@task -def setup(): - """setup plugin dependencies""" - - config_file = options.base.home / 'GeoHealthCheck/config_main.py' - config_site = options.base.instance / 'config_site.py' - - # setup dirs - if not os.path.exists(options.base.static_lib): - options.base.static_lib.mkdir() - if not os.path.exists(options.base.instance): - options.base.instance.mkdir() - data_dir = options.base.instance / 'data' - data_dir.mkdir() - # data_dir.chmod(0777) gives failure on Python 2.7 Paver 1.2.1 - os.chmod(path(data_dir), 0o777) - # setup config - config_file.copy(config_site) - - # setup deps - sh('pip install -r requirements.txt') - - skin = 'http://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v3.3.7+1.zip' # noqa - - skin_dirs = ['dist', 'vendor'] - need_to_fetch = False - - for skin_dir in skin_dirs: - skin_dir_path = os.sep.join( - ['startbootstrap-sb-admin-2-3.3.7-1', skin_dir]) - if not os.path.exists(skin_dir_path): - need_to_fetch = True - - if need_to_fetch: - zipstr = BytesIO(urlopen(skin).read()) - zipfile_obj = zipfile.ZipFile(zipstr) - zipfile_obj.extractall(options.base.static_lib) - - for zf_mem in skin_dirs: - src_loc = path(options.base.static_lib / - 'startbootstrap-sb-admin-2-3.3.7-1' / zf_mem) - dest_loc = path(options.base.static_lib / zf_mem) - if not os.path.exists(dest_loc): - src_loc.move(dest_loc) - else: - info('directory already exists. Skipping') - - shutil.rmtree(path(options.base.static_lib / - 'startbootstrap-sb-admin-2-3.3.7-1')) - - # install sparklines to static/site/js - with open(path(options.base.static_lib / 'jspark.js'), 'w') as f: - content = urlopen('http://ejohn.org/files/jspark.js').read().decode() - content.replace('red', 'green') - f.write(content) - - # install bootstrap-tagsinput to static/lib - info('Getting select2') - select2 = 'https://github.com/select2/select2/archive/4.0.3.zip' - - zipstr = BytesIO(urlopen(select2).read()) - zipfile_obj = zipfile.ZipFile(zipstr) - zipfile_obj.extractall(options.base.static_lib) - dirname = glob.glob(options.base.static_lib / 'select2-*')[0] - dstdir = ''.join(dirname.rsplit('-', 1)[:-1]) - try: - os.rename(dirname, dstdir) - except OSError: - shutil.rmtree(dstdir) - os.rename(dirname, dstdir) - - # install leafletjs to static/lib - info('Getting leaflet') - leafletjs = 'http://cdn.leafletjs.com/downloads/leaflet-0.7.5.zip' - - zipstr = BytesIO(urlopen(leafletjs).read()) - zipfile_obj = zipfile.ZipFile(zipstr) - zipfile_obj.extractall(options.base.static_lib / 'leaflet') - - # install html5shiv to static/lib - with open(path(options.base.static_lib / 'html5shiv.min.js'), 'w') as f: - url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js' - content = urlopen(url).read().decode() - f.write(content) - - # install respond to static/lib - with open(path(options.base.static_lib / 'respond.min.js'), 'w') as f: - url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js' - content = urlopen(url).read().decode() - f.write(content) - - # build i18n .mo files - call_task('compile_translations') - - # build local docs - call_task('refresh_docs') - - # message user - info('GeoHealthCheck is now built. Edit settings in %s' % config_site) - info('before deploying the application. Alternatively, you can start a') - info('development instance with "python GeoHealthCheck/app.py"') - - -@task -def create_secret_key(): - """create secret key for SECRET_KEY in instance/config_site.py""" - info('Secret key: \'%s\'' % codecs.encode(os.urandom(24), 'hex').decode()) - info('Copy/paste this key to set the SECRET_KEY') - info('value in instance/config_site.py') - - -@task -@cmdopts([ - ('email=', 'e', 'email'), - ('username=', 'u', 'username'), - ('password=', 'p', 'password') -]) -def create(options): - """create database objects and superuser account""" - - args = '' - username = options.get('username', None) - password = options.get('password', None) - email = options.get('email', None) - - if all([username, password, email]): - args = '%s %s %s' % (username, password, email) - sh('python %s create %s' % (path('GeoHealthCheck/models.py'), args)) - - -@task -@cmdopts([ - ('password=', 'p', 'password') -]) -def create_hash(options): - """ - Create hash, mainly for passwords. - Usage: paver create_hash -p mypass - """ - - import sys - sys.path.insert(0, BASEDIR + '/GeoHealthCheck') - from util import create_hash - token = create_hash(options.get('password', None)) - info('Copy/paste the entire token below for example to set password') - info(token) - - -@task -def upgrade(): - """upgrade database if changed; be sure to backup first!""" - - info('Upgrading database...') - with pushd(path('%s/GeoHealthCheck' % BASEDIR)): - sh('python manage.py db upgrade') - - -@task -def create_wsgi(): - """create WSGI wrapper and Apache2 configuration""" - wsgi_script = '%s%sGeoHealthCheck.wsgi' % (options.base.instance, os.sep) - with open(wsgi_script, 'w') as ff: - ff.write('import sys\n') - ff.write('sys.path.insert(0, \'%s\')\n' % BASEDIR) - ff.write('from GeoHealthCheck.app import APP as application') - - wsgi_conf = '%s%sGeoHealthCheck.conf' % (options.base.instance, os.sep) - with open(wsgi_conf, 'w') as ff: - ff.write('WSGIScriptAlias / %s%sGeoHealthCheck.wsgi\n' % - (options.base.instance, os.sep)) - ff.write('\n' % (BASEDIR, os.sep)) - ff.write('Order deny,allow\n') - ff.write('Allow from all\n') - ff.write('') - - -@task -def refresh_docs(): - """Build sphinx docs from scratch""" - - make = sphinx_make() - - if os.path.exists(options.base.static_docs): - shutil.rmtree(options.base.static_docs) - - with pushd(options.base.docs): - sh('%s clean' % make) - sh('%s html' % make) - source_html_dir = path('%s/docs/_build/html' % BASEDIR) - source_html_dir.copytree(options.base.static_docs) - - -@task -def clean(): - """clean environment""" - - if os.path.exists(options.base.static_lib): - shutil.rmtree(options.base.static_lib) - if os.path.exists(options.base.tmp): - shutil.rmtree(options.base.tmp) - if os.path.exists(options.base.static_docs): - shutil.rmtree(options.base.static_docs) - - -@task -def extract_translations(): - """extract translations wrapped in _() or gettext()""" - - pot_dir = path('GeoHealthCheck/translations/en/LC_MESSAGES') - if not os.path.exists(pot_dir): - pot_dir.makedirs() - - sh('pybabel extract -F babel.cfg -o %s GeoHealthCheck' % options.base.pot) - - -@task -@cmdopts([ - ('lang=', 'l', '2-letter language code'), -]) -def add_language_catalogue(options): - """adds new language profile""" - - lang = options.get('lang', None) - - if lang is None: - raise RuntimeError('missing lang argument') - - sh('pybabel init -i %s -d %s -l %s' % ( - options.base.pot, options.base.translations, lang)) - - -@task -def compile_translations(): - """build .mo files""" - - sh('pybabel compile -d %s' % options.base.translations) - - -@task -def update_translations(): - """update language strings""" - - call_task('extract_translations') - sh('pybabel update -i %s -d %s' % ( - options.base.pot, options.base.translations)) - - -@task -def runner_daemon(): - """Run the HealthCheck runner daemon scheduler""" - sh('python %s' % path('GeoHealthCheck/scheduler.py')) - - -@task -def run_healthchecks(): - """Run all HealthChecks directly""" - sh('python %s' % path('GeoHealthCheck/healthcheck.py')) - - -def sphinx_make(): - """return what command Sphinx is using for make""" - - if os.name == 'nt': - return 'make.bat' - return 'make' - - -@task -def run_tests(): - """Run all tests""" - sh('python %s' % path('tests/run_tests.py')) diff --git a/requirements-dev.txt b/requirements-dev.txt index bd869515..d4d8edf7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +click==7.0 flake8 -Paver==1.3.4 pylint +wheel==0.33.6 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..85a3effd --- /dev/null +++ b/setup.py @@ -0,0 +1,89 @@ +# ================================================================= +# +# Authors: Rob van Loon +# Tom Kralidis +# +# Copyright (c) 2019 Rob van Loon +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= +import io +import re +from setuptools import find_packages, setup + + +def read(filename, encoding='utf-8'): + """read file contents""" + with io.open(filename, encoding=encoding) as fh: + contents = fh.read().strip() + return contents + + +def get_package_version(): + """get version from top-level package init""" + version_file = read('GeoHealthCheck/__init__.py') + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + + +LONG_DESCRIPTION = read('README.md') + +DESCRIPTION = 'GeoHealthCheck is a Quality of Service Checker for OGC Web' \ + 'Services and web APIs.' + +setup( + name='GeoHealthCheck', + version=get_package_version(), + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + long_description_content_type="text/markdown", + author='Tom Kralidis', + author_email='tomkralidis@gmail.com', + maintainer='Tom Kralidis', + maintainer_email='tomkralidis@gmail.com', + url='https://geohealthcheck.org', + license='MIT', + install_requires=read('requirements.txt').splitlines(), + packages=find_packages(exclude=['tests']), + include_package_data=True, + entry_points={ + 'console_scripts': [ + 'geohc=GeoHealthCheck.cli:cli', + ] + }, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Framework :: Flask', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Scientific/Engineering :: GIS' + ], +) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1ee3cdcd..864fcc1c 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -30,11 +30,11 @@ import unittest import os -from models import DB, load_data, Resource -from views import get_probes_avail -from plugin import Plugin -from probe import Probe -from factory import Factory +from GeoHealthCheck.models import DB, load_data, Resource +from GeoHealthCheck.views import get_probes_avail +from GeoHealthCheck.plugin import Plugin +from GeoHealthCheck.probe import Probe +from GeoHealthCheck.factory import Factory TEST_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/test_resources.py b/tests/test_resources.py index ab9df17e..ea418165 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -31,11 +31,11 @@ import unittest import os -from init import App -from models import (DB, Resource, Run, load_data, Recipient) -from healthcheck import run_test_resource -from notifications import _parse_webhook_location -from resourceauth import ResourceAuth +from GeoHealthCheck.init import App +from GeoHealthCheck.models import (DB, Resource, Run, load_data, Recipient) +from GeoHealthCheck.healthcheck import run_test_resource +from GeoHealthCheck.notifications import _parse_webhook_location +from GeoHealthCheck.resourceauth import ResourceAuth TEST_DIR = os.path.dirname(os.path.abspath(__file__))