diff --git a/.flake8 b/.flake8 index 9c6e81dd..cb5470d5 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ max_line_length = 120 select = E, W, F ignore = W503 -exclude = node_modules,ci,build,docs \ No newline at end of file +exclude = node_modules,ci,build,docs,custom_loaders \ No newline at end of file diff --git a/.gitignore b/.gitignore index fb290956..7132f9d9 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,12 @@ npm-debug.log .vscode .project .pydevproject +docs/source/dtale*rst +docs/source/modules.rst # built JS files dtale/static/dist jest_tmp + +# custom CLI loaders +custom_loaders/ diff --git a/CHANGES.md b/CHANGES.md index 3f5c79d5..91830e37 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,3 +3,9 @@ Changelog ### 1.0.0 (2019-09-06) * Initial public release + +### 1.1.0 (2019-10-08) + + * IE support + * **Describe** & **About** popups + * Custom CLI support diff --git a/README.md b/README.md index b7541421..e0a174a9 100644 --- a/README.md +++ b/README.md @@ -16,15 +16,24 @@ Setup/Activate your environment and install the egg +**Python 3** ```bash -# create a virtualenv, if you haven't already created one (use "python -m virtualenv ~/pyenvs/dtale" if you're running Python2) +# create a virtualenv, if you haven't already created one $ python3 -m venv ~/pyenvs/dtale $ source ~/pyenvs/dtale/bin/activate -# install dtale egg (important to use the "-U" every time you install so it will grab the latest version) + +# install dtale egg (important to use the "--upgrade" every time you install so it will grab the latest version) $ pip install --upgrade dtale ``` +**Python 2** +```bash +# create a virtualenv, if you haven't already created one +$ python -m virtualenv ~/pyenvs/dtale +$ source ~/pyenvs/dtale/bin/activate - +# install dtale egg (important to use the "--upgrade" every time you install so it will grab the latest version) +$ pip install --upgrade dtale +``` Now you will have to ability to use D-Tale from the command-line or within a python-enabled terminal ### Command-line @@ -36,6 +45,71 @@ Loading data from **CSV** ```bash dtale --csv-path /home/jdoe/my_csv.csv --csv-parse_dates date ``` +Loading data from a **Custom** loader +- Using the DTALE_CLI_LOADERS environment variable, specify a path to a location containing some python modules +- Any python module containing the global variables LOADER_KEY & LOADER_PROPS will be picked up as a custom loader + - LOADER_KEY: the key that will be associated with your loader. By default you are given **arctic** & **csv** (if you use one of these are your key it will override these) + - LOADER_PROPS: the individual props available to be specified. + - For example, with arctic we have host, library, node, start & end. + - If you leave this property as an empty list your loader will be treated as a flag. For example, instead of using all the arctic properties we would simply specify `--arctic` (this wouldn't work well in arctic's case since it depends on all those properties) +- You will also need to specify a function with the following signature `def find_loader(kwargs)` which returns a function that returns a dataframe or `None` +- Here is an example of a custom loader: +``` +from dtale.cli.clickutils import get_loader_options + +''' + IMPORTANT!!! This global variable is required for building any customized CLI loader. + When find loaders on startup it will search for any modules containing the global variable LOADER_KEY. +''' +LOADER_KEY = 'testdata' +LOADER_PROPS = ['rows', 'columns'] + + +def test_data(rows, columns): + import pandas as pd + import numpy as np + import random + from past.utils import old_div + from pandas.tseries.offsets import Day + from dtale.utils import dict_merge + import string + + now = pd.Timestamp(pd.Timestamp('now').date()) + dates = pd.date_range(now - Day(364), now) + num_of_securities = old_div(rows, len(dates)) + securities = [ + dict(security_id=100000 + sec_id, int_val=random.randint(1, 100000000000), + str_val=random.choice(string.ascii_letters) * 5) + for sec_id in range(num_of_securities) + ] + data = pd.concat([ + pd.DataFrame([dict_merge(dict(date=date), sd) for sd in securities]) + for date in dates + ], ignore_index=True)[['date', 'security_id', 'int_val', 'str_val']] + + col_names = ['Col{}'.format(c) for c in range(columns)] + return pd.concat([data, pd.DataFrame(np.random.randn(len(data), columns), columns=col_names)], axis=1) + + +# IMPORTANT!!! This function is required for building any customized CLI loader. +def find_loader(kwargs): + test_data_opts = get_loader_options(LOADER_KEY, kwargs) + if len([f for f in test_data_opts.values() if f]): + def _testdata_loader(): + return test_data(int(test_data_opts.get('rows', 1000500)), int(test_data_opts.get('columns', 96))) + + return _testdata_loader + return None +``` +In this example we simplying building a dataframe with some dummy data based on dimensions specified on the command-line: +- `--testdata-rows` +- `--testdata-columns` + +Here's how you would use this loader: +```bash +DTALE_CLI_LOADERS=./path_to_loaders bash -c 'dtale --testdata-rows 10 --testdata-columns 5' +``` + ### Python Terminal This comes courtesy of PyCharm @@ -61,6 +135,15 @@ Selecting/Deselecting Columns ![Menu](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Info_menu.png "Menu") +- **Describe**: view all the columns & their data types as well as individual details of each column ![Describe](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe.png "Describe") + +|Data Type|Display|Notes| +|--------|:------:|:------:| +|date|![Describe date](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe_date.png "Describe Date")|| +|string|![Describe string](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe_string.png "Describe String")|If you have less than or equal to 100 unique values they will be displayed at the bottom of your popup| +|int|![Describe int](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe_int.png "Describe Int")|Anything with standard numeric classifications (min, max, 25%, 50%, 75%) will have a nice boxplot with the mean (if it exists) displayed as an outlier if you look closely.| +|float|![Describe float](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe_float.png "Describe Float")|| + - **Filter**: apply a simple pandas `query` to your data (link to pandas documentation included in popup) |Editing|Result| @@ -88,6 +171,13 @@ Selecting/Deselecting Columns |------|----------|-------| |![Correlations](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Correlations.png "Correlations")|![Timeseries](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Correlations_ts.png "Timeseries")|![Scatter](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Correlations_scatter.png "Scatter")| +- **About**: This will give you information about what version of D-Tale you're running as well as if its out of date to whats on PyPi. + +|Up To Date|Out Of Date| +|--------|:------:| +|![About-up-to-date](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/About-up-to-date.png "About - Out of Date")|![About-out-of-date](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/About-out-of-date.png "About - Up to Date")| + + - Resize: mostly a fail-safe in the event that your columns are no longer lining up. Click this and should fix that - Shutdown: pretty self-explanatory, kills your D-Tale session (there is also an auto-kill process that will kill your D-Tale after an hour of inactivity) @@ -237,6 +327,7 @@ Have a look at the [detailed documentation](https://dtale.readthedocs.io). D-Tale works with: * Back-end + * arctic * Flask * Flask-Caching * Flask-Compress @@ -258,6 +349,7 @@ Contributors: * [Wilfred Hughes](https://github.com/Wilfred) * [Dominik Christ](https://github.com/DominikMChrist) + * [Chris Boddy](https://github.com/cboddy) * [Jason Holden](https://github.com/jasonkholden) * [Youssef Habchi](http://youssef-habchi.com/) - title font * ... and many others ... diff --git a/docker/2_7/Dockerfile b/docker/2_7/Dockerfile index 8ebf8a2d..ef4e7a1b 100644 --- a/docker/2_7/Dockerfile +++ b/docker/2_7/Dockerfile @@ -44,4 +44,4 @@ WORKDIR /app RUN set -eux \ ; . /root/.bashrc \ - ; easy_install dtale-1.0.0-py2.7.egg \ No newline at end of file + ; easy_install dtale-1.1.0-py2.7.egg diff --git a/docker/3_6/Dockerfile b/docker/3_6/Dockerfile index fd6331f2..340b67da 100644 --- a/docker/3_6/Dockerfile +++ b/docker/3_6/Dockerfile @@ -44,4 +44,4 @@ WORKDIR /app RUN set -eux \ ; . /root/.bashrc \ - ; easy_install dtale-1.0.0-py3.7.egg \ No newline at end of file + ; easy_install dtale-1.1.0-py3.7.egg diff --git a/docs/images/About-out-of-date.png b/docs/images/About-out-of-date.png new file mode 100644 index 00000000..fda1c5b7 Binary files /dev/null and b/docs/images/About-out-of-date.png differ diff --git a/docs/images/About-up-to-date.png b/docs/images/About-up-to-date.png new file mode 100644 index 00000000..bf497861 Binary files /dev/null and b/docs/images/About-up-to-date.png differ diff --git a/docs/images/Browser1.png b/docs/images/Browser1.png index 28bad15e..abefc93e 100644 Binary files a/docs/images/Browser1.png and b/docs/images/Browser1.png differ diff --git a/docs/images/Col_select.png b/docs/images/Col_select.png index 5e773f95..6f693c55 100644 Binary files a/docs/images/Col_select.png and b/docs/images/Col_select.png differ diff --git a/docs/images/Describe.png b/docs/images/Describe.png new file mode 100644 index 00000000..e9f71f26 Binary files /dev/null and b/docs/images/Describe.png differ diff --git a/docs/images/Describe_date.png b/docs/images/Describe_date.png new file mode 100644 index 00000000..409e8020 Binary files /dev/null and b/docs/images/Describe_date.png differ diff --git a/docs/images/Describe_float.png b/docs/images/Describe_float.png new file mode 100644 index 00000000..20a95f23 Binary files /dev/null and b/docs/images/Describe_float.png differ diff --git a/docs/images/Describe_int.png b/docs/images/Describe_int.png new file mode 100644 index 00000000..6f526fdb Binary files /dev/null and b/docs/images/Describe_int.png differ diff --git a/docs/images/Describe_string.png b/docs/images/Describe_string.png new file mode 100644 index 00000000..7a9f87ca Binary files /dev/null and b/docs/images/Describe_string.png differ diff --git a/docs/images/Histogram.png b/docs/images/Histogram.png index 4d09edcf..40c93712 100644 Binary files a/docs/images/Histogram.png and b/docs/images/Histogram.png differ diff --git a/docs/images/Info_menu.png b/docs/images/Info_menu.png index 3307ec23..627cad37 100644 Binary files a/docs/images/Info_menu.png and b/docs/images/Info_menu.png differ diff --git a/docs/images/Info_menu_small.png b/docs/images/Info_menu_small.png index 6847d095..98b66134 100644 Binary files a/docs/images/Info_menu_small.png and b/docs/images/Info_menu_small.png differ diff --git a/docs/images/Menu_one_col.png b/docs/images/Menu_one_col.png index f963bf8a..64b7796c 100644 Binary files a/docs/images/Menu_one_col.png and b/docs/images/Menu_one_col.png differ diff --git a/docs/source/conf.py b/docs/source/conf.py index 8591b160..dfc4e1cb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -63,9 +63,9 @@ # built documents. # # The short X.Y version. -version = u'1.0.0' +version = u'1.1.0' # The full version, including alpha/beta/rc tags. -release = u'1.0.0' +release = u'1.1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/dtale.rst b/docs/source/dtale.rst deleted file mode 100644 index 188f9e8f..00000000 --- a/docs/source/dtale.rst +++ /dev/null @@ -1,54 +0,0 @@ -dtale package -============= - -Submodules ----------- - -dtale\.app module ------------------ - -.. automodule:: dtale.app - :members: - :undoc-members: - :show-inheritance: - -dtale\.cli module ------------------ - -.. automodule:: dtale.cli - :members: - :undoc-members: - :show-inheritance: - -dtale\.clickutils module ------------------------- - -.. automodule:: dtale.clickutils - :members: - :undoc-members: - :show-inheritance: - -dtale\.utils module -------------------- - -.. automodule:: dtale.utils - :members: - :undoc-members: - :show-inheritance: - -dtale\.views module -------------------- - -.. automodule:: dtale.views - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: dtale - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/index.rst b/docs/source/index.rst index a0c745b4..5f4b3647 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ General use The following section is available as a Jupyter notebook in `docs/source/running_dtale.ipynb`. .. toctree:: - :maxdepth: 2 + :maxdepth: 4 running_dtale.ipynb @@ -19,4 +19,4 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` -* :ref:`search` +* :ref:`search` \ No newline at end of file diff --git a/docs/source/modules.rst b/docs/source/modules.rst deleted file mode 100644 index b811f48f..00000000 --- a/docs/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -dtale -===== - -.. toctree:: - :maxdepth: 4 - - dtale diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt index 69d6d414..c8453d15 100644 --- a/docs/source/requirements.txt +++ b/docs/source/requirements.txt @@ -3,6 +3,8 @@ flasgger==0.9.3 sphinx==1.6.7 nbsphinx matplotlib +prompt-toolkit<2.0.0,>=1.0.4,!=1.0.17; python_version < '3.0' +prompt-toolkit<2.1.0,>=2.0.0; python_version > '3.0' ipython[notebook] pyyaml bs4 diff --git a/dtale/app.py b/dtale/app.py index d12b2724..f4c235b0 100644 --- a/dtale/app.py +++ b/dtale/app.py @@ -18,7 +18,7 @@ from six import PY3 from dtale import dtale -from dtale.clickutils import retrieve_meta_info_and_version, setup_logging +from dtale.cli.clickutils import retrieve_meta_info_and_version, setup_logging from dtale.utils import dict_merge from dtale.views import startup @@ -71,28 +71,32 @@ class DtaleFlask(Flask): Overriding Flask's implementation of get_send_file_max_age, test_client & run + :param import_name: the name of the application package + :param reaper_on: whether to run auto-reaper subprocess + :type reaper_on: bool :param args: Optional arguments to be passed to :class:`flask.Flask` :param kwargs: Optional keyword arguments to be passed to :class:`flask.Flask` """ - def __init__(self, *args, **kwargs): + def __init__(self, import_name, reaper_on=True, *args, **kwargs): """ Constructor method + :param reaper_on: whether to run auto-reaper subprocess + :type reaper_on: bool """ - super(DtaleFlask, self).__init__(*args, **kwargs) - self.reaper_on = True + self.reaper_on = reaper_on self.reaper = None self.shutdown_url = None + super(DtaleFlask, self).__init__(import_name, *args, **kwargs) - def run(self, reaper_on=True, *args, **kwargs): + def run(self, *args, **kwargs): """ - :param reaper_on: whether to run auto-reaper subprocess - :type reaper_on: bool :param args: Optional arguments to be passed to :meth:`flask.run` :param kwargs: Optional keyword arguments to be passed to :meth:`flask.run` """ self.shutdown_url = 'http://{}:{}/shutdown'.format(socket.gethostname(), kwargs.get('port')) - self.reaper_on = reaper_on and not kwargs.get('debug', False) + if kwargs.get('debug', False): + self.reaper_on = False self.build_reaper() super(DtaleFlask, self).run(use_reloader=kwargs.get('debug', False), *args, **kwargs) @@ -148,14 +152,15 @@ def get_send_file_max_age(self, name): :param name: filename :return: Flask's default behavior for get_send_max_age if filename is not in SHORT_LIFE_PATHS - otherwise SHORT_LIFE_TIMEOUT + otherwise SHORT_LIFE_TIMEOUT + """ if name and any([name.startswith(path) for path in SHORT_LIFE_PATHS]): return SHORT_LIFE_TIMEOUT return super(DtaleFlask, self).get_send_file_max_age(name) -def build_app(): +def build_app(reaper_on=True): """ Builds Flask application encapsulating endpoints for D-Tale's front-end @@ -163,7 +168,7 @@ def build_app(): :rtype: :class:`dtale.app.DtaleFlask` """ - app = DtaleFlask('dtale', static_url_path='') + app = DtaleFlask('dtale', reaper_on=reaper_on, static_url_path='') app.config['SECRET_KEY'] = 'Dtale' app.jinja_env.trim_blocks = True @@ -301,7 +306,8 @@ def version_info(): :return: text/html version information """ - return retrieve_meta_info_and_version('dtale') + _, version = retrieve_meta_info_and_version('dtale') + return str(version) return app @@ -355,7 +361,7 @@ def show(data=None, host='0.0.0.0', port=None, debug=False, subprocess=True, dat def _show(): selected_port = int(port or find_free_port()) startup(data=data, data_loader=data_loader, port=selected_port) - app = build_app() + app = build_app(reaper_on=reaper_on) if debug: app.jinja_env.auto_reload = True @@ -364,7 +370,7 @@ def _show(): getLogger("werkzeug").setLevel(LOG_ERROR) logger.info('D-Tale started at: http://{}:{}'.format(socket.gethostname(), selected_port)) - app.run(host=host, port=selected_port, debug=debug, reaper_on=reaper_on) + app.run(host=host, port=selected_port, debug=debug) if subprocess: _thread.start_new_thread(_show, ()) diff --git a/dtale/cli.py b/dtale/cli.py deleted file mode 100644 index 73e33ee1..00000000 --- a/dtale/cli.py +++ /dev/null @@ -1,77 +0,0 @@ -from builtins import map -from logging import getLogger - -import click -import pandas as pd - -from dtale.app import find_free_port, show -from dtale.clickutils import (LOG_LEVELS, get_loader_options, get_log_options, - loader_options, run, setup_logging) - -logger = getLogger(__name__) - - -@click.command(name='main', help='Run dtale from command-line') -@click.option('--host', type=str, default='0.0.0.0') -@click.option('--port', type=int) -@click.option('--debug', is_flag=True) -@click.option('--no-reaper', is_flag=True) -@loader_options('arctic', ['host', 'library', 'node', 'start', 'end']) -@loader_options('csv', ['path', 'parse_dates']) -@click.option('--log', 'logfile', help='Log file name') -@click.option('--log-level', - help='Set the logging level', - type=click.Choice(list(LOG_LEVELS.keys())), - default='info', - show_default=True) -@click.option('-v', '--verbose', help='Set the logging level to debug', is_flag=True) -def main(host, port=None, debug=False, no_reaper=False, **kwargs): - """ - Runs a local server for the D-Tale application. - - This local server is recommended when you have a pandas object stored in a CSV - or retrievable from :class:`arctic.Arctic` data store. - """ - log_opts = get_log_options(kwargs) - setup_logging(log_opts.get('logfile'), log_opts.get('log_level'), log_opts.get('verbose')) - - # Setup arctic loader - arctic_opts = get_loader_options('arctic', kwargs) - if len([f for f in arctic_opts.values() if f]): - def _arctic_loader(): - try: - from arctic import Arctic - from arctic.store.versioned_item import VersionedItem - except BaseException as ex: - logger.exception('In order to use the arctic loader you must arctic!') - raise ex - host = Arctic(arctic_opts['host']) - lib = host.get_library(arctic_opts['library']) - read_kwargs = {} - start, end = map(arctic_opts.get, ['start', 'end']) - if start and end: - read_kwargs['chunk_range'] = pd.date_range(start, end) - data = lib.read(arctic_opts['node'], **read_kwargs) - if isinstance(data, VersionedItem): - data = data.data - return data - - data_loader = _arctic_loader - - # Setup csv loader - csv_opts = get_loader_options('csv', kwargs) - if len([f for f in csv_opts.values() if f]): - def _csv_loader(): - csv_arg_parsers = { # TODO: add additional arg parsers - 'parse_dates': lambda v: v.split(',') if v else None - } - kwargs = {k: csv_arg_parsers.get(k, lambda v: v)(v) for k, v in csv_opts.items() if k != 'path'} - return pd.read_csv(csv_opts['path'], **kwargs) - data_loader = _csv_loader - - show(host=host, port=int(port or find_free_port()), debug=debug, subprocess=False, data_loader=data_loader, - reaper_on=not no_reaper, **kwargs) - - -if __name__ == '__main__': - run(main) diff --git a/dtale/cli/__init__.py b/dtale/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dtale/clickutils.py b/dtale/cli/clickutils.py similarity index 100% rename from dtale/clickutils.py rename to dtale/cli/clickutils.py diff --git a/dtale/cli/loaders/__init__.py b/dtale/cli/loaders/__init__.py new file mode 100644 index 00000000..75c4c65b --- /dev/null +++ b/dtale/cli/loaders/__init__.py @@ -0,0 +1,133 @@ +import os +import platform +from logging import getLogger + +import click + +from dtale.cli.loaders import arctic_loader, csv_loader + +logger = getLogger(__name__) + + +def build_custom_module_loader_args(fname, path): + return 'dtale.cli.loaders.{}'.format(fname), '{}/{}.py'.format(path, fname) + + +def get_py35_loader(fname, path): + """ + Utility function for loading dynamic modules (CLI configurations) when python_version >= 3.5 + + """ + import importlib.util + + spec = importlib.util.spec_from_file_location(*build_custom_module_loader_args(fname, path)) + custom_loader = importlib.util.module_from_spec(spec) + spec.loader.exec_module(custom_loader) + return custom_loader + + +def get_py33_loader(fname, path): + """ + Utility function for loading dynamic modules (CLI configurations) when python_version in (3.3, 3.4) + + """ + from importlib.machinery import SourceFileLoader + return SourceFileLoader(*build_custom_module_loader_args(fname, path)).load_module() + + +def get_py2_loader(fname, path): + """ + Utility function for loading dynamic modules (CLI configurations) when python_version < 3 + + """ + import imp + return imp.load_source(*build_custom_module_loader_args(fname, path)) + + +def unsupported_python_version(version_tuple): + return ( + 'Unsupported version of python used for custom CLI loaders, {}. If you do not plan on using any custom ' + 'CLI loaders please remove your DTALE_CLI_LOADERS environment variable.' + ).format(version_tuple) + + +def custom_module_loader(): + """ + Utility function for using different module loaders based on python version: + * :func:dtale.cli.loaders.get_py35_loader + * :func:dtale.cli.loaders.get_py33_loader + * :func:dtale.cli.loaders.get_py2_loader + + """ + major, minor, revision = [int(i) for i in platform.python_version_tuple()] + if major == 2: + return get_py2_loader + if major == 3: + if minor >= 5: + return get_py35_loader + elif minor in (3, 4): + return get_py33_loader + raise ValueError(unsupported_python_version(platform.python_version_tuple())) + + +LOADERS = { + arctic_loader.LOADER_KEY: arctic_loader, + csv_loader.LOADER_KEY: csv_loader +} + + +def build_loaders(): + """ + Utility function executed at runtime to load dynamic CLI options from the environment variable, DTALE_CLI_LOADERS. + You either override one of the two default loader configurations, arctic or csv, or create a brand new + configuration which can be referenced from the command line. + + """ + global LOADERS + + custom_loader_path = os.environ.get('DTALE_CLI_LOADERS') + if custom_loader_path is not None: + custom_loader_func = custom_module_loader() + for full_filename in os.listdir(custom_loader_path): + filename, file_extension = os.path.splitext(full_filename) + if file_extension == '.py': + custom_loader = custom_loader_func(filename, custom_loader_path) + if hasattr(custom_loader, 'LOADER_KEY') and hasattr(custom_loader, 'LOADER_PROPS'): + LOADERS[custom_loader.LOADER_KEY] = custom_loader + + +def setup_loader_options(): + """ + Utility function executed at runtime to find dynamic CLI options as well as the defaults and create their + `click` decorators to keyword arguments will be processed accordingly. + + """ + build_loaders() + + def decorator(f): + for cli_loader in LOADERS.values(): + if len(cli_loader.LOADER_PROPS): + for p in cli_loader.LOADER_PROPS: + f = click.option( + '--' + cli_loader.LOADER_KEY + '-' + p, help='Override {} {}'.format(cli_loader.LOADER_KEY, p) + )(f) + else: + f = click.option( + '--' + cli_loader.LOADER_KEY, is_flag=True, + help='Use {} loader'.format(cli_loader.LOADER_KEY) + )(f) + return f + return decorator + + +def check_loaders(kwargs): + """ + Utility function to find which CLI loader is being used based on the `click` options/flags provided from the + command line + + """ + for cli_loader in LOADERS.values(): + cli_loader_func = cli_loader.find_loader(kwargs) + if cli_loader_func is not None: + return cli_loader_func + return None diff --git a/dtale/cli/loaders/arctic_loader.py b/dtale/cli/loaders/arctic_loader.py new file mode 100644 index 00000000..539472af --- /dev/null +++ b/dtale/cli/loaders/arctic_loader.py @@ -0,0 +1,42 @@ +from arctic import Arctic # isort:skip + +from builtins import map + +import pandas as pd +from arctic.store.versioned_item import VersionedItem + +from dtale.cli.clickutils import get_loader_options + +''' + IMPORTANT!!! This global variable is required for building any customized CLI loader. + When find loaders on startup it will search for any modules containing the global variable LOADER_KEY. +''' +LOADER_KEY = 'arctic' +LOADER_PROPS = ['host', 'library', 'node', 'start', 'end'] + + +# IMPORTANT!!! This function is required for building any customized CLI loader. +def find_loader(kwargs): + """ + Arctic implementation of data loader which will return a function if any of the + `click` options based on LOADER_KEY & LOADER_PROPS have been used, otherwise return None + + :param kwargs: Optional keyword arguments to be passed from `click` + :return: data loader function for arctic implementation + """ + arctic_opts = get_loader_options(LOADER_KEY, kwargs) + if len([f for f in arctic_opts.values() if f]): + def _arctic_loader(): + host = Arctic(arctic_opts['host']) + lib = host.get_library(arctic_opts['library']) + read_kwargs = {} + start, end = map(arctic_opts.get, ['start', 'end']) + if start and end: + read_kwargs['chunk_range'] = pd.date_range(start, end) + data = lib.read(arctic_opts['node'], **read_kwargs) + if isinstance(data, VersionedItem): + data = data.data + return data + + return _arctic_loader + return None diff --git a/dtale/cli/loaders/csv_loader.py b/dtale/cli/loaders/csv_loader.py new file mode 100644 index 00000000..2c3a5353 --- /dev/null +++ b/dtale/cli/loaders/csv_loader.py @@ -0,0 +1,31 @@ +import pandas as pd + +from dtale.cli.clickutils import get_loader_options + +''' + IMPORTANT!!! These global variables are required for building any customized CLI loader. + When build_loaders runs startup it will search for any modules containing the global variable LOADER_KEY. +''' +LOADER_KEY = 'csv' +LOADER_PROPS = ['path', 'parse_dates'] + + +# IMPORTANT!!! This function is required for building any customized CLI loader. +def find_loader(kwargs): + """ + CSV implementation of data loader which will return a function if any of the + `click` options based on LOADER_KEY & LOADER_PROPS have been used, otherwise return None + + :param kwargs: Optional keyword arguments to be passed from `click` + :return: data loader function for CSV implementation + """ + csv_opts = get_loader_options(LOADER_KEY, kwargs) + if len([f for f in csv_opts.values() if f]): + def _csv_loader(): + csv_arg_parsers = { # TODO: add additional arg parsers + 'parse_dates': lambda v: v.split(',') if v else None + } + kwargs = {k: csv_arg_parsers.get(k, lambda v: v)(v) for k, v in csv_opts.items() if k != 'path'} + return pd.read_csv(csv_opts['path'], **kwargs) + return _csv_loader + return None diff --git a/dtale/cli/script.py b/dtale/cli/script.py new file mode 100644 index 00000000..5d899cb4 --- /dev/null +++ b/dtale/cli/script.py @@ -0,0 +1,43 @@ +from logging import getLogger + +import click + +from dtale.app import find_free_port, show +from dtale.cli.clickutils import (LOG_LEVELS, get_log_options, run, + setup_logging) +from dtale.cli.loaders import check_loaders, setup_loader_options + +logger = getLogger(__name__) + + +@click.command(name='main', help='Run dtale from command-line') +@click.option('--host', type=str, default='0.0.0.0') +@click.option('--port', type=int) +@click.option('--debug', is_flag=True) +@click.option('--no-reaper', is_flag=True) +@setup_loader_options() +@click.option('--log', 'logfile', help='Log file name') +@click.option('--log-level', + help='Set the logging level', + type=click.Choice(list(LOG_LEVELS.keys())), + default='info', + show_default=True) +@click.option('-v', '--verbose', help='Set the logging level to debug', is_flag=True) +def main(host, port=None, debug=False, no_reaper=False, **kwargs): + """ + Runs a local server for the D-Tale application. + + This local server is recommended when you have a pandas object stored in a CSV + or retrievable from :class:`arctic.Arctic` data store. + """ + log_opts = get_log_options(kwargs) + setup_logging(log_opts.get('logfile'), log_opts.get('log_level'), log_opts.get('verbose')) + + data_loader = check_loaders(kwargs) + + show(host=host, port=int(port or find_free_port()), debug=debug, subprocess=False, data_loader=data_loader, + reaper_on=not no_reaper, **kwargs) + + +if __name__ == '__main__': + run(main) diff --git a/dtale/static/css/main.css b/dtale/static/css/main.css index 03a2c43f..9a00137e 100644 --- a/dtale/static/css/main.css +++ b/dtale/static/css/main.css @@ -100,6 +100,10 @@ html { -webkit-tap-highlight-color: transparent; } +*:focus { + outline: transparent; +} + @-ms-viewport { width: device-width; } @@ -4543,7 +4547,7 @@ button.close { margin: 30px auto; } .modal-sm { - max-width: 300px; + max-width: 400px; } } diff --git a/dtale/swagger/dtale/views/data.yml b/dtale/swagger/dtale/views/data.yml index 3b060059..64fee376 100644 --- a/dtale/swagger/dtale/views/data.yml +++ b/dtale/swagger/dtale/views/data.yml @@ -33,7 +33,7 @@ parameters: maxItems: 2 responses: 200: - description: JSON object containing success flag for pass/fail of pandas query + description: JSON object containing rows, columns and success flag content: application/json: schema: diff --git a/dtale/swagger/dtale/views/describe.yml b/dtale/swagger/dtale/views/describe.yml new file mode 100644 index 00000000..e314cce0 --- /dev/null +++ b/dtale/swagger/dtale/views/describe.yml @@ -0,0 +1,32 @@ +summary: Fetch descriptive information for a specific column +description: | + * [pandas.Series.describe](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.describe.html) +tags: + - D-Tale API +responses: + 200: + description: JSON object containing describe information, unique values and success flag + content: + application/json: + schema: + oneOf: + - properties: + describe: + type: object + description: object containing serialized output of pandas.Series.describe + uniques: + type: array + description: array of unique values for column when it has <= 100 unique values + success: + type: boolean + default: true + - properties: + error: + type: string + description: Exception summary + traceback: + type: string + description: Exception traceback + success: + type: boolean + default: false diff --git a/dtale/swagger/dtale/views/dtypes.yml b/dtale/swagger/dtale/views/dtypes.yml new file mode 100644 index 00000000..d8d1d04a --- /dev/null +++ b/dtale/swagger/dtale/views/dtypes.yml @@ -0,0 +1,42 @@ +summary: Fetch list of column names and dtypes +tags: + - D-Tale API +responses: + 200: + description: JSON object containing column name/dtype pairs and success flag + content: + application/json: + schema: + oneOf: + - properties: + dtypes: + type: array + description: list of columns, indexes and dtypes + items: + type: object + properties: + index: + type: integer + description: index of column within original dataframe + name: + type: string + description: column name + dtype: + type: string + description: "data type of column in pandas dataframe (EX: int64, float64, object)" + required: + - name + - dtype + success: + type: boolean + default: true + - properties: + error: + type: string + description: Exception summary + traceback: + type: string + description: Exception traceback + success: + type: boolean + default: false diff --git a/dtale/templates/dtale/main.html b/dtale/templates/dtale/main.html index deb3ff7a..4db6fb29 100644 --- a/dtale/templates/dtale/main.html +++ b/dtale/templates/dtale/main.html @@ -2,6 +2,7 @@ {% block full_content %} +
diff --git a/dtale/utils.py b/dtale/utils.py index e96dbb4d..034404ac 100644 --- a/dtale/utils.py +++ b/dtale/utils.py @@ -1,5 +1,6 @@ from __future__ import division +import decimal import json import os import sys @@ -122,33 +123,50 @@ def json_string(x, nan_display=''): return nan_display -def json_int(x, nan_display=''): +def json_int(x, nan_display='', as_string=False): """ Convert value to integer to be used within JSON output :param x: value to be converted to integer :param nan_display: if `x` is nan then return this value + :param as_string: return integer as a formatted string (EX: 1,000,000) :return: integer value :rtype: int """ try: - return int(x) if not np.isnan(x) and not np.isinf(x) else nan_display + if not np.isnan(x) and not np.isinf(x): + return '{:,d}'.format(int(x)) if as_string else int(x) + return nan_display except BaseException: return nan_display -def json_float(x, precision=2, nan_display='nan'): +# hack to solve issues with formatting floats with a precision more than 4 decimal points +# https://stackoverflow.com/questions/38847690/convert-float-to-string-without-scientific-notation-and-false-precision +DECIMAL_CTX = decimal.Context() +DECIMAL_CTX.prec = 20 + + +def json_float(x, precision=2, nan_display='nan', as_string=False): """ Convert value to float to be used within JSON output :param x: value to be converted to integer :param precision: precision of float to be returned :param nan_display: if `x` is nan then return this value + :param as_string: return float as a formatted string (EX: 1,234.5643) :return: float value :rtype: float """ try: - return float(round(x, precision)) if not np.isnan(x) and not np.isinf(x) else nan_display + if not np.isnan(x) and not np.isinf(x): + output = float(round(x, precision)) + if as_string: + str_output = format(DECIMAL_CTX.create_decimal(repr(x)), ',.{}f'.format(str(precision))) + # drop trailing zeroes off & trailing decimal points if necessary + return str_output.rstrip('0').rstrip('.') + return output + return nan_display except BaseException: return nan_display @@ -163,6 +181,8 @@ def json_date(x, nan_display=''): :rtype: str (YYYY-MM-DD) """ try: + if isinstance(x, np.datetime64): # calling unique on a pandas datetime column returns numpy datetime64 + return pd.Timestamp(x).strftime('%Y-%m-%d') return x.strftime('%Y-%m-%d') except BaseException: return nan_display @@ -204,12 +224,14 @@ def __init__(self, nan_display=''): def add_string(self, idx, name=None): self.fmts.append([idx, name, json_string]) - def add_int(self, idx, name=None): - self.fmts.append([idx, name, json_int]) + def add_int(self, idx, name=None, as_string=False): + def f(x, nan_display): + return json_int(x, nan_display=nan_display, as_string=as_string) + self.fmts.append([idx, name, f]) - def add_float(self, idx, name=None, precision=6): + def add_float(self, idx, name=None, precision=6, as_string=False): def f(x, nan_display): - return json_float(x, precision, nan_display=nan_display) + return json_float(x, precision, nan_display=nan_display, as_string=as_string) self.fmts.append([idx, name, f]) def add_timestamp(self, idx, name=None): @@ -422,7 +444,13 @@ def get_dtypes(df): """ Build dictionary of column/dtype name pairs from :class:`pandas.DataFrame` """ - return {c: d.name for c, d in df.dtypes.to_dict().items()} + def _load(): + for col, dtype in df.dtypes.to_dict().items(): + if dtype.name == 'object': + yield col, pd.api.types.infer_dtype(df[col], skipna=True) + else: + yield col, dtype.name + return dict(list(_load())) def grid_columns(df): @@ -441,6 +469,17 @@ def grid_columns(df): } +def find_dtype_formatter(dtype): + type_classification = classify_type(dtype) + if type_classification == 'I': + return json_int + if type_classification == 'D': + return json_date + if type_classification == 'F': + return json_float + return json_string + + def grid_formatter(col_types, nan_display='', overrides=None): """ Build :class:`dtale.utils.JSONFormatter` from :class:`pandas.DataFrame` @@ -542,16 +581,12 @@ def dict_merge(d1, d2): Either dictionary can be None. An empty dictionary {} will be returned if both dictionaries are None. - Parameters - ---------- - d1: dictionary - First dictionary can be None - d2: dictionary - Second dictionary can be None - - Returns - ------- - Dictionary + :param d1: First dictionary can be None + :type d1: dict + :param d2: Second dictionary can be None + :type d1: dict + :return: new dictionary with the contents of d2 overlaying the contents of d1 + :rtype: dict """ if not d1: return d2 or {} diff --git a/dtale/views.py b/dtale/views.py index 25285db2..ff59eed5 100644 --- a/dtale/views.py +++ b/dtale/views.py @@ -13,15 +13,18 @@ from pandas.tseries.offsets import Day, MonthBegin, QuarterBegin, YearBegin from dtale import dtale -from dtale.utils import (dict_merge, filter_df_for_grid, find_selected_column, - get_int_arg, get_str_arg, grid_columns, - grid_formatter, json_float, json_int, jsonify, - make_list, retrieve_grid_params, running_with_flask, - running_with_pytest, sort_df_for_grid) +from dtale.cli.clickutils import retrieve_meta_info_and_version +from dtale.utils import (dict_merge, filter_df_for_grid, find_dtype_formatter, + find_selected_column, get_dtypes, get_int_arg, + get_str_arg, grid_columns, grid_formatter, json_float, + json_int, jsonify, make_list, retrieve_grid_params, + running_with_flask, running_with_pytest, + sort_df_for_grid) logger = getLogger(__name__) DATA = None +DTYPES = None SETTINGS = {} @@ -36,18 +39,30 @@ def startup(data=None, data_loader=None, port=None): :param data_loader: function which returns pandas.DataFrame :param port: integer port for running Flask process """ - global DATA, SETTINGS + global DATA, DTYPES, SETTINGS if data_loader is not None: data = data_loader() - elif data is None: - logger.debug('pytest: {}, flask: {}'.format(running_with_pytest(), running_with_flask())) if data is not None: + if not isinstance(data, (pd.DataFrame, pd.Series, pd.DatetimeIndex, pd.MultiIndex)): + raise Exception( + 'data loaded must be one of the following types: pandas.DataFrame, pandas.Series, pandas.DatetimeIndex' + ) + + if isinstance(data, (pd.DatetimeIndex, pd.MultiIndex)): + data = data.to_frame(index=False) + + logger.debug('pytest: {}, flask: {}'.format(running_with_pytest(), running_with_flask())) curr_index = [i for i in make_list(data.index.name or data.index.names) if i is not None] logger.debug('pre-locking index columns ({}) to settings[{}]'.format(curr_index, port)) SETTINGS[str(port)] = dict(locked=curr_index) DATA = data.reset_index().drop('index', axis=1, errors='ignore') + dtypes = get_dtypes(DATA) + DTYPES = [dict(name=c, dtype=dtypes[c], index=i) for i, c in enumerate(DATA.columns)] + + else: + raise Exception('data loaded is None!') @dtale.route('/main') @@ -59,7 +74,8 @@ def view_main(): :return: HTML """ curr_settings = SETTINGS.get(request.environ.get('SERVER_PORT', 'curr'), {}) - return render_template('dtale/main.html', settings=json.dumps(curr_settings)) + _, version = retrieve_meta_info_and_version('dtale') + return render_template('dtale/main.html', settings=json.dumps(curr_settings), version=str(version)) @dtale.route('/update-settings') @@ -103,6 +119,81 @@ def test_filter(): return jsonify(dict(error=str(e), traceback=str(traceback.format_exc()))) +@dtale.route('/dtypes') +@swag_from('swagger/dtale/views/dtypes.yaml') +def dtypes(): + """ + Flask route which returns a list of column names and dtypes to the front-end as JSON + + :return: JSON { + dtypes: [ + {index: 1, name: col1, dtype: int64}, + ..., + {index: N, name: colN, dtype: float64} + ], + success: True/False + } + """ + try: + global DTYPES + return jsonify(dtypes=DTYPES, success=True) + except BaseException as e: + return jsonify(error=str(e), traceback=str(traceback.format_exc())) + + +def load_describe(column_series): + """ + Helper function for grabbing the output from pandas.Series.describe in a JSON serializable format + + :param column_series: data to describe + :type column_series: pandas.Series + :return: JSON serializable dictionary of the output from calling pandas.Series.describe + """ + desc = column_series.describe().to_frame().T + desc_f_overrides = { + 'I': lambda f, i, c: f.add_int(i, c, as_string=True), + 'F': lambda f, i, c: f.add_float(i, c, precision=4, as_string=True), + } + desc_f = grid_formatter(grid_columns(desc), nan_display='N/A', overrides=desc_f_overrides) + desc = desc_f.format_dict(next(desc.itertuples(), None)) + if 'count' in desc: + # pandas always returns 'count' as a float and it adds useless decimal points + desc['count'] = desc['count'].split('.')[0] + return desc + + +@dtale.route('/describe/') +@swag_from('swagger/dtale/views/describe.yaml') +def describe(column): + """ + Flask route which returns standard details about column data using pandas.DataFrame[col].describe to + the front-end as JSON + + :param column: required dash separated string "START-END" stating a range of row indexes to be returned + to the screen + :return: JSON { + describe: object representing output from pandas.Series.describe, + unique_data: array of unique values when data has <= 100 unique values + success: True/False + } + + """ + try: + global DATA + + desc = load_describe(DATA[column]) + return_data = dict(describe=desc, success=True) + uniq_vals = DATA[column].unique() + if 'unique' not in return_data['describe']: + return_data['describe']['unique'] = json_int(len(uniq_vals), as_string=True) + if len(uniq_vals) <= 100: + uniq_f = find_dtype_formatter(get_dtypes(DATA)[column]) + return_data['uniques'] = [uniq_f(u, nan_display='N/A') for u in uniq_vals] + return jsonify(return_data) + except BaseException as e: + return jsonify(dict(error=str(e), traceback=str(traceback.format_exc()))) + + @dtale.route('/data') @swag_from('swagger/dtale/views/data.yml') def get_data(): @@ -119,7 +210,7 @@ def get_data(): results: [ {dtale_index: 1, col1: val1_1, ...,colN: valN_1}, ..., - {dtale_index: N2, col1: val1_N2, ...,colN: valN_N2}. + {dtale_index: N2, col1: val1_N2, ...,colN: valN_N2} ], columns: [{name: col1, dtype: 'int64'},...,{name: colN, dtype: 'datetime'}], total: N2, @@ -195,7 +286,8 @@ def get_histogram(): selected_col = find_selected_column(DATA, col) data = data[~pd.isnull(data[selected_col])][[selected_col]] hist = np.histogram(data, bins=bins) - desc = data.describe()[selected_col].to_dict() + + desc = load_describe(data[selected_col]) return jsonify(data=[json_float(h) for h in hist[0]], labels=['{0:.1f}'.format(l) for l in hist[1]], desc=desc) except BaseException as e: return jsonify(dict(error=str(e), traceback=str(traceback.format_exc()))) @@ -227,6 +319,17 @@ def get_correlations(): def _build_timeseries_chart_data(name, df, cols, min=None, max=None, sub_group=None): + """ + Helper function for grabbing JSON serialized data for one or many date groupings + + :param name: base name of series in chart + :param df: data frame to be grouped + :param cols: columns whose data is to be returned + :param min: optional hardcoded minimum to be returned for all series + :param max: optional hardcoded maximum to be returned for all series + :param sub_group: optional sub group to be used in addition to date + :return: generator of string keys and JSON serialized dictionaries + """ base_cols = ['date'] if sub_group in df: dfs = df.groupby(sub_group) diff --git a/package.json b/package.json index 37be9f08..d3c5aff1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtale", - "version": "0.1.0", + "version": "1.1.0", "description": "Numeric Data Viewer", "main": "main.js", "directories": { @@ -10,8 +10,8 @@ "test-file": "jest \"$TEST\"", "test-all": "jest \"/static/__tests__/.*test\\.jsx?$\" --verbose", "test": "npm run test-all -s", - "test-with-coverage": "JEST_JUNIT_OUTPUT=/tmp/circleci-test-results/js_junit.xml jest \"/static/__tests__/.*test\\.jsx?$\" --coverage --reporters=jest-junit", - "test-with-junit": "JEST_JUNIT_OUTPUT=/tmp/circleci-test-results/js_junit.xml jest \"/static/__tests__/.*test\\.jsx?$\" --reporters=jest-junit", + "test-with-coverage": "jest \"/static/__tests__/.*test\\.jsx?$\" --coverage --reporters=jest-junit", + "test-with-junit": "jest \"/static/__tests__/.*test\\.jsx?$\" --reporters=jest-junit", "report-duplicate-code": "jsinspect -t 20 static --reporter pmd > /tmp/circleci-test-results/duplicates.xml; echo 'Wrote duplicates.xml'", "watch-js": "parallel-webpack --watch --progress", "watch": "npm run watch-js", @@ -51,17 +51,21 @@ "text-summary" ] }, + "jest-junit": { + "outputDirectory": "/tmp/circleci-test-results", + "outputName": "js_junit.xml" + }, "author": "", "private": true, "license": "SEE LICENSE IN proprietary", "devDependencies": { - "@babel/core": "7.6.0", + "@babel/core": "7.6.2", "@babel/plugin-proposal-class-properties": "7.5.5", "@babel/plugin-transform-classes": "7.5.5", - "@babel/preset-env": "7.6.0", + "@babel/preset-env": "7.6.2", "@babel/preset-flow": "7.0.0", "@babel/preset-react": "7.0.0", - "@babel/register": "7.6.0", + "@babel/register": "7.6.2", "autoprefixer": "9.6.1", "babel-core": "7.0.0-bridge.0", "babel-eslint": "10.0.3", @@ -72,19 +76,19 @@ "css-loader": "3.2.0", "enzyme": "3.10.0", "enzyme-adapter-react-16": "1.14.0", - "eslint": "6.3.0", + "eslint": "6.5.1", "eslint-clang-formatter": "1.3.0", "eslint-config-prettier": "6.3.0", "eslint-plugin-babel": "5.3.0", "eslint-plugin-flowtype": "4.3.0", "eslint-plugin-lodash": "6.0.0", - "eslint-plugin-prettier": "3.1.0", + "eslint-plugin-prettier": "3.1.1", "eslint-plugin-promise": "4.2.1", - "eslint-plugin-react": "7.14.3", + "eslint-plugin-react": "7.15.1", "eslint-plugin-tape": "1.1.0", "exports-loader": "0.7.0", "file-loader": "4.2.0", - "flow-bin": "0.107.0", + "flow-bin": "0.108.0", "imports-loader": "0.8.0", "jest": "24.9.0", "jest-junit": "8.0.0", @@ -95,22 +99,23 @@ "postcss-loader": "3.0.0", "postcss-nested": "4.1.2", "prettier": "1.18.2", - "react-test-renderer": "16.9.0", + "react-test-renderer": "16.10.1", "regenerator-runtime": "0.13.3", "sass-loader": "8.0.0", "style-loader": "1.0.0", - "terser-webpack-plugin": "2.0.1", + "terser-webpack-plugin": "2.1.2", "url-loader": "2.1.0", - "webpack": "4.40.2", - "webpack-cli": "3.3.8" + "webpack": "4.41.0", + "webpack-cli": "3.3.9" }, "dependencies": { - "@fortawesome/fontawesome-free": "5.10.2", + "@fortawesome/fontawesome-free": "5.11.2", "any-promise": "1.3.0", "babel-polyfill": "6.26.0", "bootstrap": "4.3.1", "chart.js": "2.8.0", - "chartjs-plugin-zoom": "0.7.3", + "chartjs-chart-box-and-violin-plot": "2.1.0", + "chartjs-plugin-zoom": "0.7.4", "chroma-js": "2.0.6", "create-react-class": "15.6.3", "dom-helpers": "5.1.0", @@ -130,9 +135,9 @@ "postcss-cli": "6.1.3", "prop-types": "15.7.2", "querystring": "0.2.0", - "react": "16.9.0", + "react": "16.10.1", "react-addons-shallow-compare": "15.6.2", - "react-dom": "16.9.0", + "react-dom": "16.10.1", "react-modal-bootstrap": "1.1.1", "react-motion": "0.5.2", "react-redux": "7.1.1", diff --git a/setup.py b/setup.py index 800d57b6..462dd6ad 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ def run_tests(self): setup( name="dtale", - version="1.0.0", + version="1.1.0", author="MAN Alpha Technology", author_email="ManAlphaTech@man.com", description="Web Client for Visualizing Pandas Objects", @@ -68,6 +68,7 @@ def run_tests(self): "future", "itsdangerous", "pandas", + "requests", "scipy", "six" ], @@ -98,6 +99,6 @@ def run_tests(self): "swagger/**/**/*", "templates/**/*", "templates/**/**/*"]}, - entry_points={"console_scripts": ["dtale = dtale.cli:main"]}, + entry_points={"console_scripts": ["dtale = dtale.cli.script:main"]}, zip_safe=False ) diff --git a/static/__tests__/dtale/DataViewer-about-expired-test.jsx b/static/__tests__/dtale/DataViewer-about-expired-test.jsx new file mode 100644 index 00000000..6a40ad74 --- /dev/null +++ b/static/__tests__/dtale/DataViewer-about-expired-test.jsx @@ -0,0 +1,97 @@ +import { mount } from "enzyme"; +import _ from "lodash"; +import React from "react"; +import { Provider } from "react-redux"; + +import { DataViewerMenu } from "../../dtale/DataViewerMenu"; +import mockPopsicle from "../MockPopsicle"; +import * as t from "../jest-assertions"; +import reduxUtils from "../redux-test-utils"; +import { withGlobalJquery } from "../test-utils"; + +const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); +const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); + +describe("DataViewer tests", () => { + beforeAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", { configurable: true, value: 500 }); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", { configurable: true, value: 500 }); + + const mockBuildLibs = withGlobalJquery(() => + mockPopsicle.mock(url => { + if (_.includes(url, "pypi.org")) { + return { info: { version: "2.0.0" } }; + } + const { urlFetcher } = require("../redux-test-utils").default; + return urlFetcher(url); + }) + ); + jest.mock("popsicle", () => mockBuildLibs); + + const mockChartUtils = withGlobalJquery(() => (ctx, cfg) => { + const chartCfg = { ctx, cfg, data: cfg.data, destroyed: false }; + chartCfg.destroy = () => (chartCfg.destroyed = true); + chartCfg.getElementsAtXAxis = _evt => [{ _index: 0 }]; + chartCfg.getElementAtEvent = _evt => [{ _datasetIndex: 0, _index: 0, _chart: { config: cfg, data: cfg.data } }]; + return chartCfg; + }); + + jest.mock("chart.js", () => mockChartUtils); + jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); + }); + + afterAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", originalOffsetHeight); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth); + }); + + test("DataViewer: about expired version", done => { + const { DataViewer } = require("../../dtale/DataViewer"); + const About = require("../../popups/About").default; + + const store = reduxUtils.createDtaleStore(); + const body = document.getElementsByTagName("body")[0]; + body.innerHTML += ''; + body.innerHTML += ''; + body.innerHTML += '
'; + const result = mount( + + + , + { attachTo: document.getElementById("content") } + ); + + setTimeout(() => { + result.update(); + result + .find(DataViewerMenu) + .find("ul li button") + .at(5) + .simulate("click"); + setTimeout(() => { + result.update(); + + const about = result.find(About).first(); + t.equal( + about + .find("div.modal-body div.row") + .first() + .text(), + "Your Version:1.0.0", + "renders our version" + ); + t.equal( + about + .find("div.modal-body div.row") + .at(1) + .text(), + "PyPi Version:2.0.0", + "renders PyPi version" + ); + t.equal(about.find("div.dtale-alert").length, 1, "should render alert"); + done(); + }, 400); + }, 600); + }); +}); diff --git a/static/__tests__/dtale/DataViewer-about-test.jsx b/static/__tests__/dtale/DataViewer-about-test.jsx new file mode 100644 index 00000000..675c33ce --- /dev/null +++ b/static/__tests__/dtale/DataViewer-about-test.jsx @@ -0,0 +1,108 @@ +import { mount } from "enzyme"; +import React from "react"; +import { ModalClose } from "react-modal-bootstrap"; +import { Provider } from "react-redux"; + +import { DataViewerMenu } from "../../dtale/DataViewerMenu"; +import mockPopsicle from "../MockPopsicle"; +import * as t from "../jest-assertions"; +import reduxUtils from "../redux-test-utils"; +import { withGlobalJquery } from "../test-utils"; + +const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); +const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); + +describe("DataViewer tests", () => { + beforeAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", { configurable: true, value: 500 }); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", { configurable: true, value: 500 }); + + const mockBuildLibs = withGlobalJquery(() => + mockPopsicle.mock(url => { + const { urlFetcher } = require("../redux-test-utils").default; + return urlFetcher(url); + }) + ); + jest.mock("popsicle", () => mockBuildLibs); + + const mockChartUtils = withGlobalJquery(() => (ctx, cfg) => { + const chartCfg = { ctx, cfg, data: cfg.data, destroyed: false }; + chartCfg.destroy = () => (chartCfg.destroyed = true); + chartCfg.getElementsAtXAxis = _evt => [{ _index: 0 }]; + chartCfg.getElementAtEvent = _evt => [{ _datasetIndex: 0, _index: 0, _chart: { config: cfg, data: cfg.data } }]; + return chartCfg; + }); + + jest.mock("chart.js", () => mockChartUtils); + jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); + }); + + afterAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", originalOffsetHeight); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth); + }); + + test("DataViewer: about", done => { + const { DataViewer } = require("../../dtale/DataViewer"); + const About = require("../../popups/About").default; + + const store = reduxUtils.createDtaleStore(); + const body = document.getElementsByTagName("body")[0]; + body.innerHTML += ''; + body.innerHTML += ''; + body.innerHTML += '
'; + const result = mount( + + + , + { attachTo: document.getElementById("content") } + ); + + setTimeout(() => { + result.update(); + result + .find(DataViewerMenu) + .find("ul li button") + .at(5) + .simulate("click"); + setTimeout(() => { + result.update(); + t.equal(result.find(About).length, 1, "should show describe"); + result + .find(ModalClose) + .first() + .simulate("click"); + t.equal(result.find(About).length, 0, "should hide describe"); + result + .find(DataViewerMenu) + .find("ul li button") + .at(5) + .simulate("click"); + setTimeout(() => { + result.update(); + + const about = result.find(About).first(); + t.equal( + about + .find("div.modal-body div.row") + .first() + .text(), + "Your Version:1.0.0", + "renders our version" + ); + t.equal( + about + .find("div.modal-body div.row") + .at(1) + .text(), + "PyPi Version:1.0.0", + "renders PyPi version" + ); + t.equal(about.find("div.dtale-alert").length, 0, "should not render alert"); + done(); + }, 400); + }, 400); + }, 600); + }); +}); diff --git a/static/__tests__/dtale/DataViewer-base-test.jsx b/static/__tests__/dtale/DataViewer-base-test.jsx index 52e731aa..79ca348c 100644 --- a/static/__tests__/dtale/DataViewer-base-test.jsx +++ b/static/__tests__/dtale/DataViewer-base-test.jsx @@ -44,6 +44,7 @@ describe("DataViewer tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); afterAll(() => { @@ -58,6 +59,7 @@ describe("DataViewer tests", () => { const store = reduxUtils.createDtaleStore(); const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const result = mount( @@ -75,7 +77,7 @@ describe("DataViewer tests", () => { .first() .instance(); t.deepEqual( - result.find("div.headerCell").map(hc => hc.text()), + result.find(".main-grid div.headerCell").map(hc => hc.text()), ["col1", "col2", "col3", "col4"], "should render column headers" ); @@ -91,24 +93,24 @@ describe("DataViewer tests", () => { .find(DataViewerMenu) .find("ul li span.font-weight-bold") .map(s => s.text()), - ["Filter", "Correlations", "Coverage", "Resize", "Shutdown"], + ["Describe", "Filter", "Correlations", "Coverage", "Resize", "About", "Shutdown"], "Should render default menu options" ); result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .last() .simulate("click"); result.update(); - t.equal(result.find("div.headerCell.selected").length, 1, "should select col4"); + t.equal(result.find(".main-grid div.headerCell.selected").length, 1, "should select col4"); result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .last() .simulate("click"); result.update(); - t.equal(result.find("div.headerCell.selected").length, 0, "should clear selection"); + t.equal(result.find(".main-grid div.headerCell.selected").length, 0, "should clear selection"); result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .last() .simulate("click"); result.update(); @@ -119,8 +121,8 @@ describe("DataViewer tests", () => { .find("ul li span.font-weight-bold") .map(s => s.text()), _.concat( - ["Move To Front", "Lock", "Sort Ascending", "Sort Descending", "Clear Sort", "Filter", "Formats"], - ["Histogram", "Correlations", "Coverage", "Resize", "Shutdown"] + ["Describe", "Move To Front", "Lock", "Sort Ascending", "Sort Descending", "Clear Sort", "Filter", "Formats"], + ["Histogram", "Correlations", "Coverage", "Resize", "About", "Shutdown"] ), "Should render menu options associated with selected column" ); @@ -128,7 +130,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .at(3) + .at(4) .simulate("click"); t.equal( result @@ -149,7 +151,7 @@ describe("DataViewer tests", () => { setTimeout(() => { result.update(); result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .at(2) .simulate("click"); result.update(); @@ -169,7 +171,7 @@ describe("DataViewer tests", () => { .simulate("click"); result.update(); result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .last() .simulate("click"); result.update(); @@ -178,11 +180,11 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .first() + .at(1) .simulate("click"); result.update(); t.deepEqual( - result.find("div.headerCell").map(hc => hc.text()), + result.find(".main-grid div.headerCell").map(hc => hc.text()), ["â–²col4", "col1", "col2", "col3"], "should move col4 to front of main grid" ); @@ -191,7 +193,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .at(1) + .at(2) .simulate("click"); result.update(); t.deepEqual( @@ -206,14 +208,14 @@ describe("DataViewer tests", () => { //unlock result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .first() .simulate("click"); result.update(); result .find(DataViewerMenu) .find("ul li button") - .at(1) + .at(2) .simulate("click"); result.update(); t.deepEqual( @@ -234,15 +236,15 @@ describe("DataViewer tests", () => { setTimeout(() => { result.update(); t.equal(result.find("div.row").length, 0, "should remove information row"); - result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .last() .simulate("click"); + result.update(); result .find(DataViewerMenu) .find("ul li button") - .at(7) + .at(8) .simulate("click"); setTimeout(() => { result.update(); @@ -257,7 +259,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .at(10) + .at(11) .simulate("click"); result .find(DataViewerMenu) diff --git a/static/__tests__/dtale/DataViewer-correlations-test.jsx b/static/__tests__/dtale/DataViewer-correlations-test.jsx index f77bb16b..7798851a 100644 --- a/static/__tests__/dtale/DataViewer-correlations-test.jsx +++ b/static/__tests__/dtale/DataViewer-correlations-test.jsx @@ -35,6 +35,7 @@ describe("DataViewer tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); afterAll(() => { @@ -50,6 +51,7 @@ describe("DataViewer tests", () => { const store = reduxUtils.createDtaleStore(); const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const result = mount( @@ -63,7 +65,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .at(1) + .at(2) .simulate("click"); setTimeout(() => { result.update(); @@ -76,7 +78,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .at(1) + .at(2) .simulate("click"); setTimeout(() => { result.update(); diff --git a/static/__tests__/dtale/DataViewer-coverage-test.jsx b/static/__tests__/dtale/DataViewer-coverage-test.jsx index dc348fdd..91b4abd3 100644 --- a/static/__tests__/dtale/DataViewer-coverage-test.jsx +++ b/static/__tests__/dtale/DataViewer-coverage-test.jsx @@ -37,6 +37,7 @@ describe("DataViewer tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); afterAll(() => { @@ -53,6 +54,7 @@ describe("DataViewer tests", () => { const store = reduxUtils.createDtaleStore(); const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const result = mount( @@ -66,14 +68,14 @@ describe("DataViewer tests", () => { setTimeout(() => { result.update(); result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .last() .simulate("click"); result.update(); result .find(DataViewerMenu) .find("ul li button") - .at(9) + .at(10) .simulate("click"); result.update(); t.ok(result.find(PopupChart).instance().props.chartData.visible, "should open coverage"); @@ -89,7 +91,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .at(9) + .at(10) .simulate("click"); result.update(); result diff --git a/static/__tests__/dtale/DataViewer-describe-test.jsx b/static/__tests__/dtale/DataViewer-describe-test.jsx new file mode 100644 index 00000000..3e286951 --- /dev/null +++ b/static/__tests__/dtale/DataViewer-describe-test.jsx @@ -0,0 +1,158 @@ +import { mount } from "enzyme"; +import React from "react"; +import { ModalClose } from "react-modal-bootstrap"; +import { Provider } from "react-redux"; + +import { DataViewerMenu } from "../../dtale/DataViewerMenu"; +import mockPopsicle from "../MockPopsicle"; +import * as t from "../jest-assertions"; +import reduxUtils from "../redux-test-utils"; +import { withGlobalJquery } from "../test-utils"; + +const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); +const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); + +describe("DataViewer tests", () => { + beforeAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", { configurable: true, value: 500 }); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", { configurable: true, value: 500 }); + + const mockBuildLibs = withGlobalJquery(() => + mockPopsicle.mock(url => { + const { urlFetcher } = require("../redux-test-utils").default; + return urlFetcher(url); + }) + ); + + const mockChartUtils = withGlobalJquery(() => (ctx, cfg) => { + const chartCfg = { ctx, cfg, data: cfg.data, destroyed: false }; + chartCfg.destroy = () => (chartCfg.destroyed = true); + chartCfg.getElementsAtXAxis = _evt => [{ _index: 0 }]; + chartCfg.getElementAtEvent = _evt => [{ _datasetIndex: 0, _index: 0, _chart: { config: cfg, data: cfg.data } }]; + return chartCfg; + }); + + jest.mock("popsicle", () => mockBuildLibs); + jest.mock("chart.js", () => mockChartUtils); + jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); + }); + + afterAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", originalOffsetHeight); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth); + }); + + test("DataViewer: describe", done => { + const { DataViewer } = require("../../dtale/DataViewer"); + const Describe = require("../../popups/Describe").ReactDescribe; + const DtypesGrid = require("../../popups/describe/DtypesGrid").DtypesGrid; + + const store = reduxUtils.createDtaleStore(); + const body = document.getElementsByTagName("body")[0]; + body.innerHTML += ''; + body.innerHTML += ''; + body.innerHTML += '
'; + const result = mount( + + + , + { attachTo: document.getElementById("content") } + ); + + setTimeout(() => { + result.update(); + result + .find(DataViewerMenu) + .find("ul li button") + .first() + .simulate("click"); + setTimeout(() => { + result.update(); + t.equal(result.find(Describe).length, 1, "should show describe"); + result + .find(ModalClose) + .first() + .simulate("click"); + t.equal(result.find(Describe).length, 0, "should hide describe"); + result + .find(DataViewerMenu) + .find("ul li button") + .first() + .simulate("click"); + setTimeout(() => { + result.update(); + let dtypesGrid = result.find(DtypesGrid).first(); + t.equal(dtypesGrid.find("div[role='row']").length, 5, "should render dtypes"); + + dtypesGrid + .find("div[role='columnheader']") + .first() + .simulate("click"); + dtypesGrid = result.find(DtypesGrid).first(); + t.equal( + dtypesGrid + .find("div.headerCell") + .first() + .find("svg.ReactVirtualized__Table__sortableHeaderIcon--ASC").length, + 1, + "should sort col1 ASC" + ); + dtypesGrid + .find("div[role='columnheader']") + .first() + .simulate("click"); + dtypesGrid = result.find(DtypesGrid).first(); + t.equal( + dtypesGrid + .find("div.headerCell") + .first() + .find("svg.ReactVirtualized__Table__sortableHeaderIcon--DESC").length, + 1, + "should sort col1 DESC" + ); + dtypesGrid + .find("div[role='columnheader']") + .first() + .simulate("click"); + dtypesGrid = result.find(DtypesGrid).first(); + t.equal( + dtypesGrid + .find("div.headerCell") + .first() + .find("svg.ReactVirtualized__Table__sortableHeaderIcon").length, + 0, + "should remove col1 sort" + ); + dtypesGrid + .find("div.headerCell") + .first() + .find("input") + .first() + .simulate("change", { target: { value: "1" } }); + dtypesGrid = result.find(DtypesGrid).first(); + t.equal(dtypesGrid.find("div[role='row']").length, 2, "should render filtered dtypes"); + + dtypesGrid + .find("div[title='col1']") + .first() + .simulate("click"); + setTimeout(() => { + result.update(); + t.equal( + result + .find(Describe) + .first() + .find("h1") + .first() + .text(), + "col1", + "should describe col1" + ); + done(); + }, 400); + }, 400); + }, 400); + }, 600); + }); +}); diff --git a/static/__tests__/dtale/DataViewer-filter-test.jsx b/static/__tests__/dtale/DataViewer-filter-test.jsx index 5e2f1db5..409f7dbf 100644 --- a/static/__tests__/dtale/DataViewer-filter-test.jsx +++ b/static/__tests__/dtale/DataViewer-filter-test.jsx @@ -40,6 +40,7 @@ describe("DataViewer tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); afterAll(() => { @@ -54,6 +55,7 @@ describe("DataViewer tests", () => { const store = reduxUtils.createDtaleStore(); const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const result = mount( @@ -71,7 +73,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .first() + .at(1) .simulate("click"); result.update(); t.equal(result.find(Filter).length, 1, "should open filter"); @@ -86,7 +88,21 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") + .at(1) + .simulate("click"); + result.update(); + result + .find(ModalFooter) .first() + .find("button") + .at(1) + .simulate("click"); + result.update(); + t.notOk(result.find(Filter).instance().props.visible, "should close filter"); + result + .find(DataViewerMenu) + .find("ul li button") + .at(1) .simulate("click"); result.update(); result @@ -133,6 +149,7 @@ describe("DataViewer tests", () => { const store = reduxUtils.createDtaleStore(); const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const result = mount( @@ -150,7 +167,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .first() + .at(1) .simulate("click"); result.update(); result diff --git a/static/__tests__/dtale/DataViewer-formatting-test.jsx b/static/__tests__/dtale/DataViewer-formatting-test.jsx index e2a13bca..e3228b9f 100644 --- a/static/__tests__/dtale/DataViewer-formatting-test.jsx +++ b/static/__tests__/dtale/DataViewer-formatting-test.jsx @@ -50,6 +50,7 @@ describe("DataViewer tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); afterAll(() => { @@ -64,6 +65,7 @@ describe("DataViewer tests", () => { const store = reduxUtils.createDtaleStore(); const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const result = mount( @@ -78,14 +80,14 @@ describe("DataViewer tests", () => { result.update(); // select column result - .find("div.headerCell div") + .find(".main-grid div.headerCell div") .at(1) .simulate("click"); result.update(); result .find(DataViewerMenu) .find("ul li button") - .at(6) + .at(7) .simulate("click"); result.update(); t.equal(result.find(Formatting).length, 1, "should open formatting"); @@ -100,7 +102,7 @@ describe("DataViewer tests", () => { result .find(DataViewerMenu) .find("ul li button") - .at(6) + .at(7) .simulate("click"); result.update(); diff --git a/static/__tests__/dtale/DataViewer-reload-test.jsx b/static/__tests__/dtale/DataViewer-reload-test.jsx index a0e3eb94..f6469789 100644 --- a/static/__tests__/dtale/DataViewer-reload-test.jsx +++ b/static/__tests__/dtale/DataViewer-reload-test.jsx @@ -34,6 +34,7 @@ describe("DataViewer tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); afterAll(() => { @@ -47,6 +48,7 @@ describe("DataViewer tests", () => { const store = reduxUtils.createDtaleStore(); const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const result = mount( @@ -92,7 +94,7 @@ describe("DataViewer tests", () => { dv.getData(dv.state.ids, true); setTimeout(() => { result.update(); - t.equals(result.find(RemovableError).length, 1, "should display error"); + t.equal(result.find(RemovableError).length, 1, "should display error"); done(); }, 400); }, 400); diff --git a/static/__tests__/main-test.jsx b/static/__tests__/main-test.jsx index 23a24152..b9af1eee 100644 --- a/static/__tests__/main-test.jsx +++ b/static/__tests__/main-test.jsx @@ -11,6 +11,7 @@ function testMain(mainName, isDev = false) { const body = document.getElementsByTagName("body")[0]; const settings = "{"sort":[["col1","ASC"]]}"; body.innerHTML += ``; + body.innerHTML += ``; body.innerHTML += '
'; const mockReactDOM = { renderStatus: false }; mockReactDOM.render = () => { diff --git a/static/__tests__/popups/Correlations-test.jsx b/static/__tests__/popups/Correlations-test.jsx index 925a53c8..089ae001 100644 --- a/static/__tests__/popups/Correlations-test.jsx +++ b/static/__tests__/popups/Correlations-test.jsx @@ -54,6 +54,7 @@ describe("Correlations tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); afterAll(() => { @@ -67,6 +68,7 @@ describe("Correlations tests", () => { const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const result = mount(, { attachTo: document.getElementById("content") }); diff --git a/static/__tests__/popups/CoverageChartBody-test.jsx b/static/__tests__/popups/CoverageChartBody-test.jsx index 3625c852..e75889b2 100644 --- a/static/__tests__/popups/CoverageChartBody-test.jsx +++ b/static/__tests__/popups/CoverageChartBody-test.jsx @@ -31,6 +31,7 @@ describe("CoverageChartBody tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); test("CoverageChartBody missing data", done => { diff --git a/static/__tests__/popups/Histogram-test.jsx b/static/__tests__/popups/Histogram-test.jsx index 9be0d165..ec00845c 100644 --- a/static/__tests__/popups/Histogram-test.jsx +++ b/static/__tests__/popups/Histogram-test.jsx @@ -80,6 +80,7 @@ describe("Histogram tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); test("Histogram rendering data", done => { diff --git a/static/__tests__/popups/TimeseriesChartBody-test.jsx b/static/__tests__/popups/TimeseriesChartBody-test.jsx index 7ae33a91..5e605e1d 100644 --- a/static/__tests__/popups/TimeseriesChartBody-test.jsx +++ b/static/__tests__/popups/TimeseriesChartBody-test.jsx @@ -62,6 +62,7 @@ describe("TimeseriesChartBody tests", () => { jest.mock("popsicle", () => mockBuildLibs); jest.mock("chart.js", () => mockChartUtils); jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); }); test("TimeseriesChartBody rendering chart per dataset", done => { @@ -69,6 +70,7 @@ describe("TimeseriesChartBody tests", () => { const body = document.getElementsByTagName("body")[0]; body.innerHTML += ''; + body.innerHTML += ''; body.innerHTML += '
'; const url = buildURL("ts-test", { tsColumns: { x1: ["y1"], x2: ["y2"] } }); diff --git a/static/__tests__/redux-test-utils.jsx b/static/__tests__/redux-test-utils.jsx index 0bea66c9..405dcc1e 100644 --- a/static/__tests__/redux-test-utils.jsx +++ b/static/__tests__/redux-test-utils.jsx @@ -1,5 +1,7 @@ import qs from "querystring"; +import _ from "lodash"; + import dtaleApp from "../reducers/dtale"; import { createStore } from "../reducers/store"; import correlationsData from "./data/correlations"; @@ -26,6 +28,23 @@ const DATA = { success: true, }; +const DTYPES = { + dtypes: [ + { index: 0, name: "col1", dtype: "int64" }, + { index: 1, name: "col2", dtype: "float64" }, + { index: 2, name: "col3", dtype: "string" }, + { index: 3, name: "col4", dtype: "datetime[ns]" }, + ], + success: true, +}; + +const DESCRIBE = { + col1: { describe: { count: 4, max: 4, mean: 2.5, min: 1, std: 0, unique: 4, "25%": 1, "50%": 2.5, "75%": 4 } }, + col2: { describe: { count: 4, max: 4, mean: 4, min: 2.5, std: 0, unique: 4, "25%": 2.5, "50%": 4, "75%": 5.5 } }, + col3: { describe: { count: 4, freq: 4, top: "foo", unique: 1 }, uniques: ["foo"] }, + col4: { describe: { count: 3, first: "2000-01-01", freq: 1, last: "2000-01-01", top: "2000-01-01", unique: 1 } }, +}; + function urlFetcher(url) { const urlParams = qs.parse(url.split("?")[1]); const query = urlParams.query; @@ -51,6 +70,16 @@ function urlFetcher(url) { return { error: "No data found" }; } return { success: true }; + } else if (url.startsWith("/dtale/dtypes")) { + return DTYPES; + } else if (url.startsWith("/dtale/describe")) { + const column = _.last(url.split("/")); + if (_.has(DESCRIBE, column)) { + return _.assignIn({ success: true }, DESCRIBE[column]); + } + return { error: "Column not found!" }; + } else if (_.includes(url, "pypi.org")) { + return { info: { version: "1.0.0" } }; } return {}; } diff --git a/static/actions/charts.js b/static/actions/charts.js index 0e57fe87..70265749 100644 --- a/static/actions/charts.js +++ b/static/actions/charts.js @@ -4,9 +4,9 @@ function openChart(chartData) { }; } -function closeChart() { +function closeChart(chartData) { return function(dispatch) { - dispatch({ type: "close-chart" }); + dispatch({ type: "close-chart", chartData }); }; } diff --git a/static/chartUtils.jsx b/static/chartUtils.jsx index 49cc647e..1fd545cf 100644 --- a/static/chartUtils.jsx +++ b/static/chartUtils.jsx @@ -1,4 +1,5 @@ import Chart from "chart.js"; +import "chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js"; import "chartjs-plugin-zoom"; import _ from "lodash"; diff --git a/static/dtale/DataViewer.jsx b/static/dtale/DataViewer.jsx index ac1dd5ae..c4823ffd 100644 --- a/static/dtale/DataViewer.jsx +++ b/static/dtale/DataViewer.jsx @@ -248,7 +248,7 @@ class ReactDataViewer extends React.Component { {({ onRowsRendered }) => { this._onRowsRendered = onRowsRendered; return ( - this._grid.recomputeGridSize()}> + this._grid.recomputeGridSize()}> {({ width, height }) => (
D-TALE