New features in preparation of version 1.1.0:

- "Describe" popup for viewing quick details about each column in your dataframe - "About" popup so you can compare the version of your local D-Tale against PyPi - support for customized CLI options
man-group · Oct 14, 2019 · f1eb1ee · f1eb1ee
1 parent 8d06393
commit f1eb1ee
Show file tree

Hide file tree

Showing 76 changed files with 2,039 additions and 404 deletions.
diff --git a/.flake8 b/.flake8
@@ -2,4 +2,4 @@
 max_line_length = 120
 select = E, W, F
 ignore = W503
-exclude = node_modules,ci,build,docs
+exclude = node_modules,ci,build,docs,custom_loaders
diff --git a/.gitignore b/.gitignore
@@ -23,7 +23,12 @@ npm-debug.log
 .vscode
 .project
 .pydevproject
+docs/source/dtale*rst
+docs/source/modules.rst
 
 # built JS files
 dtale/static/dist
 jest_tmp
+
+# custom CLI loaders
+custom_loaders/
diff --git a/CHANGES.md b/CHANGES.md
@@ -3,3 +3,9 @@ Changelog
 ### 1.0.0 (2019-09-06)
 
   * Initial public release
+
+### 1.1.0 (2019-10-08)
+
+  * IE support
+  * **Describe** & **About** popups
+  * Custom CLI support
diff --git a/README.md b/README.md
@@ -16,15 +16,24 @@
 
 Setup/Activate your environment and install the egg
 
+**Python 3**
 ```bash
-# create a virtualenv, if you haven't already created one (use "python -m virtualenv ~/pyenvs/dtale" if you're running Python2)
+# create a virtualenv, if you haven't already created one
 $ python3 -m venv ~/pyenvs/dtale
 $ source ~/pyenvs/dtale/bin/activate
-# install dtale egg (important to use the "-U" every time you install so it will grab the latest version)
+
+# install dtale egg (important to use the "--upgrade" every time you install so it will grab the latest version)
 $ pip install --upgrade dtale
 ```
+**Python 2**
+```bash
+# create a virtualenv, if you haven't already created one
+$ python -m virtualenv ~/pyenvs/dtale
+$ source ~/pyenvs/dtale/bin/activate
 
-
+# install dtale egg (important to use the "--upgrade" every time you install so it will grab the latest version)
+$ pip install --upgrade dtale
+```
 Now you will have to ability to use D-Tale from the command-line or within a python-enabled terminal
 
 ### Command-line
@@ -36,6 +45,71 @@ Loading data from **CSV**
 ```bash
 dtale --csv-path /home/jdoe/my_csv.csv --csv-parse_dates date
 ```
+Loading data from a **Custom** loader
+- Using the DTALE_CLI_LOADERS environment variable, specify a path to a location containing some python modules
+- Any python module containing the global variables LOADER_KEY & LOADER_PROPS will be picked up as a custom loader
+  - LOADER_KEY: the key that will be associated with your loader.  By default you are given **arctic** & **csv** (if you use one of these are your key it will override these)
+  - LOADER_PROPS: the individual props available to be specified.
+    - For example, with arctic we have host, library, node, start & end.
+    - If you leave this property as an empty list your loader will be treated as a flag.  For example, instead of using all the arctic properties we would simply specify `--arctic` (this wouldn't work well in arctic's case since it depends on all those properties)
+- You will also need to specify a function with the following signature `def find_loader(kwargs)` which returns a function that returns a dataframe or `None`
+- Here is an example of a custom loader:
+```
+from dtale.cli.clickutils import get_loader_options
+
+'''
+  IMPORTANT!!! This global variable is required for building any customized CLI loader.
+  When find loaders on startup it will search for any modules containing the global variable LOADER_KEY.
+'''
+LOADER_KEY = 'testdata'
+LOADER_PROPS = ['rows', 'columns']
+
+
+def test_data(rows, columns):
+    import pandas as pd
+    import numpy as np
+    import random
+    from past.utils import old_div
+    from pandas.tseries.offsets import Day
+    from dtale.utils import dict_merge
+    import string
+
+    now = pd.Timestamp(pd.Timestamp('now').date())
+    dates = pd.date_range(now - Day(364), now)
+    num_of_securities = old_div(rows, len(dates))
+    securities = [
+        dict(security_id=100000 + sec_id, int_val=random.randint(1, 100000000000),
+             str_val=random.choice(string.ascii_letters) * 5)
+        for sec_id in range(num_of_securities)
+    ]
+    data = pd.concat([
+        pd.DataFrame([dict_merge(dict(date=date), sd) for sd in securities])
+        for date in dates
+    ], ignore_index=True)[['date', 'security_id', 'int_val', 'str_val']]
+
+    col_names = ['Col{}'.format(c) for c in range(columns)]
+    return pd.concat([data, pd.DataFrame(np.random.randn(len(data), columns), columns=col_names)], axis=1)
+
+
+# IMPORTANT!!! This function is required for building any customized CLI loader.
+def find_loader(kwargs):
+    test_data_opts = get_loader_options(LOADER_KEY, kwargs)
+    if len([f for f in test_data_opts.values() if f]):
+        def _testdata_loader():
+            return test_data(int(test_data_opts.get('rows', 1000500)), int(test_data_opts.get('columns', 96)))
+
+        return _testdata_loader
+    return None
+```
+In this example we simplying building a dataframe with some dummy data based on dimensions specified on the command-line:
+- `--testdata-rows`
+- `--testdata-columns`
+
+Here's how you would use this loader:
+```bash
+DTALE_CLI_LOADERS=./path_to_loaders bash -c 'dtale --testdata-rows 10 --testdata-columns 5'
+```
+
 
 ### Python Terminal
 This comes courtesy of PyCharm
@@ -61,6 +135,15 @@ Selecting/Deselecting Columns
 
 ![Menu](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Info_menu.png "Menu")
 
+- **Describe**: view all the columns & their data types as well as individual details of each column ![Describe](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe.png "Describe")
+
+|Data Type|Display|Notes|
+|--------|:------:|:------:|
+|date|![Describe date](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe_date.png "Describe Date")||
+|string|![Describe string](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe_string.png "Describe String")|If you have less than or equal to 100 unique values they will be displayed at the bottom of your popup|
+|int|![Describe int](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe_int.png "Describe Int")|Anything with standard numeric classifications (min, max, 25%, 50%, 75%) will have a nice boxplot with the mean (if it exists) displayed as an outlier if you look closely.|
+|float|![Describe float](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Describe_float.png "Describe Float")||
+
 - **Filter**: apply a simple pandas `query` to your data (link to pandas documentation included in popup)
 
 |Editing|Result|
@@ -88,6 +171,13 @@ Selecting/Deselecting Columns
 |------|----------|-------|
 |![Correlations](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Correlations.png "Correlations")|![Timeseries](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Correlations_ts.png "Timeseries")|![Scatter](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/Correlations_scatter.png "Scatter")|
 
+- **About**: This will give you information about what version of D-Tale you're running as well as if its out of date to whats on PyPi.
+
+|Up To Date|Out Of Date|
+|--------|:------:|
+|![About-up-to-date](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/About-up-to-date.png "About - Out of Date")|![About-out-of-date](https://raw.githubusercontent.com/manahl/dtale/master/docs/images/About-out-of-date.png "About - Up to Date")|
+
+
 - Resize: mostly a fail-safe in the event that your columns are no longer lining up. Click this and should fix that
 - Shutdown: pretty self-explanatory, kills your D-Tale session (there is also an auto-kill process that will kill your D-Tale after an hour of inactivity)
 
@@ -237,6 +327,7 @@ Have a look at the [detailed documentation](https://dtale.readthedocs.io).
 D-Tale works with:
 
   * Back-end
+    * arctic
     * Flask
     * Flask-Caching
     * Flask-Compress
@@ -258,6 +349,7 @@ Contributors:
 
  * [Wilfred Hughes](https://github.com/Wilfred)
  * [Dominik Christ](https://github.com/DominikMChrist)
+ * [Chris Boddy](https://github.com/cboddy)
  * [Jason Holden](https://github.com/jasonkholden)
  * [Youssef Habchi](http://youssef-habchi.com/) - title font
  * ... and many others ...

diff --git a/docker/2_7/Dockerfile b/docker/2_7/Dockerfile
@@ -44,4 +44,4 @@ WORKDIR /app
 
 RUN set -eux \
   ; . /root/.bashrc \
-  ; easy_install dtale-1.0.0-py2.7.egg
+  ; easy_install dtale-1.1.0-py2.7.egg
diff --git a/docker/3_6/Dockerfile b/docker/3_6/Dockerfile
@@ -44,4 +44,4 @@ WORKDIR /app
 
 RUN set -eux \
   ; . /root/.bashrc \
-  ; easy_install dtale-1.0.0-py3.7.egg
+  ; easy_install dtale-1.1.0-py3.7.egg
diff --git a/docs/images/About-out-of-date.png b/docs/images/About-out-of-date.png
diff --git a/docs/images/About-up-to-date.png b/docs/images/About-up-to-date.png
diff --git a/docs/images/Browser1.png b/docs/images/Browser1.png
diff --git a/docs/images/Col_select.png b/docs/images/Col_select.png
diff --git a/docs/images/Describe.png b/docs/images/Describe.png
diff --git a/docs/images/Describe_date.png b/docs/images/Describe_date.png
diff --git a/docs/images/Describe_float.png b/docs/images/Describe_float.png
diff --git a/docs/images/Describe_int.png b/docs/images/Describe_int.png
diff --git a/docs/images/Describe_string.png b/docs/images/Describe_string.png
diff --git a/docs/images/Histogram.png b/docs/images/Histogram.png
diff --git a/docs/images/Info_menu.png b/docs/images/Info_menu.png
diff --git a/docs/images/Info_menu_small.png b/docs/images/Info_menu_small.png
diff --git a/docs/images/Menu_one_col.png b/docs/images/Menu_one_col.png
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -63,9 +63,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'1.0.0'
+version = u'1.1.0'
 # The full version, including alpha/beta/rc tags.
-release = u'1.0.0'
+release = u'1.1.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/docs/source/dtale.rst b/docs/source/dtale.rst
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -10,7 +10,7 @@ General use
 The following section is available as a Jupyter notebook in `docs/source/running_dtale.ipynb`.
 
 .. toctree::
-  :maxdepth: 2
+  :maxdepth: 4
 
   running_dtale.ipynb
 
@@ -19,4 +19,4 @@ Indices and tables
 
 * :ref:`genindex`
 * :ref:`modindex`
-* :ref:`search`
+* :ref:`search`
diff --git a/docs/source/modules.rst b/docs/source/modules.rst
diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt
@@ -3,6 +3,8 @@ flasgger==0.9.3
 sphinx==1.6.7
 nbsphinx
 matplotlib
+prompt-toolkit<2.0.0,>=1.0.4,!=1.0.17; python_version < '3.0'
+prompt-toolkit<2.1.0,>=2.0.0; python_version > '3.0'
 ipython[notebook]
 pyyaml
 bs4

diff --git a/dtale/app.py b/dtale/app.py
@@ -18,7 +18,7 @@
 from six import PY3
 
 from dtale import dtale
-from dtale.clickutils import retrieve_meta_info_and_version, setup_logging
+from dtale.cli.clickutils import retrieve_meta_info_and_version, setup_logging
 from dtale.utils import dict_merge
 from dtale.views import startup
 
@@ -71,28 +71,32 @@ class DtaleFlask(Flask):
     Overriding Flask's implementation of
     get_send_file_max_age, test_client & run
 
+    :param import_name: the name of the application package
+    :param reaper_on: whether to run auto-reaper subprocess
+    :type reaper_on: bool
     :param args: Optional arguments to be passed to :class:`flask.Flask`
     :param kwargs: Optional keyword arguments to be passed to :class:`flask.Flask`
     """
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, import_name, reaper_on=True, *args, **kwargs):
         """
         Constructor method
+        :param reaper_on: whether to run auto-reaper subprocess
+        :type reaper_on: bool
         """
-        super(DtaleFlask, self).__init__(*args, **kwargs)
-        self.reaper_on = True
+        self.reaper_on = reaper_on
         self.reaper = None
         self.shutdown_url = None
+        super(DtaleFlask, self).__init__(import_name, *args, **kwargs)
 
-    def run(self, reaper_on=True, *args, **kwargs):
+    def run(self, *args, **kwargs):
         """
-        :param reaper_on: whether to run auto-reaper subprocess
-        :type reaper_on: bool
         :param args: Optional arguments to be passed to :meth:`flask.run`
         :param kwargs: Optional keyword arguments to be passed to :meth:`flask.run`
         """
         self.shutdown_url = 'http://{}:{}/shutdown'.format(socket.gethostname(), kwargs.get('port'))
-        self.reaper_on = reaper_on and not kwargs.get('debug', False)
+        if kwargs.get('debug', False):
+            self.reaper_on = False
         self.build_reaper()
         super(DtaleFlask, self).run(use_reloader=kwargs.get('debug', False), *args, **kwargs)
 
@@ -148,22 +152,23 @@ def get_send_file_max_age(self, name):
 
         :param name: filename
         :return: Flask's default behavior for get_send_max_age if filename is not in SHORT_LIFE_PATHS
-        otherwise SHORT_LIFE_TIMEOUT
+                 otherwise SHORT_LIFE_TIMEOUT
+
         """
         if name and any([name.startswith(path) for path in SHORT_LIFE_PATHS]):
             return SHORT_LIFE_TIMEOUT
         return super(DtaleFlask, self).get_send_file_max_age(name)
 
 
-def build_app():
+def build_app(reaper_on=True):
     """
     Builds Flask application encapsulating endpoints for D-Tale's front-end
 
     :return: Flask application
     :rtype: :class:`dtale.app.DtaleFlask`
     """
 
-    app = DtaleFlask('dtale', static_url_path='')
+    app = DtaleFlask('dtale', reaper_on=reaper_on, static_url_path='')
     app.config['SECRET_KEY'] = 'Dtale'
 
     app.jinja_env.trim_blocks = True
@@ -301,7 +306,8 @@ def version_info():
 
         :return: text/html version information
         """
-        return retrieve_meta_info_and_version('dtale')
+        _, version = retrieve_meta_info_and_version('dtale')
+        return str(version)
 
     return app
 
@@ -355,7 +361,7 @@ def show(data=None, host='0.0.0.0', port=None, debug=False, subprocess=True, dat
     def _show():
         selected_port = int(port or find_free_port())
         startup(data=data, data_loader=data_loader, port=selected_port)
-        app = build_app()
+        app = build_app(reaper_on=reaper_on)
 
         if debug:
             app.jinja_env.auto_reload = True
@@ -364,7 +370,7 @@ def _show():
             getLogger("werkzeug").setLevel(LOG_ERROR)
 
         logger.info('D-Tale started at: http://{}:{}'.format(socket.gethostname(), selected_port))
-        app.run(host=host, port=selected_port, debug=debug, reaper_on=reaper_on)
+        app.run(host=host, port=selected_port, debug=debug)
 
     if subprocess:
         _thread.start_new_thread(_show, ())