1.7.12

 - syntax highlighting on code exports
 - arctic integration test
 - "Column Analysis" popup showing histograms & value counts

.circleci/config.yml

@@ -5,7 +5,7 @@ defaults: &defaults
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
       CODECOV_TOKEN: b0d35139-0a75-427a-907b-2c78a762f8f0
-      VERSION: 1.7.11
+      VERSION: 1.7.12
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
     steps:
     - checkout
@@ -32,6 +32,15 @@ defaults: &defaults
         key: yarn-packages-{{ checksum "yarn.lock" }}
         paths:
           - ~/.cache/yarn
+    - run:
+        name: Install MongoDB
+        command: |
+          # run "cat /etc/os-release" to view information about the OS
+          # this article really helped with this madness: https://linuxize.com/post/how-to-install-mongodb-on-debian-9/
+          sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 9DA31620334BD75D9DCB49F368818C72E52529D4
+          echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/4.0 main" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.0.list
+          sudo apt-get update
+          sudo apt-get install -y mongodb-org
     - run:
         name: Lint & Format JS Code
         command: |
@@ -95,7 +104,8 @@ defaults: &defaults
         command: |
           if [ "${CIRCLE_BRANCH}" == "master" ]; then
             # the 'latest' URL redirects to the name of the latest tag.
-            export PANDOC_VERSION=$(curl --connect-timeout 5  --max-time 10  --retry 5  --retry-delay 0  --retry-max-time 40 -L -I "$PANDOC_RELEASES_URL/latest" | sed -ne 's#Location:.*tag/\(.*\)$#\1#p' | tr -d "\n\r")
+            # export PANDOC_VERSION=$(curl --connect-timeout 5  --max-time 10  --retry 5  --retry-delay 0  --retry-max-time 40 -L -I "$PANDOC_RELEASES_URL/latest" | sed -ne 's#Location:.*tag/\(.*\)$#\1#p' | tr -d "\n\r")
+            export PANDOC_VERSION=2.9.2
             echo $PANDOC_VERSION
             export PANDOC_FILE=pandoc-$PANDOC_VERSION-1-amd64.deb
             echo $PANDOC_FILE

CHANGES.md

@@ -1,5 +1,12 @@
 ## Changelog
 
+### 1.7.12 (2020-3-1)
+ * added syntax highlighting to code exports with react-syntax-highlighting
+ * added arctic integration test
+ * updated Histogram popup to "Column Analysis" which allows for the following
+   * Histograms -> integers and floats
+   * Value Counts -> integers, strings & dates
+
 ### 1.7.11 (2020-2-27)
  * hotfix for dash custom.js file missing from production webpack build script
 

README.md

@@ -35,9 +35,9 @@ D-Tale was the product of a SAS to Python conversion.  What was originally a per
   - [Dimensions/Main Menu](#dimensionsmain-menu)
   - [Selecting/Deselecting Columns](#selectingdeselecting-columns)
   - [Main Menu Functions](#main-menu-functions)
-    - [Describe](#describe), [Filter](#filter), [Charts](#charts), [Correlations](#correlations), [Heat Map](#heat-map), [Instances](#instances), [About](#about), [Resize](#resize), [Shutdown](#shutdown)
+    - [Describe](#describe), [Filter](#filter), [Charts](#charts), [Correlations](#correlations), [Heat Map](#heat-map), [Instances](#instances), [Code Exports](#code-exports), [About](#about), [Resize](#resize), [Shutdown](#shutdown)
   - [Column Menu Functions](#column-menu-functions)
-    - [Moving Columns](#moving-columns), [Hiding Columns](#hiding-columns), [Building Columns](#building-columns), [Lock](#lock), [Unlock](#unlock), [Sorting](#sorting), [Formats](#formats), [Histogram](#histogram)
+    - [Moving Columns](#moving-columns), [Hiding Columns](#hiding-columns), [Building Columns](#building-columns), [Lock](#lock), [Unlock](#unlock), [Sorting](#sorting), [Formats](#formats), [Column Analysis](#column-analysis)
   - [Menu Functions within a Jupyter Notebook](#menu-functions-within-a-jupyter-notebook)
 - [For Developers](#for-developers)
   - [Cloning](#cloning)
@@ -131,7 +131,7 @@ If you are running ipython<=5.0 then you also have the ability to adjust the siz
 
 One thing of note is that a lot of the modal popups you see in the standard browser version will now open separate browser windows for spacial convienence:
 
-|Column Menus|Correlations|Describe|Histogram|Instances|
+|Column Menus|Correlations|Describe|Column Analysis|Instances|
 |:------:|:------:|:------:|:------:|:------:|
 |![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/Column_menu.png)|![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/correlations_popup.png)|![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/describe_popup.png)|![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/histogram_popup.png)|![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/instances_popup.png)|
 
@@ -415,6 +415,19 @@ This will hide any non-float columns (with the exception of the index on the rig
 Turn off Heat Map by clicking menu option again
 ![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/Heatmap_toggle.png)
 
+#### Code Exports
+
+*Code Exports* are small snippets of code representing the current state of the grid you're viewing including things like:
+ - columns built
+ - filtering
+ - sorting
+
+Other code exports available are:
+ - Column Analysis
+ - Correlations (grid, timeseries chart & scatter chart)
+ - Describe
+ - Charts built using the Chart Builder
+
 #### Instances
 This will give you information about other D-Tale instances are running under your current Python process.
 
@@ -533,11 +546,22 @@ Here's a grid of all the formats available with -123456.789 as input:
 | BPS           | -1234567890BPS |
 | Red Negatives | <span style="color: red;">-123457</span>|
 
-#### Histogram
-Display histograms in any number of bins (default: 20), simply type a new integer value in the bins input
+#### Column Analysis
+Based on the data type of a column different charts will be shown.
+
+| Data Type     | Chart          |
+|---------------|----------------|
+| Integer       | Histogram, Value Counts|
+| Float         | Value Counts   |
+| Date          | Value Counts   |
+| String        | Value Counts   |
+
+*Histograms* can be displayed in any number of bins (default: 20), simply type a new integer value in the bins input
 
 ![](https://raw.githubusercontent.com/aschonfeld/dtale-media/master/images/Histogram.png)
 
+*Value Counts* are a bar chart containing the counts of each unique value in a column.
+
 ### Menu Functions Depending on Browser Dimensions
 Depending on the dimensions of your browser window the following buttons will not open modals, but rather separate browser windows:  Correlations, Describe & Instances (see images from [Jupyter Notebook](#jupyter-notebook), also Charts will always open in a separate browser window)
 

docker/2_7/Dockerfile

@@ -44,4 +44,4 @@ WORKDIR /app
 
 RUN set -eux \
   ; . /root/.bashrc \
-  ; easy_install dtale-1.7.11-py2.7.egg
+  ; easy_install dtale-1.7.12-py2.7.egg

docker/3_6/Dockerfile

@@ -44,4 +44,4 @@ WORKDIR /app
 
 RUN set -eux \
   ; . /root/.bashrc \
-  ; easy_install dtale-1.7.11-py3.7.egg
+  ; easy_install dtale-1.7.12-py3.7.egg

docs/source/conf.py

@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'1.7.11'
+version = u'1.7.12'
 # The full version, including alpha/beta/rc tags.
-release = u'1.7.11'
+release = u'1.7.12'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

dtale/cli/loaders/json_loader.py

@@ -1,5 +1,6 @@
 import pandas as pd
 import requests
+from pkg_resources import parse_version
 
 from dtale.app import show
 from dtale.cli.clickutils import get_loader_options
@@ -20,6 +21,10 @@ def show_loader(**kwargs):
     return show(data_loader=lambda: loader_func(**kwargs), **kwargs)
 
 
+def is_pandas1():
+    return parse_version(pd.__version__) >= parse_version('1.0.0')
+
+
 def loader_func(**kwargs):
     path = kwargs.pop('path')
     normalize = kwargs.pop('normalize', False)
@@ -31,7 +36,8 @@ def loader_func(**kwargs):
         resp = requests.get(path, **req_kwargs)
         path = resp.json() if normalize else resp.text
     if normalize:
-        return pd.io.json.json_normalize(path, **kwargs)
+        normalize_func = pd.json_normalize if is_pandas1() else pd.io.json.json_normalize
+        return normalize_func(path, **kwargs)
     return pd.read_json(path, **{k: v for k, v in kwargs.items() if k in LOADER_PROPS})
 
 

dtale/static/css/main.css

@@ -4562,7 +4562,8 @@ div.build-modal > div.modal-lg {
   div.build-modal > div.modal-lg {
     max-width: 720px;
   }
-  div.histogram-modal > div.modal-lg {
+  div.histogram-modal > div.modal-lg,
+  div.code-modal > div.modal-lg {
     max-width: 700px;
   }
 }

dtale/views.py

@@ -984,37 +984,53 @@ def get_data(data_id):
         return jsonify(dict(error=str(e), traceback=str(traceback.format_exc())))
 
 
-@dtale.route('/histogram/<data_id>')
-def get_histogram(data_id):
+@dtale.route('/column-analysis/<data_id>')
+def get_column_analysis(data_id):
     """
-    :class:`flask:flask.Flask` route which returns output from numpy.histogram to front-end as JSON
+    :class:`flask:flask.Flask` route which returns output from numpy.histogram/pd.value_counts to front-end as JSON
 
     :param data_id: integer string identifier for a D-Tale process's data
     :type data_id: str
     :param col: string from flask.request.args['col'] containing name of a column in your dataframe
+    :param type: string from flask.request.args['type'] to signify either a histogram or value counts
     :param query: string from flask.request.args['query'] which is applied to DATA using the query() function
     :param bins: the number of bins to display in your histogram, options on the front-end are 5, 10, 20, 50
     :returns: JSON {results: DATA, desc: output from pd.DataFrame[col].describe(), success: True/False}
     """
     try:
         col = get_str_arg(request, 'col', 'values')
         bins = get_int_arg(request, 'bins', 20)
+        data_type = get_str_arg(request, 'type') or 'histogram'
         data = run_query(
             global_state.get_data(data_id),
             get_str_arg(request, 'query'),
             global_state.get_context_variables(data_id)
         )
         selected_col = find_selected_column(data, col)
         data = data[~pd.isnull(data[selected_col])][[selected_col]]
-        hist = np.histogram(data, bins=bins)
 
-        desc, desc_code = load_describe(data[selected_col])
         code = build_code_export(data_id, imports='import numpy as np\nimport pandas as pd\n\n')
-        code.append("hist = np.histogram(df[~pd.isnull(df['{col}'])][['{col}']], bins={bins})".format(
-            col=selected_col, bins=bins))
-        code += desc_code
-        return jsonify(data=[json_float(h) for h in hist[0]], labels=['{0:.1f}'.format(l) for l in hist[1]],
-                       desc=desc, code='\n'.join(code))
+        dtype = get_dtypes(data)[selected_col]
+        classifier = classify_type(dtype)
+        if classifier in ['S', 'D'] or data_type != 'histogram':
+            hist = pd.value_counts(data[selected_col]).reset_index()
+            hist.columns = ['labels', 'data']
+            col_types = grid_columns(hist)
+            f = grid_formatter(col_types, nan_display=None)
+            return_data = f.format_lists(hist)
+            return_data['dtype'] = dtype
+            return_data['chart_type'] = 'value_counts'
+            code.append("hist = pd.value_counts(df[~pd.isnull(df['{col}'])]['{col}'])".format(col=selected_col))
+        else:
+            hist_data, hist_labels = np.histogram(data, bins=bins)
+            hist_data = [json_float(h) for h in hist_data]
+            hist_labels = ['{0:.1f}'.format(l) for l in hist_labels]
+            code.append("hist = np.histogram(df[~pd.isnull(df['{col}'])][['{col}']], bins={bins})".format(
+                col=selected_col, bins=bins))
+            desc, desc_code = load_describe(data[selected_col])
+            code += desc_code
+            return_data = dict(labels=hist_labels, data=hist_data, desc=desc, dtype=dtype, chart_type='histogram')
+        return jsonify(code='\n'.join(code), **return_data)
     except BaseException as e:
         return jsonify(dict(error=str(e), traceback=str(traceback.format_exc())))
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dtale",
-  "version": "1.7.11",
+  "version": "1.7.12",
   "description": "Visualizer for Pandas Data Structures",
   "main": "main.js",
   "directories": {
@@ -143,6 +143,7 @@
     "react-motion": "0.5.2",
     "react-redux": "7.1.3",
     "react-select": "3.0.8",
+    "react-syntax-highlighter": "12.2.1",
     "react-virtualized": "9.21.2",
     "react-wordcloud": "1.1.1",
     "redux": "4.0.4",

setup.py

@@ -50,7 +50,7 @@ def run_tests(self):
 
 setup(
     name="dtale",
-    version="1.7.11",
+    version="1.7.12",
     author="MAN Alpha Technology",
     author_email="[email protected]",
     description="Web Client for Visualizing Pandas Objects",
@@ -85,6 +85,8 @@ def run_tests(self):
         "mock",
         "pytest==4.6.4",
         "pytest-cov",
+        "pytest-server-fixtures",
+        "arctic",
     ],
     classifiers=[
         "Development Status :: 4 - Beta",

static/__tests__/data/histogram.json → static/__tests__/data/column-analysis.json

@@ -53,5 +53,7 @@
     "2.4",
     "2.7",
     "3.0"
-  ]
+  ],
+  "type": "histogram",
+  "dtype": "float64"
 }

static/__tests__/dtale/DataViewer-base-test.jsx

@@ -77,7 +77,7 @@ describe("DataViewer tests", () => {
     Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth);
   });
 
-  test("DataViewer: base operations (column selection, locking, sorting, moving to front, histograms,...", done => {
+  test("DataViewer: base operations (column selection, locking, sorting, moving to front, col-analysis,...", done => {
     const { DataViewer } = require("../../dtale/DataViewer");
 
     const store = reduxUtils.createDtaleStore();

static/__tests__/dtale/DataViewer-reload-test.jsx

@@ -48,7 +48,7 @@ describe("DataViewer tests", () => {
     Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth);
   });
 
-  test("DataViewer: base operations (column selection, locking, sorting, moving to front, histograms,...", done => {
+  test("DataViewer: base operations (column selection, locking, sorting, moving to front, col-analysis,...", done => {
     const { DataViewer, ReactDataViewer } = require("../../dtale/DataViewer");
 
     const store = reduxUtils.createDtaleStore();

static/__tests__/iframe/DataViewer-base-test.jsx

@@ -11,7 +11,7 @@ import mockPopsicle from "../MockPopsicle";
 import * as t from "../jest-assertions";
 import reduxUtils from "../redux-test-utils";
 import { buildInnerHTML, clickMainMenuButton, withGlobalJquery } from "../test-utils";
-import { clickColMenuButton, clickColMenuSubButton, findColMenuButton } from "./iframe-utils";
+import { clickColMenuButton, clickColMenuSubButton } from "./iframe-utils";
 
 const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight");
 const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth");
@@ -95,7 +95,7 @@ describe("DataViewer iframe tests", () => {
     window.self = self;
   });
 
-  test("DataViewer: base operations (column selection, locking, sorting, moving to front, histograms,...", done => {
+  test("DataViewer: base operations (column selection, locking, sorting, moving to front, col-analysis,...", done => {
     const { DataViewer } = require("../../dtale/DataViewer");
     const ColumnMenu = require("../../dtale/iframe/ColumnMenu").ReactColumnMenu;
     const Header = require("../../dtale/Header").ReactHeader;
@@ -169,7 +169,7 @@ describe("DataViewer iframe tests", () => {
       );
       t.deepEqual(
         colMenu.find("ul li span.font-weight-bold").map(s => s.text()),
-        ["Lock", "Hide", "Describe", "Histogram", "Formats"],
+        ["Lock", "Hide", "Describe", "Column Analysis", "Formats"],
         "Should render column menu options"
       );
       clickColMenuSubButton(result, "Asc");
@@ -282,14 +282,13 @@ describe("DataViewer iframe tests", () => {
             .find(".main-grid div.headerCell div")
             .last()
             .simulate("click");
-          t.equal(findColMenuButton(result, "Histogram").length, 0, "string col shouldn't have histogram button");
           result
             .find(".main-grid div.headerCell div")
             .at(2)
             .simulate("click");
-          clickColMenuButton(result, "Histogram");
+          clickColMenuButton(result, "Column Analysis");
           expect(window.open.mock.calls[window.open.mock.calls.length - 1][0]).toBe(
-            "/dtale/popup/histogram/1?selectedCol=col2"
+            "/dtale/popup/column-analysis/1?selectedCol=col2"
           );
           clickColMenuButton(result, "Describe");
           expect(window.open.mock.calls[window.open.mock.calls.length - 1][0]).toBe(

static/__tests__/iframe/DataViewer-modal-test.jsx

@@ -60,10 +60,10 @@ describe("DataViewer iframe tests", () => {
     Object.defineProperty(window, "innerHeight", originalInnerHeight);
   });
 
-  test("DataViewer: histogram display in a modal", done => {
+  test("DataViewer: column analysis display in a modal", done => {
     const { DataViewer } = require("../../dtale/DataViewer");
     const ColumnMenu = require("../../dtale/iframe/ColumnMenu").ReactColumnMenu;
-    const Histogram = require("../../popups/Histogram").ReactHistogram;
+    const ColumnAnalysis = require("../../popups/ColumnAnalysis").ReactColumnAnalysis;
 
     const store = reduxUtils.createDtaleStore();
     buildInnerHTML({ settings: "", iframe: "True" }, store);
@@ -102,10 +102,10 @@ describe("DataViewer iframe tests", () => {
         'Column "col2"',
         "should show col2 menu"
       );
-      clickColMenuButton(result, "Histogram");
+      clickColMenuButton(result, "Column Analysis");
       setTimeout(() => {
         result.update();
-        t.equal(result.find(Histogram).length, 1, "should show histogram");
+        t.equal(result.find(ColumnAnalysis).length, 1, "should show column analysis");
         done();
       }, 400);
     }, 600);

static/__tests__/main-test.jsx

@@ -76,7 +76,7 @@ describe("main tests", () => {
     done();
   });
 
-  _.forEach(["correlations", "charts", "describe", "histogram", "instances", "code"], popup => {
+  _.forEach(["correlations", "charts", "describe", "column-analysis", "instances", "code"], popup => {
     test(`${popup} popup rendering`, done => {
       testMain(`popup/${popup}`);
       done();