Added handling for ArcticDB locations with high libraries & symbols

dtale/query.py

@@ -44,7 +44,7 @@ def load_index_filter(data_id):
                 return {"date_range": [pd.Timestamp(start), None]}
             elif end:
                 return {"date_range": [None, pd.Timestamp(end)]}
-    return None
+    return {}
 
 
 def build_query_builder(data_id):

dtale/utils.py

@@ -909,3 +909,7 @@ def _format_colname(colname):
             data.loc[:, col] = data[col].dt.tz_localize(None)
 
     return data, index
+
+
+def option(v):
+    return dict(value=v, label="{}".format(v))

dtale/views.py

@@ -17,6 +17,7 @@
     Response,
 )
 
+import itertools
 import missingno as msno
 import networkx as nx
 import numpy as np
@@ -98,6 +99,7 @@
     read_file,
     make_list,
     optimize_df,
+    option,
     retrieve_grid_params,
     running_with_flask_debug,
     running_with_pytest,
@@ -1177,7 +1179,7 @@ def startup(
             if hide_columns and col["name"] in hide_columns:
                 col["visible"] = False
                 continue
-            if col["index"] > 100:
+            if col["index"] >= 100:
                 col["visible"] = False
         if auto_hide_empty_columns and not global_state.is_arcticdb:
             is_empty = data.isnull().all()
@@ -2589,10 +2591,6 @@ def get_async_column_filter_data(data_id):
     dtype = find_dtype(s)
     fmt = find_dtype_formatter(dtype)
     vals = s[s.astype("str").str.startswith(input)]
-
-    def option(v):
-        return dict(value=v, label="{}".format(v))
-
     vals = [option(fmt(v)) for v in sorted(vals.unique())[:5]]
     return jsonify(vals)
 
@@ -2662,11 +2660,10 @@ def get_data(data_id):
                 if export_rows:
                     if query_builder:
                         data = instance.load_data(
-                            query_builder=query_builder,
-                            **(date_range or {}),
+                            query_builder=query_builder, **date_range
                         )
                         data = data.head(export_rows)
-                    elif date_range:
+                    elif len(date_range):
                         data = instance.load_data(**date_range)
                         data = data.head(export_rows)
                     else:
@@ -2680,8 +2677,10 @@ def get_data(data_id):
             elif query_builder:
                 df = instance.load_data(
                     query_builder=query_builder,
-                    **(date_range or {}),
                     columns=columns_to_load,
+                    # fmt: off
+                    **date_range
+                    # fmt: on
                 )
                 total = len(df)
                 df, _ = format_data(df)
@@ -2704,7 +2703,7 @@ def get_data(data_id):
                         sub_df = f.format_dicts(sub_df.itertuples())
                         for i, d in zip(range(start, end + 1), sub_df):
                             results[i] = dict_merge({IDX_COL: i}, d)
-            elif date_range:
+            elif len(date_range):
                 df = instance.load_data(columns=columns_to_load, **date_range)
                 total = len(df)
                 df, _ = format_data(df)
@@ -4183,18 +4182,52 @@ def get_timeseries_analysis(data_id):
 def get_arcticdb_libraries():
     if get_bool_arg(request, "refresh"):
         global_state.store.load_libraries()
-    ret_data = dict(success=True, libraries=global_state.store.libraries)
+
+    libraries = global_state.store.libraries
+    is_async = False
+    if len(libraries) > 500:
+        is_async = True
+        libraries = libraries[:5]
+    ret_data = {"success": True, "libraries": libraries, "async": is_async}
     if global_state.store.lib is not None:
         ret_data["library"] = global_state.store.lib.name
     return jsonify(ret_data)
 
 
+@dtale.route("/arcticdb/async-libraries")
+@exception_decorator
+def get_async_arcticdb_libraries():
+    libraries = global_state.store.libraries
+    input = get_str_arg(request, "input")
+    vals = list(
+        itertools.islice((option(lib) for lib in libraries if lib.startswith(input)), 5)
+    )
+    return jsonify(vals)
+
+
 @dtale.route("/arcticdb/<library>/symbols")
 @exception_decorator
 def get_arcticdb_symbols(library):
     if get_bool_arg(request, "refresh") or library not in global_state.store._symbols:
         global_state.store.load_symbols(library)
-    return jsonify(dict(success=True, symbols=global_state.store._symbols[library]))
+
+    symbols = global_state.store._symbols[library]
+    is_async = False
+    if len(symbols) > 500:
+        is_async = True
+        symbols = symbols[:5]
+    return jsonify({"success": True, "symbols": symbols, "async": is_async})
+
+
+@dtale.route("/arcticdb/<library>/async-symbols")
+@exception_decorator
+def get_async_arcticdb_symbols(library):
+    symbols = global_state.store._symbols[library]
+    input = get_str_arg(request, "input")
+    vals = list(
+        itertools.islice((option(sym) for sym in symbols if sym.startswith(input)), 5)
+    )
+    return jsonify(vals)
 
 
 @dtale.route("/arcticdb/load-description")

frontend/static/__tests__/dtale/GridCell-test.tsx

@@ -29,11 +29,7 @@ describe('GridCell', () => {
   let editCellSpy: jest.SpyInstance;
   let loadFilterDataSpy: jest.SpyInstance;
 
-  const buildMock = async (
-    propOverrides?: Partial<GridCellProps>,
-    state?: { [key: string]: any },
-    useRerender = false,
-  ): Promise<void> => {
+  const buildMock = async (propOverrides?: Partial<GridCellProps>, state?: { [key: string]: any }): Promise<void> => {
     store = mockStore({
       dataId: '1',
       editedCell: '1|1',
@@ -45,6 +41,7 @@ describe('GridCell', () => {
       ctrlRows: null,
       ctrlCols: null,
       selectedRow: null,
+      columnCount: 0,
       ...state,
     });
     props = {
@@ -81,6 +78,7 @@ describe('GridCell', () => {
       data: {},
       rowCount: 2,
       propagateState: jest.fn(),
+      loading: false,
       ...propOverrides,
     };
 
@@ -124,32 +122,24 @@ describe('GridCell', () => {
   });
 
   it('adds resized class to cell', async () => {
-    await buildMock(undefined, { editedCell: null }, true);
+    await buildMock(undefined, { editedCell: null });
     const divs = container.getElementsByTagName('div');
     expect(divs[divs.length - 1]).toHaveClass('resized');
   });
 
   it('does not add editable class to cell when ArcticDB is active', async () => {
-    await buildMock(undefined, { isArcticDB: 100 }, true);
+    await buildMock(undefined, { isArcticDB: 100 });
     const divs = container.getElementsByTagName('div');
     expect(divs[divs.length - 1]).not.toHaveClass('editable');
   });
 
   it('renders checkbox for boolean column', async () => {
-    await buildMock(
-      { columnIndex: 2, data: { 0: { bar: { raw: 'True', view: 'True' } } } },
-      { editedCell: null },
-      true,
-    );
+    await buildMock({ columnIndex: 2, data: { 0: { bar: { raw: 'True', view: 'True' } } } }, { editedCell: null });
     expect(container.getElementsByClassName('ico-check-box')).toHaveLength(1);
   });
 
   it('renders checkbox for boolean column when edited', async () => {
-    await buildMock(
-      { columnIndex: 2, data: { 0: { bar: { raw: 'True', view: 'True' } } } },
-      { editedCell: '2|1' },
-      true,
-    );
+    await buildMock({ columnIndex: 2, data: { 0: { bar: { raw: 'True', view: 'True' } } } }, { editedCell: '2|1' });
     expect(container.getElementsByClassName('ico-check-box')).toHaveLength(1);
     const checkbox = container.getElementsByClassName('ico-check-box')[0];
     await act(async () => {
@@ -159,7 +149,7 @@ describe('GridCell', () => {
   });
 
   it('renders select for category column when edited', async () => {
-    await buildMock({ columnIndex: 3, data: { 0: { baz: { raw: 'a', view: 'a' } } } }, { editedCell: '3|1' }, true);
+    await buildMock({ columnIndex: 3, data: { 0: { baz: { raw: 'a', view: 'a' } } } }, { editedCell: '3|1' });
     expect(container.getElementsByClassName('Select')).toHaveLength(1);
     const select = document.body.getElementsByClassName('Select')[0] as HTMLElement;
     await act(async () => {
@@ -176,11 +166,7 @@ describe('GridCell', () => {
 
   it('renders select for column w/ custom options when edited', async () => {
     const settings = { column_edit_options: { baz: ['foo', 'bar', 'bizzle'] } };
-    await buildMock(
-      { columnIndex: 3, data: { 0: { baz: { raw: 'a', view: 'a' } } } },
-      { editedCell: '3|1', settings },
-      true,
-    );
+    await buildMock({ columnIndex: 3, data: { 0: { baz: { raw: 'a', view: 'a' } } } }, { editedCell: '3|1', settings });
     expect(container.getElementsByClassName('Select')).toHaveLength(1);
     const select = document.body.getElementsByClassName('Select')[0] as HTMLElement;
     await act(async () => {
@@ -194,4 +180,9 @@ describe('GridCell', () => {
     ]);
     expect(loadFilterDataSpy).not.toHaveBeenCalled();
   });
+
+  it('renders bouncer when loading', async () => {
+    await buildMock({ columnIndex: 0, rowIndex: 0, loading: true });
+    expect(container.getElementsByClassName('bouncer')).toHaveLength(1);
+  });
 });

frontend/static/__tests__/dtale/Header-test.tsx

@@ -41,6 +41,7 @@ describe('Header', () => {
       rowCount: 1,
       propagateState: jest.fn(),
       style: {},
+      loading: false,
       ...propOverrides,
     };
     const result = render(

frontend/static/__tests__/dtale/info/DataViewerInfo-test.tsx

@@ -4,9 +4,10 @@ import { Provider } from 'react-redux';
 import { Store } from 'redux';
 
 import DataViewerInfo, { DataViewerInfoProps } from '../../../dtale/info/DataViewerInfo';
+import * as menuFuncs from '../../../dtale/menu/dataViewerMenuUtils';
 import * as serverState from '../../../dtale/serverStateManagement';
 import { ActionType } from '../../../redux/actions/AppActions';
-import { InstanceSettings, SortDir } from '../../../redux/state/AppState';
+import { InstanceSettings, PopupType, SortDir } from '../../../redux/state/AppState';
 import { RemovableError } from '../../../RemovableError';
 import * as GenericRepository from '../../../repository/GenericRepository';
 import reduxUtils from '../../redux-test-utils';
@@ -17,23 +18,27 @@ describe('DataViewerInfo tests', () => {
   let props: DataViewerInfoProps;
   let postSpy: jest.SpyInstance;
   let updateSettingsSpy: jest.SpyInstance;
+  let menuFuncsOpenPopupSpy: jest.SpyInstance;
 
   beforeEach(() => {
     updateSettingsSpy = jest.spyOn(serverState, 'updateSettings');
     updateSettingsSpy.mockResolvedValue(Promise.resolve({ settings: {} }));
     postSpy = jest.spyOn(GenericRepository, 'postDataToService');
     postSpy.mockResolvedValue(Promise.resolve({ data: {} }));
+    menuFuncsOpenPopupSpy = jest.spyOn(menuFuncs, 'openPopup');
+    menuFuncsOpenPopupSpy.mockImplementation(() => undefined);
   });
 
   afterEach(jest.restoreAllMocks);
 
   const buildInfo = async (
     additionalProps?: Partial<DataViewerInfoProps>,
     settings?: Partial<InstanceSettings>,
+    hiddenProps?: Record<string, string>,
   ): Promise<Element> => {
     props = { propagateState: jest.fn(), columns: [], ...additionalProps };
     store = reduxUtils.createDtaleStore();
-    buildInnerHTML({ settings: '' }, store);
+    buildInnerHTML({ settings: '', ...hiddenProps }, store);
     if (settings) {
       store.dispatch({ type: ActionType.UPDATE_SETTINGS, settings });
     }
@@ -140,4 +145,40 @@ describe('DataViewerInfo tests', () => {
       ['baz', SortDir.ASC],
     ]);
   });
+
+  it('DataViewerInfo rendering ArcticDB', async () => {
+    const result = await buildInfo(
+      undefined,
+      {
+        allow_cell_edits: true,
+        hide_shutdown: false,
+        precision: 2,
+        verticalHeaders: false,
+        predefinedFilters: {},
+        hide_header_editor: false,
+        isArcticDB: 100,
+      },
+      { dataId: 'lib|symbol', isArcticDB: '100', arcticConn: 'arctic_uri', columnCount: '101' },
+    );
+    expect(result.getElementsByClassName('data-viewer-info')[0]).toHaveStyle({ background: 'rgb(255, 230, 0)' });
+    const arcticToggle = result.getElementsByClassName('arctic-menu-toggle')[0];
+    expect(arcticToggle.getElementsByTagName('span')[0].textContent).toBe('arctic_uri (lib: lib, symbol: symbol)');
+    await act(async () => {
+      fireEvent.click(arcticToggle);
+    });
+    expect(screen.getByText('Load ArcticDB Data')).toBeDefined();
+    await act(async () => {
+      fireEvent.click(screen.getByText('Load ArcticDB Data'));
+    });
+    expect(menuFuncsOpenPopupSpy.mock.calls[0][0]).toEqual({ type: PopupType.ARCTICDB, visible: true });
+    expect(screen.getByText('Jump To Column')).toBeDefined();
+    await act(async () => {
+      fireEvent.click(screen.getByText('Jump To Column'));
+    });
+    expect(menuFuncsOpenPopupSpy.mock.calls[1][0]).toEqual({
+      type: PopupType.JUMP_TO_COLUMN,
+      columns: [],
+      visible: true,
+    });
+  });
 });

frontend/static/__tests__/dtale/menu/DataViewerMenu-test.tsx

@@ -136,6 +136,11 @@ describe('DataViewerMenu tests', () => {
     expect(store.getState().chartData).toEqual({ visible: true, type: PopupType.EXPORT, rows: 50, size: 'sm' });
   });
 
+  it('renders "Jump To Column"', async () => {
+    buildMenu({ columnCount: '101' });
+    expect(screen.getByText('Jump To Column')).toBeDefined();
+  });
+
   describe('iframe handling', () => {
     const { self, top } = window;
     const resourceBaseUrl = (window as any).resourceBaseUrl;