diff --git a/dtale/query.py b/dtale/query.py index 122097f5..c2dfaba2 100644 --- a/dtale/query.py +++ b/dtale/query.py @@ -44,7 +44,7 @@ def load_index_filter(data_id): return {"date_range": [pd.Timestamp(start), None]} elif end: return {"date_range": [None, pd.Timestamp(end)]} - return None + return {} def build_query_builder(data_id): diff --git a/dtale/utils.py b/dtale/utils.py index 0e7572d7..6deaf1d3 100644 --- a/dtale/utils.py +++ b/dtale/utils.py @@ -909,3 +909,7 @@ def _format_colname(colname): data.loc[:, col] = data[col].dt.tz_localize(None) return data, index + + +def option(v): + return dict(value=v, label="{}".format(v)) diff --git a/dtale/views.py b/dtale/views.py index abf4a780..cb4bef87 100644 --- a/dtale/views.py +++ b/dtale/views.py @@ -17,6 +17,7 @@ Response, ) +import itertools import missingno as msno import networkx as nx import numpy as np @@ -98,6 +99,7 @@ read_file, make_list, optimize_df, + option, retrieve_grid_params, running_with_flask_debug, running_with_pytest, @@ -1177,7 +1179,7 @@ def startup( if hide_columns and col["name"] in hide_columns: col["visible"] = False continue - if col["index"] > 100: + if col["index"] >= 100: col["visible"] = False if auto_hide_empty_columns and not global_state.is_arcticdb: is_empty = data.isnull().all() @@ -2589,10 +2591,6 @@ def get_async_column_filter_data(data_id): dtype = find_dtype(s) fmt = find_dtype_formatter(dtype) vals = s[s.astype("str").str.startswith(input)] - - def option(v): - return dict(value=v, label="{}".format(v)) - vals = [option(fmt(v)) for v in sorted(vals.unique())[:5]] return jsonify(vals) @@ -2662,11 +2660,10 @@ def get_data(data_id): if export_rows: if query_builder: data = instance.load_data( - query_builder=query_builder, - **(date_range or {}), + query_builder=query_builder, **date_range ) data = data.head(export_rows) - elif date_range: + elif len(date_range): data = instance.load_data(**date_range) data = data.head(export_rows) else: @@ -2680,8 +2677,10 @@ def get_data(data_id): elif query_builder: df = instance.load_data( query_builder=query_builder, - **(date_range or {}), columns=columns_to_load, + # fmt: off + **date_range + # fmt: on ) total = len(df) df, _ = format_data(df) @@ -2704,7 +2703,7 @@ def get_data(data_id): sub_df = f.format_dicts(sub_df.itertuples()) for i, d in zip(range(start, end + 1), sub_df): results[i] = dict_merge({IDX_COL: i}, d) - elif date_range: + elif len(date_range): df = instance.load_data(columns=columns_to_load, **date_range) total = len(df) df, _ = format_data(df) @@ -4183,18 +4182,52 @@ def get_timeseries_analysis(data_id): def get_arcticdb_libraries(): if get_bool_arg(request, "refresh"): global_state.store.load_libraries() - ret_data = dict(success=True, libraries=global_state.store.libraries) + + libraries = global_state.store.libraries + is_async = False + if len(libraries) > 500: + is_async = True + libraries = libraries[:5] + ret_data = {"success": True, "libraries": libraries, "async": is_async} if global_state.store.lib is not None: ret_data["library"] = global_state.store.lib.name return jsonify(ret_data) +@dtale.route("/arcticdb/async-libraries") +@exception_decorator +def get_async_arcticdb_libraries(): + libraries = global_state.store.libraries + input = get_str_arg(request, "input") + vals = list( + itertools.islice((option(lib) for lib in libraries if lib.startswith(input)), 5) + ) + return jsonify(vals) + + @dtale.route("/arcticdb//symbols") @exception_decorator def get_arcticdb_symbols(library): if get_bool_arg(request, "refresh") or library not in global_state.store._symbols: global_state.store.load_symbols(library) - return jsonify(dict(success=True, symbols=global_state.store._symbols[library])) + + symbols = global_state.store._symbols[library] + is_async = False + if len(symbols) > 500: + is_async = True + symbols = symbols[:5] + return jsonify({"success": True, "symbols": symbols, "async": is_async}) + + +@dtale.route("/arcticdb//async-symbols") +@exception_decorator +def get_async_arcticdb_symbols(library): + symbols = global_state.store._symbols[library] + input = get_str_arg(request, "input") + vals = list( + itertools.islice((option(sym) for sym in symbols if sym.startswith(input)), 5) + ) + return jsonify(vals) @dtale.route("/arcticdb/load-description") diff --git a/frontend/static/__tests__/dtale/GridCell-test.tsx b/frontend/static/__tests__/dtale/GridCell-test.tsx index e78ec3ae..a94882a9 100644 --- a/frontend/static/__tests__/dtale/GridCell-test.tsx +++ b/frontend/static/__tests__/dtale/GridCell-test.tsx @@ -29,11 +29,7 @@ describe('GridCell', () => { let editCellSpy: jest.SpyInstance; let loadFilterDataSpy: jest.SpyInstance; - const buildMock = async ( - propOverrides?: Partial, - state?: { [key: string]: any }, - useRerender = false, - ): Promise => { + const buildMock = async (propOverrides?: Partial, state?: { [key: string]: any }): Promise => { store = mockStore({ dataId: '1', editedCell: '1|1', @@ -45,6 +41,7 @@ describe('GridCell', () => { ctrlRows: null, ctrlCols: null, selectedRow: null, + columnCount: 0, ...state, }); props = { @@ -81,6 +78,7 @@ describe('GridCell', () => { data: {}, rowCount: 2, propagateState: jest.fn(), + loading: false, ...propOverrides, }; @@ -124,32 +122,24 @@ describe('GridCell', () => { }); it('adds resized class to cell', async () => { - await buildMock(undefined, { editedCell: null }, true); + await buildMock(undefined, { editedCell: null }); const divs = container.getElementsByTagName('div'); expect(divs[divs.length - 1]).toHaveClass('resized'); }); it('does not add editable class to cell when ArcticDB is active', async () => { - await buildMock(undefined, { isArcticDB: 100 }, true); + await buildMock(undefined, { isArcticDB: 100 }); const divs = container.getElementsByTagName('div'); expect(divs[divs.length - 1]).not.toHaveClass('editable'); }); it('renders checkbox for boolean column', async () => { - await buildMock( - { columnIndex: 2, data: { 0: { bar: { raw: 'True', view: 'True' } } } }, - { editedCell: null }, - true, - ); + await buildMock({ columnIndex: 2, data: { 0: { bar: { raw: 'True', view: 'True' } } } }, { editedCell: null }); expect(container.getElementsByClassName('ico-check-box')).toHaveLength(1); }); it('renders checkbox for boolean column when edited', async () => { - await buildMock( - { columnIndex: 2, data: { 0: { bar: { raw: 'True', view: 'True' } } } }, - { editedCell: '2|1' }, - true, - ); + await buildMock({ columnIndex: 2, data: { 0: { bar: { raw: 'True', view: 'True' } } } }, { editedCell: '2|1' }); expect(container.getElementsByClassName('ico-check-box')).toHaveLength(1); const checkbox = container.getElementsByClassName('ico-check-box')[0]; await act(async () => { @@ -159,7 +149,7 @@ describe('GridCell', () => { }); it('renders select for category column when edited', async () => { - await buildMock({ columnIndex: 3, data: { 0: { baz: { raw: 'a', view: 'a' } } } }, { editedCell: '3|1' }, true); + await buildMock({ columnIndex: 3, data: { 0: { baz: { raw: 'a', view: 'a' } } } }, { editedCell: '3|1' }); expect(container.getElementsByClassName('Select')).toHaveLength(1); const select = document.body.getElementsByClassName('Select')[0] as HTMLElement; await act(async () => { @@ -176,11 +166,7 @@ describe('GridCell', () => { it('renders select for column w/ custom options when edited', async () => { const settings = { column_edit_options: { baz: ['foo', 'bar', 'bizzle'] } }; - await buildMock( - { columnIndex: 3, data: { 0: { baz: { raw: 'a', view: 'a' } } } }, - { editedCell: '3|1', settings }, - true, - ); + await buildMock({ columnIndex: 3, data: { 0: { baz: { raw: 'a', view: 'a' } } } }, { editedCell: '3|1', settings }); expect(container.getElementsByClassName('Select')).toHaveLength(1); const select = document.body.getElementsByClassName('Select')[0] as HTMLElement; await act(async () => { @@ -194,4 +180,9 @@ describe('GridCell', () => { ]); expect(loadFilterDataSpy).not.toHaveBeenCalled(); }); + + it('renders bouncer when loading', async () => { + await buildMock({ columnIndex: 0, rowIndex: 0, loading: true }); + expect(container.getElementsByClassName('bouncer')).toHaveLength(1); + }); }); diff --git a/frontend/static/__tests__/dtale/Header-test.tsx b/frontend/static/__tests__/dtale/Header-test.tsx index c4d5c0e4..81e1412b 100644 --- a/frontend/static/__tests__/dtale/Header-test.tsx +++ b/frontend/static/__tests__/dtale/Header-test.tsx @@ -41,6 +41,7 @@ describe('Header', () => { rowCount: 1, propagateState: jest.fn(), style: {}, + loading: false, ...propOverrides, }; const result = render( diff --git a/frontend/static/__tests__/dtale/info/DataViewerInfo-test.tsx b/frontend/static/__tests__/dtale/info/DataViewerInfo-test.tsx index 0a8c2ef6..a8c880f8 100644 --- a/frontend/static/__tests__/dtale/info/DataViewerInfo-test.tsx +++ b/frontend/static/__tests__/dtale/info/DataViewerInfo-test.tsx @@ -4,9 +4,10 @@ import { Provider } from 'react-redux'; import { Store } from 'redux'; import DataViewerInfo, { DataViewerInfoProps } from '../../../dtale/info/DataViewerInfo'; +import * as menuFuncs from '../../../dtale/menu/dataViewerMenuUtils'; import * as serverState from '../../../dtale/serverStateManagement'; import { ActionType } from '../../../redux/actions/AppActions'; -import { InstanceSettings, SortDir } from '../../../redux/state/AppState'; +import { InstanceSettings, PopupType, SortDir } from '../../../redux/state/AppState'; import { RemovableError } from '../../../RemovableError'; import * as GenericRepository from '../../../repository/GenericRepository'; import reduxUtils from '../../redux-test-utils'; @@ -17,12 +18,15 @@ describe('DataViewerInfo tests', () => { let props: DataViewerInfoProps; let postSpy: jest.SpyInstance; let updateSettingsSpy: jest.SpyInstance; + let menuFuncsOpenPopupSpy: jest.SpyInstance; beforeEach(() => { updateSettingsSpy = jest.spyOn(serverState, 'updateSettings'); updateSettingsSpy.mockResolvedValue(Promise.resolve({ settings: {} })); postSpy = jest.spyOn(GenericRepository, 'postDataToService'); postSpy.mockResolvedValue(Promise.resolve({ data: {} })); + menuFuncsOpenPopupSpy = jest.spyOn(menuFuncs, 'openPopup'); + menuFuncsOpenPopupSpy.mockImplementation(() => undefined); }); afterEach(jest.restoreAllMocks); @@ -30,10 +34,11 @@ describe('DataViewerInfo tests', () => { const buildInfo = async ( additionalProps?: Partial, settings?: Partial, + hiddenProps?: Record, ): Promise => { props = { propagateState: jest.fn(), columns: [], ...additionalProps }; store = reduxUtils.createDtaleStore(); - buildInnerHTML({ settings: '' }, store); + buildInnerHTML({ settings: '', ...hiddenProps }, store); if (settings) { store.dispatch({ type: ActionType.UPDATE_SETTINGS, settings }); } @@ -140,4 +145,40 @@ describe('DataViewerInfo tests', () => { ['baz', SortDir.ASC], ]); }); + + it('DataViewerInfo rendering ArcticDB', async () => { + const result = await buildInfo( + undefined, + { + allow_cell_edits: true, + hide_shutdown: false, + precision: 2, + verticalHeaders: false, + predefinedFilters: {}, + hide_header_editor: false, + isArcticDB: 100, + }, + { dataId: 'lib|symbol', isArcticDB: '100', arcticConn: 'arctic_uri', columnCount: '101' }, + ); + expect(result.getElementsByClassName('data-viewer-info')[0]).toHaveStyle({ background: 'rgb(255, 230, 0)' }); + const arcticToggle = result.getElementsByClassName('arctic-menu-toggle')[0]; + expect(arcticToggle.getElementsByTagName('span')[0].textContent).toBe('arctic_uri (lib: lib, symbol: symbol)'); + await act(async () => { + fireEvent.click(arcticToggle); + }); + expect(screen.getByText('Load ArcticDB Data')).toBeDefined(); + await act(async () => { + fireEvent.click(screen.getByText('Load ArcticDB Data')); + }); + expect(menuFuncsOpenPopupSpy.mock.calls[0][0]).toEqual({ type: PopupType.ARCTICDB, visible: true }); + expect(screen.getByText('Jump To Column')).toBeDefined(); + await act(async () => { + fireEvent.click(screen.getByText('Jump To Column')); + }); + expect(menuFuncsOpenPopupSpy.mock.calls[1][0]).toEqual({ + type: PopupType.JUMP_TO_COLUMN, + columns: [], + visible: true, + }); + }); }); diff --git a/frontend/static/__tests__/dtale/menu/DataViewerMenu-test.tsx b/frontend/static/__tests__/dtale/menu/DataViewerMenu-test.tsx index db16ca21..2ef3cdb7 100644 --- a/frontend/static/__tests__/dtale/menu/DataViewerMenu-test.tsx +++ b/frontend/static/__tests__/dtale/menu/DataViewerMenu-test.tsx @@ -136,6 +136,11 @@ describe('DataViewerMenu tests', () => { expect(store.getState().chartData).toEqual({ visible: true, type: PopupType.EXPORT, rows: 50, size: 'sm' }); }); + it('renders "Jump To Column"', async () => { + buildMenu({ columnCount: '101' }); + expect(screen.getByText('Jump To Column')).toBeDefined(); + }); + describe('iframe handling', () => { const { self, top } = window; const resourceBaseUrl = (window as any).resourceBaseUrl; diff --git a/frontend/static/__tests__/popups/arcticdb/JumpToColumn-test.tsx b/frontend/static/__tests__/popups/arcticdb/JumpToColumn-test.tsx new file mode 100644 index 00000000..1a38e055 --- /dev/null +++ b/frontend/static/__tests__/popups/arcticdb/JumpToColumn-test.tsx @@ -0,0 +1,85 @@ +import { act, render } from '@testing-library/react'; +import axios from 'axios'; +import * as React from 'react'; +import { Provider } from 'react-redux'; +import selectEvent from 'react-select-event'; + +import * as serverState from '../../../dtale/serverStateManagement'; +import JumpToColumn from '../../../popups/arcticdb/JumpToColumn'; +import { ActionType } from '../../../redux/actions/AppActions'; +import { JumpToColumnPopupData, PopupType } from '../../../redux/state/AppState'; +import { mockColumnDef } from '../../mocks/MockColumnDef'; +import reduxUtils from '../../redux-test-utils'; +import { buildInnerHTML, selectOption } from '../../test-utils'; + +const props: JumpToColumnPopupData = { + visible: true, + type: PopupType.JUMP_TO_COLUMN, + title: 'JumpToColumn Test', + columns: [ + mockColumnDef({ index: 0, visible: true, locked: true }), + mockColumnDef({ + name: 'foo', + index: 1, + dtype: 'int64', + visible: true, + width: 100, + resized: true, + }), + mockColumnDef({ + name: 'bar', + index: 2, + dtype: 'bool', + visible: true, + width: 100, + resized: false, + }), + ], +}; + +describe('JumpToColumn tests', () => { + let serverStateSpy: jest.SpyInstance; + let propagateStateSpy: jest.Mock; + + const updateProps = async (chartData?: Partial): Promise => { + const store = reduxUtils.createDtaleStore(); + buildInnerHTML({ settings: '', isArcticDB: '100' }, store); + store.dispatch({ type: ActionType.OPEN_CHART, chartData: { ...props, ...chartData } }); + if (chartData?.visible === false) { + store.dispatch({ type: ActionType.CLOSE_CHART }); + } + propagateStateSpy = jest.fn(); + await act( + async () => + await render( + + + , + { + container: document.getElementById('content') ?? undefined, + }, + ).container, + ); + }; + + beforeEach(async () => { + (axios.get as any).mockImplementation((url: string) => { + return Promise.resolve({ data: reduxUtils.urlFetcher(url) }); + }); + serverStateSpy = jest.spyOn(serverState, 'updateVisibility'); + serverStateSpy.mockImplementation(() => undefined); + }); + + afterEach(jest.restoreAllMocks); + + it('JumpToColumn rendering columns', async () => { + await updateProps(); + const columnSelect = document.body.getElementsByClassName('Select')[0] as HTMLElement; + await act(async () => { + await selectEvent.openMenu(columnSelect); + }); + await selectOption(columnSelect, 'foo'); + expect(serverStateSpy).toHaveBeenCalledWith('1', { bar: false, col: true, foo: true }); + expect(propagateStateSpy).toHaveBeenCalledWith(expect.objectContaining({ refresh: true, triggerResize: true })); + }); +}); diff --git a/frontend/static/__tests__/popups/arcticdb/LibrarySymbolSelector-test.tsx b/frontend/static/__tests__/popups/arcticdb/LibrarySymbolSelector-test.tsx index c33e1742..84174905 100644 --- a/frontend/static/__tests__/popups/arcticdb/LibrarySymbolSelector-test.tsx +++ b/frontend/static/__tests__/popups/arcticdb/LibrarySymbolSelector-test.tsx @@ -55,7 +55,9 @@ describe('LibrarySymbolSelector tests', () => { } else if (url.startsWith('/dtale/arcticdb/foo/symbols')) { return Promise.resolve({ data: { symbols: SYMBOLS.foo } }); } else if (url.startsWith('/dtale/arcticdb/bar/symbols')) { - return Promise.resolve({ data: { symbols: SYMBOLS.bar } }); + return Promise.resolve({ data: { symbols: SYMBOLS.bar, async: true } }); + } else if (url.startsWith('/dtale/arcticdb/bar/async-symbols')) { + return Promise.resolve({ data: [{ label: 'bar4', value: 'bar4' }] }); } else if (url.startsWith('/dtale/arcticdb/baz/symbols')) { return Promise.resolve({ data: { symbols: SYMBOLS.baz } }); } else if (url.startsWith('/dtale/arcticdb/load-description')) { @@ -120,4 +122,16 @@ describe('LibrarySymbolSelector tests', () => { expect(loadSymbolSpy).toHaveBeenCalledWith('foo', 'foo1'); expect(jumpToDatasetSpy).toHaveBeenLastCalledWith('2'); }); + + it('LibrarySymbolSelector rendering async symbols', async () => { + await updateProps(); + const librarySelect = document.body.getElementsByClassName('Select')[0] as HTMLElement; + await selectOption(librarySelect, 'bar'); + const symbolSelect = (): HTMLElement => document.body.getElementsByClassName('Select')[1] as HTMLElement; + const asyncSymbolsSpy = jest.spyOn(ArcticDBRepository, 'asyncSymbols'); + await act(async () => { + fireEvent.change(symbolSelect().getElementsByTagName('input')[0], { target: { value: 'b' } }); + }); + expect(asyncSymbolsSpy).toHaveBeenCalledWith('bar', 'b'); + }); }); diff --git a/frontend/static/__tests__/reducers/dtale-test.tsx b/frontend/static/__tests__/reducers/dtale-test.tsx index 71cfb917..f3fd5013 100644 --- a/frontend/static/__tests__/reducers/dtale-test.tsx +++ b/frontend/static/__tests__/reducers/dtale-test.tsx @@ -73,6 +73,8 @@ describe('reducer tests', () => { showAllHeatmapColumns: false, isVSCode: false, isArcticDB: 0.0, + arcticConn: '', + columnCount: 0, queryEngine: 'python', openCustomFilterOnStartup: false, openPredefinedFiltersOnStartup: false, diff --git a/frontend/static/__tests__/test-utils.tsx b/frontend/static/__tests__/test-utils.tsx index fd9687b0..98fbba01 100644 --- a/frontend/static/__tests__/test-utils.tsx +++ b/frontend/static/__tests__/test-utils.tsx @@ -76,6 +76,7 @@ export const buildInnerHTML = (props: Record = {}, s buildHidden('xarray_dim', props.xarrayDim ?? '{}'), buildHidden('allow_cell_edits', props.allowCellEdits ?? 'true'), buildHidden('is_vscode', props.isVSCode ?? 'False'), + buildHidden('arctic_conn', props.arcticConn ?? ''), buildHidden('is_arcticdb', props.isArcticDB ?? '0'), buildHidden('column_count', props.columnCount ?? '0'), buildHidden('theme', props.theme ?? 'light'), diff --git a/frontend/static/dtale/gridUtils.tsx b/frontend/static/dtale/gridUtils.tsx index e39f0e31..255c38f2 100644 --- a/frontend/static/dtale/gridUtils.tsx +++ b/frontend/static/dtale/gridUtils.tsx @@ -194,7 +194,6 @@ export const calcColWidth = ( sortInfo?: SortDef[], backgroundMode?: string, maxColumnWidth?: number, - isWide?: boolean, ): Partial => { const { name, dtype, hasMissing, hasOutliers, lowVariance, resized, width, headerWidth, dataWidth } = colCfg; if (resized === true) { @@ -215,12 +214,7 @@ export const calcColWidth = ( } else if (backgroundMode === 'lowVariance' && lowVariance) { updatedHeaderWidth += 15; // star emoji } - let updatedDataWidth = updatedHeaderWidth; - if (isWide) { - updatedDataWidth = updatedDataWidth < 100 ? 100 : updatedDataWidth; - } else { - updatedDataWidth = calcDataWidth(name, dtype, data) ?? DEFAULT_COL_WIDTH; - } + const updatedDataWidth = calcDataWidth(name, dtype, data) ?? DEFAULT_COL_WIDTH; w = updatedHeaderWidth > updatedDataWidth ? updatedHeaderWidth : updatedDataWidth; w = maxColumnWidth && w >= maxColumnWidth ? { width: maxColumnWidth, resized: true } : { width: w }; w = { ...w, headerWidth: updatedHeaderWidth, dataWidth: updatedDataWidth }; diff --git a/frontend/static/dtale/info/DataViewerInfo.tsx b/frontend/static/dtale/info/DataViewerInfo.tsx index 8c073b0c..3307050a 100644 --- a/frontend/static/dtale/info/DataViewerInfo.tsx +++ b/frontend/static/dtale/info/DataViewerInfo.tsx @@ -215,7 +215,7 @@ const DataViewerInfo: React.FC = ({ colum lib {`: ${library}, `} symbol - {`: ${symbol})`} + {`: ${symbol ?? ''})`}
= ({ loadi }} onClick={menuHandler} > - {!loading && } - {loading && } + {loading ? : }
)}
{Math.max(rowCount - 1, 0)}
diff --git a/frontend/static/dtale/ribbon/RibbonDropdown.tsx b/frontend/static/dtale/ribbon/RibbonDropdown.tsx index d3206920..a24518d0 100644 --- a/frontend/static/dtale/ribbon/RibbonDropdown.tsx +++ b/frontend/static/dtale/ribbon/RibbonDropdown.tsx @@ -34,6 +34,7 @@ import HeatMapOption from '../menu/HeatMapOption'; import HideHeaderEditor from '../menu/HideHeaderEditor'; import HighlightOption from '../menu/HighlightOption'; import InstancesOption from '../menu/InstancesOption'; +import JumpToColumnOption from '../menu/JumpToColumnOption'; import LanguageOption from '../menu/LanguageOption'; import LogoutOption from '../menu/LogoutOption'; import LowVarianceOption from '../menu/LowVarianceOption'; @@ -179,6 +180,11 @@ const RibbonDropdown: React.FC = ({ colum {!!isArcticDB && ( )} + {columnCount > 100 && ( + + )} {!isArcticDB && ( )} diff --git a/frontend/static/filters/AsyncValueSelect.tsx b/frontend/static/filters/AsyncValueSelect.tsx index cd8df221..7c9126a3 100644 --- a/frontend/static/filters/AsyncValueSelect.tsx +++ b/frontend/static/filters/AsyncValueSelect.tsx @@ -13,6 +13,7 @@ export interface AsyncValueSelectProps { selectedCol: string; selected?: T | T[]; isMulti?: boolean; + loader?: (input: string) => Promise>; } /** State properties for AsyncValueSelect */ @@ -105,6 +106,9 @@ export default class AsyncValueSelect extends React.Component> { + if (this.props.loader) { + return this.props.loader(input); + } return ColumnFilterRepository.loadAsyncData(this.props.dataId, this.props.selectedCol, input).then( (response) => response!, ); diff --git a/frontend/static/popups/arcticdb/LibrarySymbolSelector.tsx b/frontend/static/popups/arcticdb/LibrarySymbolSelector.tsx index c32f5893..5b89b5ed 100644 --- a/frontend/static/popups/arcticdb/LibrarySymbolSelector.tsx +++ b/frontend/static/popups/arcticdb/LibrarySymbolSelector.tsx @@ -3,6 +3,7 @@ import { withTranslation, WithTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; import { BouncerWrapper } from '../../BouncerWrapper'; +import AsyncValueSelect from '../../filters/AsyncValueSelect'; import { AppState, BaseOption } from '../../redux/state/AppState'; import { RemovableError } from '../../RemovableError'; import * as ArcticDBRepository from '../../repository/ArcticDBRepository'; @@ -18,10 +19,10 @@ const LibrarySymbolSelector: React.FC = ({ t }) => { }, [dataId]); const [library, setLibrary] = React.useState>(); - const [libraries, setLibraries] = React.useState([]); + const [libraries, setLibraries] = React.useState(); const [loadingLibraries, setLoadingLibraries] = React.useState(true); const [symbol, setSymbol] = React.useState>(); - const [symbols, setSymbols] = React.useState([]); + const [symbols, setSymbols] = React.useState(); const [loadingSymbols, setLoadingSymbols] = React.useState(false); const [loadingSymbol, setLoadingSymbol] = React.useState(false); const [error, setError] = React.useState(); @@ -38,7 +39,7 @@ const LibrarySymbolSelector: React.FC = ({ t }) => { } if (response) { setError(undefined); - setLibraries(response.libraries); + setLibraries(response); if (response.library) { setLibrary({ value: response.library }); } @@ -56,8 +57,8 @@ const LibrarySymbolSelector: React.FC = ({ t }) => { } if (response) { setError(undefined); - setSymbols(response.symbols); - if (response.symbols.includes(currentSymbol)) { + setSymbols(response); + if (response.async && response.symbols.includes(currentSymbol)) { setSymbol({ value: currentSymbol }); } else { setSymbol(undefined); @@ -73,7 +74,7 @@ const LibrarySymbolSelector: React.FC = ({ t }) => { if (library) { loadSymbols(library.value); } else { - setSymbols([]); + setSymbols(undefined); } }, [library]); @@ -119,27 +120,77 @@ const LibrarySymbolSelector: React.FC = ({ t }) => { )} - ({ value: l }))} - value={library} - onChange={(selected) => setLibrary(selected as BaseOption)} - > -
- loadLibraries(true)} /> -
-
- + {!libraries?.async && ( ({ value: s }))} - value={symbol} - onChange={(selected) => setSymbol(selected as BaseOption)} + label={t('Library')} + options={(libraries?.libraries ?? []).map((l) => ({ value: l }))} + value={library} + onChange={(selected) => setLibrary(selected as BaseOption)} >
- loadSymbols(library!.value, true)} /> + loadLibraries(true)} />
+ )} + {!!libraries?.async && ( +
+ +
+ + {...{ + dataId, + selected: library?.value, + selectedCol: '', + uniques: libraries.libraries, + missing: false, + }} + isMulti={false} + loader={(input: string): Promise> => + ArcticDBRepository.asyncLibraries(input).then((response) => response!) + } + updateState={async (option?: string[] | string) => { + setLibrary(option ? { value: option as string } : undefined); + }} + /> +
+
+ loadLibraries(true)} /> +
+
+ )} + + {!symbols?.async && ( + ({ value: s }))} + value={symbol} + onChange={(selected) => setSymbol(selected as BaseOption)} + > +
+ loadSymbols(library!.value, true)} /> +
+
+ )} + {!!symbols?.async && ( +
+ +
+ + {...{ dataId, selected: symbol?.value, selectedCol: '', uniques: symbols.symbols, missing: false }} + isMulti={false} + loader={(input: string): Promise> => + ArcticDBRepository.asyncSymbols(library!.value, input).then((response) => response!) + } + updateState={async (option?: string[] | string) => { + setSymbol(option ? { value: option as string } : undefined); + }} + /> +
+
+ loadSymbols(library!.value, true)} /> +
+
+ )}
{!!description && ( diff --git a/frontend/static/repository/ArcticDBRepository.ts b/frontend/static/repository/ArcticDBRepository.ts index 0c2b5e49..b7144670 100644 --- a/frontend/static/repository/ArcticDBRepository.ts +++ b/frontend/static/repository/ArcticDBRepository.ts @@ -1,16 +1,19 @@ import { buildURLString } from '../redux/actions/url-utils'; +import { AsyncColumnFilterDataResponse } from './ColumnFilterRepository'; import * as GenericRepository from './GenericRepository'; /** Axios response for loading libraries */ export interface LibrariesResponse extends GenericRepository.BaseResponse { libraries: string[]; library?: string; + async: boolean; } /** Axios response for loading symbols */ export interface SymbolsResponse extends GenericRepository.BaseResponse { symbols: string[]; + async: boolean; } /** Axios response for loading symbol */ @@ -37,6 +40,18 @@ export async function libraries(refresh?: boolean): Promise | undefined> { + return await GenericRepository.getDataFromService>( + buildURLString(`/dtale/arcticdb/async-libraries`, { input: input ?? '' }), + ); +} + /** * Load symbols for the current ArcticDB library. * @@ -50,6 +65,22 @@ export async function symbols(library: string, refresh?: boolean): Promise | undefined> { + return await GenericRepository.getDataFromService>( + buildURLString(`/dtale/arcticdb/${library}/async-symbols`, { input: input ?? '' }), + ); +} + /** * Load description for library/symbol in the current ArcticDB host. * diff --git a/frontend/static/repository/ColumnFilterRepository.ts b/frontend/static/repository/ColumnFilterRepository.ts index 3bc91924..95c003bd 100644 --- a/frontend/static/repository/ColumnFilterRepository.ts +++ b/frontend/static/repository/ColumnFilterRepository.ts @@ -15,7 +15,7 @@ export interface ColumnFilterData { type ColumnFilterDataResponse = GenericRepository.BaseResponse & ColumnFilterData; /** Axios response type for loading asynchronous filtering options */ -type AsyncColumnFilterDataResponse = Array<{ label: string; value: T }>; +export type AsyncColumnFilterDataResponse = Array<{ label: string; value: T }>; /** * Load information related to filtering for a column. diff --git a/frontend/static/translations/cn.json b/frontend/static/translations/cn.json index 225f1b2c..c4ba3cfb 100644 --- a/frontend/static/translations/cn.json +++ b/frontend/static/translations/cn.json @@ -57,7 +57,8 @@ "gage_rnr": "量具重复性和再现性", "Vertical Column Headers": "纵向列标题", "Hide Header Editor": "Hide Header Editor", - "Load ArcticDB Data": "Load ArcticDB Data" + "Load ArcticDB Data": "Load ArcticDB Data", + "Jump To Column": "Jump To Column" }, "menu_description": { "describe": "描述每列的值(最上面的唯一值,最小值,最大值,总和,STD,变量,...)。", @@ -107,7 +108,8 @@ "vertical_headers": "Rotate column headers vertically", "timeseries_analysis": "Analyze time series data using different \"statsmodels.tsa\" functions", "hide_header_editor": "If checked, hides the header editor when editing cells in the grid", - "arcticdb": "Load data from a specific library & symbol in ArcticDB" + "arcticdb": "Load data from a specific library & symbol in ArcticDB", + "jump_to_column": "For wide dataframes, you can shrink the columns displayed down to what is locked and a column selected from a dropdown." }, "column_menu": { "Asc": "Asc", @@ -938,7 +940,8 @@ "Invert Filter": "Invert Filter", "Move Filters To Custom": "Move Filters To Custom", "Hide Filtered Rows": "Hide Filtered Rows", - "Highlight Filtered Rows": "Highlight Filtered Rows" + "Highlight Filtered Rows": "Highlight Filtered Rows", + "ArcticDB": "ArcticDB" }, "side": { "Close": "Close", diff --git a/frontend/static/translations/en.json b/frontend/static/translations/en.json index 197aef74..bb42eced 100644 --- a/frontend/static/translations/en.json +++ b/frontend/static/translations/en.json @@ -58,7 +58,8 @@ "gage_rnr": "Gage R & R", "Vertical Column Headers": "Vertical Column Headers", "Hide Header Editor": "Hide Header Editor", - "Load ArcticDB Data": "Load ArcticDB Data" + "Load ArcticDB Data": "Load ArcticDB Data", + "Jump To Column": "Jump To Column" }, "menu_description": { "describe": "Describe column's values (Top unique values, Min, Max, Sum, STD, Var,...)", @@ -108,7 +109,8 @@ "vertical_headers": "Rotate column headers vertically", "timeseries_analysis": "Analyze time series data using different \"statsmodels.tsa\" functions", "hide_header_editor": "If checked, hides the header editor when editing cells in the grid", - "arcticdb": "Load data from a specific library & symbol in ArcticDB" + "arcticdb": "Load data from a specific library & symbol in ArcticDB", + "jump_to_column": "For wide dataframes, you can shrink the columns displayed down to what is locked and a column selected from a dropdown." }, "column_menu": { "Asc": "Asc", @@ -940,7 +942,8 @@ "Invert Filter": "Invert Filter", "Move Filters To Custom": "Move Filters To Custom", "Hide Filtered Rows": "Hide Filtered Rows", - "Highlight Filtered Rows": "Highlight Filtered Rows" + "Highlight Filtered Rows": "Highlight Filtered Rows", + "ArcticDB": "ArcticDB" }, "side": { "Close": "Close", diff --git a/frontend/static/translations/pt.json b/frontend/static/translations/pt.json index 78611441..d6c65d1d 100644 --- a/frontend/static/translations/pt.json +++ b/frontend/static/translations/pt.json @@ -58,7 +58,8 @@ "gage_rnr": "Gage R & R", "Vertical Column Headers": "Vertical Column Headers", "Hide Header Editor": "Hide Header Editor", - "Load ArcticDB Data": "Load ArcticDB Data" + "Load ArcticDB Data": "Load ArcticDB Data", + "Jump To Column": "Jump To Column" }, "menu_description": { "describe": "Descreve valores das colunas (Valores únicos, Min, Max, Soma, STD, Var,...)", @@ -108,7 +109,8 @@ "vertical_headers": "Rotate column headers vertically", "timeseries_analysis": "Analyze time series data using different \"statsmodels.tsa\" functions", "hide_header_editor": "If checked, hides the header editor when editing cells in the grid", - "arcticdb": "Load data from a specific library & symbol in ArcticDB" + "arcticdb": "Load data from a specific library & symbol in ArcticDB", + "jump_to_column": "For wide dataframes, you can shrink the columns displayed down to what is locked and a column selected from a dropdown." }, "column_menu": { "Asc": "Asc", @@ -936,7 +938,8 @@ "Invert Filter": "Inverter Filtro", "Move Filters To Custom": "Move Filters To Custom", "Hide Filtered Rows": "Hide Filtered Rows", - "Highlight Filtered Rows": "Highlight Filtered Rows" + "Highlight Filtered Rows": "Highlight Filtered Rows", + "ArcticDB": "ArcticDB" }, "side": { "Close": "Fechar", diff --git a/tests/arcticdb/test_instance.py b/tests/arcticdb/test_instance.py new file mode 100644 index 00000000..336971ea --- /dev/null +++ b/tests/arcticdb/test_instance.py @@ -0,0 +1,12 @@ +import pytest + + +@pytest.mark.unit +def test_instance_creation_w_bad_symbol(unittest, arcticdb_path, arcticdb): + pytest.importorskip("arcticdb") + + from dtale.global_state import DtaleArcticDB + + with pytest.raises(ValueError): + db = DtaleArcticDB(arcticdb_path) + db.build_instance("dtale|bad_symbol") diff --git a/tests/arcticdb/test_views.py b/tests/arcticdb/test_views.py index e43de580..c057cf33 100644 --- a/tests/arcticdb/test_views.py +++ b/tests/arcticdb/test_views.py @@ -4,6 +4,7 @@ import dtale.global_state as global_state from dtale.app import build_app +from tests import ExitStack URL = "http://localhost:40000" @@ -186,6 +187,41 @@ def test_loading_data_w_filters(unittest, arcticdb_path, arcticdb): unittest.assertEqual(response_data["final_query"], "`index` == '20000101'") +@pytest.mark.unit +def test_loading_data_w_columns(unittest, arcticdb_path, arcticdb): + pytest.importorskip("arcticdb") + from dtale.views import startup + + global_state.use_arcticdb_store(uri=arcticdb_path, library="dtale") + startup(data="df1") + + with app.test_client() as c: + c.get( + "/dtale/save-column-filter/dtale%257Cdf1", + query_string=dict( + col="str_val", cfg=json.dumps({"type": "string", "value": ["b"]}) + ), + ) + c.post( + "/dtale/update-visibility/dtale%257Cdf1", + data=json.dumps(dict(toggle="int_val")), + content_type="application/json", + ) + response = c.get( + "/dtale/data/dtale%257Cdf1", query_string=dict(ids=json.dumps(["0"])) + ) + response_data = response.get_json() + expected_results = { + "0": { + "dtale_index": 0, + "float_val": 2.2, + "index": "2000-01-02", + "str_val": "b", + } + } + unittest.assertEqual(response_data["results"], expected_results) + + @pytest.mark.unit def test_describe(unittest, arcticdb_path, arcticdb): pytest.importorskip("arcticdb") @@ -282,7 +318,13 @@ def test_get_arcticdb_libraries(unittest, arcticdb_path, arcticdb): response = c.get("/dtale/arcticdb/libraries") response_data = response.get_json() unittest.assertEqual( - response_data, {"libraries": ["dtale"], "library": "dtale", "success": True} + response_data, + { + "libraries": ["dtale"], + "library": "dtale", + "async": False, + "success": True, + }, ) with mock.patch("dtale.global_state.store.load_libraries") as load_libs_mock: @@ -294,6 +336,31 @@ def test_get_arcticdb_libraries(unittest, arcticdb_path, arcticdb): unittest.assertEqual(response_data["libraries"], ["dtale"]) load_libs_mock.assert_called_once() + with mock.patch( + "dtale.global_state.store._libraries", + ["lib{}".format(v) for v in range(501)], + ): + with app.test_client() as c: + response = c.get("/dtale/arcticdb/libraries") + response_data = response.get_json() + unittest.assertEqual( + response_data["libraries"], ["lib0", "lib1", "lib2", "lib3", "lib4"] + ) + assert response_data["async"] + + +@pytest.mark.unit +def test_get_arcticdb_async_libraries(unittest, arcticdb_path, arcticdb): + pytest.importorskip("arcticdb") + global_state.use_arcticdb_store(uri=arcticdb_path, library="dtale") + + with app.test_client() as c: + response = c.get( + "/dtale/arcticdb/async-libraries", query_string=dict(input="d") + ) + response_data = response.get_json() + unittest.assertEqual(response_data, [{"label": "dtale", "value": "dtale"}]) + @pytest.mark.unit def test_get_arcticdb_symbols(unittest, arcticdb_path, arcticdb): @@ -318,6 +385,42 @@ def test_get_arcticdb_symbols(unittest, arcticdb_path, arcticdb): ) load_symbols_mock.assert_called_once_with("dtale") + with ExitStack() as stack: + stack.enter_context( + mock.patch( + "dtale.global_state.store.load_symbols", + mock.Mock(return_value=None), + ) + ) + stack.enter_context( + mock.patch( + "dtale.global_state.store._symbols", + dict(large_lib=["security{}".format(v) for v in range(501)]), + ) + ) + + with app.test_client() as c: + response = c.get("/dtale/arcticdb/large_lib/symbols") + response_data = response.get_json() + unittest.assertEqual( + response_data["symbols"], + ["security0", "security1", "security2", "security3", "security4"], + ) + assert response_data["async"] + + +@pytest.mark.unit +def test_get_arcticdb_async_symbols(unittest, arcticdb_path, arcticdb): + pytest.importorskip("arcticdb") + global_state.use_arcticdb_store(uri=arcticdb_path, library="dtale") + + with app.test_client() as c: + response = c.get( + "/dtale/arcticdb/dtale/async-symbols", query_string=dict(input="df") + ) + response_data = response.get_json() + unittest.assertEqual(response_data, [{"label": "df1", "value": "df1"}]) + @pytest.mark.unit def test_load_arcticdb_description(unittest, arcticdb_path, arcticdb): @@ -350,3 +453,35 @@ def test_load_arcticdb_symbol(unittest, arcticdb_path, arcticdb): assert response_data["data_id"] == "dtale|df1" validate_data_load("dtale|df1", unittest, c) + + +@pytest.mark.unit +def test_view(arcticdb_path, arcticdb): + pytest.importorskip("arcticdb") + global_state.use_arcticdb_store(uri=arcticdb_path, library="dtale") + + import dtale.views as views + + views.startup(data="df1") + + with app.test_client() as c: + response = c.get("/dtale/main/dtale%257Cdf1") + html_content = str(response.data) + assert '' in html_content + assert ( + ''.format(arcticdb_path) + in html_content + ) + assert '' in html_content + + +@pytest.mark.unit +def test_load_url_w_bad_symbol(arcticdb_path, arcticdb): + pytest.importorskip("arcticdb") + global_state.use_arcticdb_store(uri=arcticdb_path, library="dtale") + + with app.test_client() as c: + response = c.get("/dtale/main/dtale%257Cdf2") + assert "http://localhost:{}/dtale/popup/arcticdb".format(c.port).endswith( + response.location + ) diff --git a/tests/dtale/test_views.py b/tests/dtale/test_views.py index e00823d0..ba67c184 100644 --- a/tests/dtale/test_views.py +++ b/tests/dtale/test_views.py @@ -316,6 +316,16 @@ def test_startup(unittest): ["object", "category"], ) + many_cols = pd.DataFrame({"sec{}".format(v): [1] for v in range(500)}) + instance = views.startup( + URL, + data=many_cols, + ) + unittest.assertEqual( + len([v for v in global_state.get_dtypes(instance._data_id) if v["visible"]]), + 100, + ) + if PY3 and check_pandas_version("0.25.0"): s_int = pd.Series([1, 2, 3, 4, 5], index=list("abcde"), dtype=pd.Int64Dtype()) s2_int = s_int.reindex(["a", "b", "c", "f", "u"]) @@ -376,6 +386,16 @@ def test_formatting_complex_data(unittest): df, _ = format_data(pd.DataFrame({"foo": data})) unittest.assertEqual(list(df["foo"].values), ["1", "2", "[3]"]) + index_vals = [ + pd.Timestamp("20230101"), + pd.Timestamp("20230102"), + pd.Timestamp("20230103"), + ] + base_df = pd.DataFrame([1, 2, 3], index=index_vals) + df, index = format_data(base_df) + unittest.assertEqual(index, ["index"]) + assert len(df.columns) > len(base_df.columns) + @pytest.mark.unit def test_in_ipython_frontend(builtin_pkg):