Skip to content

Commit

Permalink
Close #20: Add pagination to select
Browse files Browse the repository at this point in the history
  • Loading branch information
@Tinitto
Tinitto committed Dec 17, 2022
1 parent 4221210 commit da988ec
Show file tree
Hide file tree
Showing 14 changed files with 530 additions and 106 deletions.
10 changes: 10 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/).

## [Unreleased]

## [0.4.0] - 2022-12-17

### Added

- Added pagination

### Changed

- Changed redis index to use sorted sets instead of ordinary sets

## [0.3.0] - 2022-12-15

### Added
Expand Down
143 changes: 93 additions & 50 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ from pydantic_redis import RedisConfig, Model, Store

class Author(Model):
"""
An Author model, just like a pydantic model with appropriate type annotations
An Author model, just like a pydantic model with appropriate
type annotations
NOTE: The `_primary_key_field` is mandatory
"""
_primary_key_field: str = 'name'
Expand All @@ -51,18 +52,24 @@ class Book(Model):
A Book model.
Models can have the following field types
- The usual i.e. float, int, dict, list, date, str, dict, Optional etc as long as they are serializable by orjson
- The usual i.e. float, int, dict, list, date, str, dict, Optional etc
as long as they are serializable by orjson
- Nested models e.g. `author: Author` or `author: Optional[Author]`
- List of nested models e.g. `authors: List[Author]` or `authors: Optional[List[Author]]`
- Tuples including nested models e.g. `access_log: Tuple[Author, date]` or `access_log: Optional[Tuple[Author, date]]]`
- List of nested models e.g. `authors: List[Author]`
or `authors: Optional[List[Author]]`
- Tuples including nested models e.g. `access_log: Tuple[Author, date]`
or `access_log: Optional[Tuple[Author, date]]]`
NOTE: 1. Any nested model whether plain or in a list or tuple will automatically inserted into the redis store
when the parent model is inserted. e.g. a Book with an author field, when inserted, will also insert
the author. The author can then be queried directly if that's something one wishes to do.
NOTE: 1. Any nested model whether plain or in a list or tuple will automatically
inserted into the redis store when the parent model is inserted.
e.g. a Book with an author field, when inserted, will also insert
the author. The author can then be queried directly if that's something
one wishes to do.
2. When a parent model is inserted with a nested model instance that already exists, the older nested model
instance is overwritten. This is one way of updating nested models. All parent models that contain that nested
model instance will see the change.
2. When a parent model is inserted with a nested model instance that
already exists, the older nested model instance is overwritten.
This is one way of updating nested models.
All parent models that contain that nested model instance will see the change.
"""
_primary_key_field: str = 'title'
title: str
Expand All @@ -81,8 +88,10 @@ class Library(Model):
About Nested Model Performance
---
To minimize the performance penalty for nesting models, we use REDIS EVALSHA to eagerly load the nested models
before the response is returned to the client. This ensures that only ONE network call is made every time.
To minimize the performance penalty for nesting models,
we use REDIS EVALSHA to eagerly load the nested models
before the response is returned to the client.
This ensures that only ONE network call is made every time.
"""
_primary_key_field: str = 'name'
name: str
Expand Down Expand Up @@ -180,15 +189,26 @@ print(all_books)
# Prints [Book(title="Oliver Twist", author="Charles Dickens", published_on=date(year=1215, month=4, day=4),
# in_stock=False), Book(...]

# or paginate i.e. skip some books and return only upto a given number
paginated_books = Book.select(skip=2, limit=2)
print(paginated_books)

# Get some, with all fields shown. Data returned is a list of models instances.
some_books = Book.select(ids=["Oliver Twist", "Jane Eyre"])
print(some_books)

# Note: Pagination does not work when ids are provided i.e.
assert some_books == Book.select(ids=["Oliver Twist", "Jane Eyre"], skip=100, limit=10)

# Get all, with only a few fields shown. Data returned is a list of dictionaries.
books_with_few_fields = Book.select(columns=["author", "in_stock"])
print(books_with_few_fields)
# Prints [{"author": "'Charles Dickens", "in_stock": "True"},...]

# or paginate i.e. skip some books and return only upto a given number
paginated_books_with_few_fields = Book.select(columns=["author", "in_stock"], skip=2, limit=2)
print(paginated_books_with_few_fields)

# Get some, with only some fields shown. Data returned is a list of dictionaries.
some_books_with_few_fields = Book.select(ids=["Oliver Twist", "Jane Eyre"], columns=["author", "in_stock"])
print(some_books_with_few_fields)
Expand Down Expand Up @@ -217,13 +237,15 @@ from datetime import date
from typing import Tuple, List, Optional
from pydantic_redis.asyncio import RedisConfig, Model, Store

# The features are exactly the same as the synchronous version, except for the ability
# to return coroutines when `insert`, `update`, `select` or `delete` are called.
# The features are exactly the same as the synchronous version,
# except for the ability to return coroutines when `insert`,
# `update`, `select` or `delete` are called.


class Author(Model):
"""
An Author model, just like a pydantic model with appropriate type annotations
An Author model, just like a pydantic model with appropriate
type annotations
NOTE: The `_primary_key_field` is mandatory
"""
_primary_key_field: str = 'name'
Expand All @@ -236,18 +258,24 @@ class Book(Model):
A Book model.
Models can have the following field types
- The usual i.e. float, int, dict, list, date, str, dict, Optional etc as long as they are serializable by orjson
- The usual i.e. float, int, dict, list, date, str, dict, Optional etc
as long as they are serializable by orjson
- Nested models e.g. `author: Author` or `author: Optional[Author]`
- List of nested models e.g. `authors: List[Author]` or `authors: Optional[List[Author]]`
- Tuples including nested models e.g. `access_log: Tuple[Author, date]` or `access_log: Optional[Tuple[Author, date]]]`
- List of nested models e.g. `authors: List[Author]`
or `authors: Optional[List[Author]]`
- Tuples including nested models e.g. `access_log: Tuple[Author, date]`
or `access_log: Optional[Tuple[Author, date]]]`
NOTE: 1. Any nested model whether plain or in a list or tuple will automatically inserted into the redis store
when the parent model is inserted. e.g. a Book with an author field, when inserted, will also insert
the author. The author can then be queried directly if that's something one wishes to do.
NOTE: 1. Any nested model whether plain or in a list or tuple will automatically
inserted into the redis store when the parent model is inserted.
e.g. a Book with an author field, when inserted, will also insert
the author. The author can then be queried directly if that's something
one wishes to do.
2. When a parent model is inserted with a nested model instance that already exists, the older nested model
instance is overwritten. This is one way of updating nested models. All parent models that contain that nested
model instance will see the change.
2. When a parent model is inserted with a nested model instance that
already exists, the older nested model instance is overwritten.
This is one way of updating nested models.
All parent models that contain that nested model instance will see the change.
"""
_primary_key_field: str = 'title'
title: str
Expand All @@ -266,8 +294,10 @@ class Library(Model):
About Nested Model Performance
---
To minimize the performance penalty for nesting models, we use REDIS EVALSHA to eagerly load the nested models
before the response is returned to the client. This ensures that only ONE network call is made every time.
To minimize the performance penalty for nesting models,
we use REDIS EVALSHA to eagerly load the nested models
before the response is returned to the client.
This ensures that only ONE network call is made every time.
"""
_primary_key_field: str = 'name'
name: str
Expand Down Expand Up @@ -367,15 +397,26 @@ async def run_async():
# Prints [Book(title="Oliver Twist", author="Charles Dickens", published_on=date(year=1215, month=4, day=4),
# in_stock=False), Book(...]

# or paginate i.e. skip some books and return only upto a given number
paginated_books = await Book.select(skip=2, limit=2)
print(paginated_books)

# Get some, with all fields shown. Data returned is a list of models instances.
some_books = await Book.select(ids=["Oliver Twist", "Jane Eyre"])
print(some_books)

# Note: Pagination does not work when ids are provided i.e.
assert some_books == await Book.select(ids=["Oliver Twist", "Jane Eyre"], skip=100, limit=10)

# Get all, with only a few fields shown. Data returned is a list of dictionaries.
books_with_few_fields = await Book.select(columns=["author", "in_stock"])
print(books_with_few_fields)
# Prints [{"author": "'Charles Dickens", "in_stock": "True"},...]

# or paginate i.e. skip some books and return only upto a given number
paginated_books_with_few_fields = await Book.select(columns=["author", "in_stock"], skip=2, limit=2)
print(paginated_books_with_few_fields)

# Get some, with only some fields shown. Data returned is a list of dictionaries.
some_books_with_few_fields = await Book.select(ids=["Oliver Twist", "Jane Eyre"], columns=["author", "in_stock"])
print(some_books_with_few_fields)
Expand Down Expand Up @@ -440,30 +481,32 @@ asyncio.run(run_async())
On an average PC ~16GB RAM, i7 Core

```
------------------------------------------------- benchmark: 20 tests -------------------------------------------------
Name (time in us) Mean Min Max
-----------------------------------------------------------------------------------------------------------------------
benchmark_select_columns_for_one_id[redis_store-book1] 143.5316 (1.08) 117.4340 (1.0) 347.5900 (1.0)
benchmark_select_columns_for_one_id[redis_store-book3] 151.6032 (1.14) 117.6690 (1.00) 405.4620 (1.17)
benchmark_select_columns_for_one_id[redis_store-book0] 133.0856 (1.0) 117.8720 (1.00) 403.9400 (1.16)
benchmark_select_columns_for_one_id[redis_store-book2] 156.8152 (1.18) 118.7220 (1.01) 569.9800 (1.64)
benchmark_select_columns_for_some_items[redis_store] 138.0488 (1.04) 120.1550 (1.02) 350.7040 (1.01)
benchmark_delete[redis_store-Wuthering Heights] 199.9205 (1.50) 127.6990 (1.09) 1,092.2190 (3.14)
benchmark_bulk_delete[redis_store] 178.4756 (1.34) 143.7480 (1.22) 647.6660 (1.86)
benchmark_select_all_for_one_id[redis_store-book1] 245.7787 (1.85) 195.2030 (1.66) 528.9250 (1.52)
benchmark_select_all_for_one_id[redis_store-book0] 239.1152 (1.80) 199.4360 (1.70) 767.2540 (2.21)
benchmark_select_all_for_one_id[redis_store-book3] 243.8724 (1.83) 200.8060 (1.71) 535.3640 (1.54)
benchmark_select_all_for_one_id[redis_store-book2] 256.1625 (1.92) 202.4630 (1.72) 701.3000 (2.02)
benchmark_update[redis_store-Wuthering Heights-data0] 329.1363 (2.47) 266.9700 (2.27) 742.1360 (2.14)
benchmark_select_some_items[redis_store] 301.0471 (2.26) 268.9410 (2.29) 551.1060 (1.59)
benchmark_select_columns[redis_store] 313.4356 (2.36) 281.4460 (2.40) 578.7730 (1.67)
benchmark_single_insert[redis_store-book2] 348.5624 (2.62) 297.3610 (2.53) 580.8780 (1.67)
benchmark_single_insert[redis_store-book1] 342.1879 (2.57) 297.5410 (2.53) 650.5420 (1.87)
benchmark_single_insert[redis_store-book0] 366.4513 (2.75) 310.1640 (2.64) 660.5380 (1.90)
benchmark_single_insert[redis_store-book3] 377.6208 (2.84) 327.5290 (2.79) 643.4090 (1.85)
benchmark_select_default[redis_store] 486.6931 (3.66) 428.8810 (3.65) 1,181.9620 (3.40)
benchmark_bulk_insert[redis_store] 897.7862 (6.75) 848.7410 (7.23) 1,188.5160 (3.42)
-----------------------------------------------------------------------------------------------------------------------
-------------------------------------------------- benchmark: 22 tests --------------------------------------------------
Name (time in us) Mean Min Max
-------------------------------------------------------------------------------------------------------------------------
benchmark_select_columns_for_one_id[redis_store-book2] 124.2687 (1.00) 115.4530 (1.0) 331.8030 (1.26)
benchmark_select_columns_for_one_id[redis_store-book0] 123.7213 (1.0) 115.6680 (1.00) 305.7170 (1.16)
benchmark_select_columns_for_one_id[redis_store-book3] 124.4495 (1.01) 115.9580 (1.00) 263.4370 (1.0)
benchmark_select_columns_for_one_id[redis_store-book1] 124.8431 (1.01) 117.4770 (1.02) 310.3140 (1.18)
benchmark_select_columns_for_some_items[redis_store] 128.0657 (1.04) 118.6380 (1.03) 330.2680 (1.25)
benchmark_delete[redis_store-Wuthering Heights] 131.8713 (1.07) 125.9920 (1.09) 328.9660 (1.25)
benchmark_bulk_delete[redis_store] 148.6963 (1.20) 142.3190 (1.23) 347.4750 (1.32)
benchmark_select_all_for_one_id[redis_store-book3] 211.6941 (1.71) 195.6520 (1.69) 422.8840 (1.61)
benchmark_select_all_for_one_id[redis_store-book2] 212.3612 (1.72) 195.9020 (1.70) 447.4910 (1.70)
benchmark_select_all_for_one_id[redis_store-book1] 212.9524 (1.72) 197.7530 (1.71) 423.3030 (1.61)
benchmark_select_all_for_one_id[redis_store-book0] 214.9924 (1.74) 198.8280 (1.72) 402.6310 (1.53)
benchmark_select_columns_paginated[redis_store] 227.9248 (1.84) 211.0610 (1.83) 425.8390 (1.62)
benchmark_select_some_items[redis_store] 297.5700 (2.41) 271.1510 (2.35) 572.1220 (2.17)
benchmark_select_default_paginated[redis_store] 301.7495 (2.44) 282.6500 (2.45) 490.3450 (1.86)
benchmark_select_columns[redis_store] 316.2119 (2.56) 290.6110 (2.52) 578.0310 (2.19)
benchmark_update[redis_store-Wuthering Heights-data0] 346.5816 (2.80) 304.5420 (2.64) 618.0250 (2.35)
benchmark_single_insert[redis_store-book2] 378.0613 (3.06) 337.8070 (2.93) 616.4930 (2.34)
benchmark_single_insert[redis_store-book0] 396.6513 (3.21) 347.1000 (3.01) 696.1350 (2.64)
benchmark_single_insert[redis_store-book3] 395.9082 (3.20) 361.0980 (3.13) 623.8630 (2.37)
benchmark_single_insert[redis_store-book1] 401.1377 (3.24) 363.5890 (3.15) 610.4400 (2.32)
benchmark_select_default[redis_store] 498.4673 (4.03) 428.1350 (3.71) 769.7640 (2.92)
benchmark_bulk_insert[redis_store] 1,025.0436 (8.29) 962.2230 (8.33) 1,200.3840 (4.56)
-------------------------------------------------------------------------------------------------------------------------
```

## Contributions
Expand Down
2 changes: 1 addition & 1 deletion pydantic_redis/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

__all__ = [Store, RedisConfig, Model, asyncio]

__version__ = "0.3.0"
__version__ = "0.4.0"
35 changes: 18 additions & 17 deletions pydantic_redis/asyncio/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
)

from .store import Store
from ..shared.model.delete_utils import delete_on_pipeline


class Model(AbstractModel):
Expand Down Expand Up @@ -92,41 +93,41 @@ async def delete(cls, ids: Union[Any, List[Any]]):
store = cls.get_store()

async with store.redis_store.pipeline() as pipeline:
primary_keys = []

if isinstance(ids, list):
primary_keys = ids
elif ids is not None:
primary_keys = [ids]

names = [
get_primary_key(model=cls, primary_key_value=primary_key_value)
for primary_key_value in primary_keys
]
pipeline.delete(*names)
# remove the primary keys from the index
table_index_key = get_table_index_key(model=cls)
pipeline.srem(table_index_key, *names)
delete_on_pipeline(model=cls, pipeline=pipeline, ids=ids)
return await pipeline.execute()

@classmethod
async def select(
cls,
columns: Optional[List[str]] = None,
ids: Optional[List[Any]] = None,
skip: int = 0,
limit: Optional[int] = None,
**kwargs,
):
"""
Selects given rows or sets of rows in the table
However, if `limit` is set, the number of items
returned will be less or equal to `limit`.
`skip` defaults to 0. It is the number of items to skip.
`skip` is only relevant when limit is specified.
`skip` and `limit` are irrelevant when `ids` are provided.
"""
if columns is None and ids is None:
response = await select_all_fields_all_ids(model=cls)
response = await select_all_fields_all_ids(
model=cls, skip=skip, limit=limit
)

elif columns is None and isinstance(ids, list):
response = await select_all_fields_some_ids(model=cls, ids=ids)

elif isinstance(columns, list) and ids is None:
response = await select_some_fields_all_ids(model=cls, fields=columns)
response = await select_some_fields_all_ids(
model=cls, fields=columns, skip=skip, limit=limit
)

elif isinstance(columns, list) and isinstance(ids, list):
response = await select_some_fields_some_ids(
Expand Down
Loading

0 comments on commit da988ec

Please sign in to comment.