From 290831c5ce16cab9b7fc3f34682b5b7c567bf68b Mon Sep 17 00:00:00 2001 From: Brunno Vanelli Date: Tue, 19 Mar 2024 20:31:23 +0100 Subject: [PATCH] feat: Add basic documentation and prepare merge of the first big feature. --- README.md | 89 ++++++++++++++++++++++++++++++++++++++- actual/__init__.py | 48 +++++++++++++++++---- actual/api.py | 15 ++++++- actual/exceptions.py | 26 ++++++++++++ actual/protobuf_models.py | 61 +++++++++++++++------------ docker/docker-compose.yml | 2 +- setup.py | 2 +- tests/test_models.py | 15 +++++++ tests/test_protobuf.py | 47 +++++++++++++++++++++ 9 files changed, 267 insertions(+), 38 deletions(-) create mode 100644 tests/test_protobuf.py diff --git a/README.md b/README.md index 099ad77..239dce8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,89 @@ # actualpy -Python API implementation for Actual server - reference https://actualbudget.org/ + +Python API implementation for Actual server. + +[Actual Budget](https://actualbudget.org/) is a super fast and privacy-focused app for managing your finances. + +> **WARNING:** The [Javascript API](https://actualbudget.org/docs/api/) to interact with Actual server already exists, +> and is battle-tested as it is the core of the Actual frontend libraries. If you intend to use a reliable and well +> tested library, that is the way to go. + +# Installation + +Install it via Pip using the repository url: + +```bash +pip install https://github.com/bvanelli/actualpy +``` + +# Basic usage + +The most common usage would be downloading a budget to more easily build queries. This would you could handle the +Actual database using SQLAlchemy instead of having to retrieve the data via the export. The following script will print +every single transaction registered on the Actual budget file: + +```python +from actual import Actual + +with Actual( + base_url="http://localhost:5006", # Url of the Actual Server + password="", # Password for authentication + file="", # Set the file to work with. Can be either the file id or file name, if name is unique + data_dir="" # Optional: Directory to store downloaded files. Will use a temporary if not provided +) as actual: + transactions = actual.get_transactions() + for t in transactions: + account_name = t.account.name if t.account else None + category = t.category.name if t.category else None + print(t.date, account_name, t.notes, t.amount, category) +``` + +# Experimental features + +> **WARNING:** Experimental features do not have all the testing necessary to ensure correctness in comparison to the +> files generated by the Javascript API. This means that this operations could in theory corrupt the data. Make sure +> you have backups of your data before trying any of those operations. + +## Bootstraping a new server and uploading a first file + +The following script would generate a new empty budget on the Actual server, even if the server was not bootstrapped +with an initial password. + +```python +from actual import Actual + +with Actual(base_url="http://localhost:5006", password="mypass", bootstrap=True) as actual: + actual.create_budget("My budget") + actual.upload_budget() +``` + +You will then have a freshly created new budget to use: + +![created-budget](./docs/static/new-budget.png) + +# Adding new transactions + +After you created your first budget (or when updating an existing budget), you can add new transactions by adding them +using the `actual.add()` method. You cannot use the SQLAlchemy session directly because that adds the entries to your +local database, but will not sync the results back to the server (that is only possible when reuploading the file). + +The method will make sure the local database is updated, but will also send a SYNC request with the added data so that +it will be immediately available on the frontend: + +```python +import decimal +import datetime +from actual import Actual +from actual.database import Transactions + +with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: + act = actual.get_accounts()[0] # get first account + t = Transactions.new(act.id, decimal.Decimal(10.5), datetime.date.today(), notes="My first transaction") + actual.add(t) +``` + +![added-transaction](./docs/static/added-transaction.png) + +# Contributing + +The goal is to have more features implemented and tested on the Actual API. diff --git a/actual/__init__.py b/actual/__init__.py index 841653f..e0360c1 100644 --- a/actual/__init__.py +++ b/actual/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import datetime import io import json import pathlib @@ -19,13 +20,14 @@ from actual.database import ( Accounts, Categories, + MessagesClock, Payees, Transactions, get_attribute_by_table_name, get_class_by_table_name, ) from actual.exceptions import InvalidZipFile, UnknownFileId -from actual.protobuf_models import Message, SyncRequest +from actual.protobuf_models import HULC_Client, Message, SyncRequest if TYPE_CHECKING: from actual.database import BaseModel @@ -39,6 +41,7 @@ def __init__( password: str = None, file: str = None, data_dir: Union[str, pathlib.Path] = None, + bootstrap: bool = False, ): """ Implements the Python API for the Actual Server in order to be able to read and modify information on Actual @@ -53,12 +56,14 @@ def __init__( :param file: the name or id of the file to be set :param data_dir: where to store the downloaded files from the server. If not specified, a temporary folder will be created instead. + :param bootstrap: if the server is not bootstrapped, bootstrap it with the password. """ - super().__init__(base_url, token, password) + super().__init__(base_url, token, password, bootstrap) self._file: RemoteFileListDTO | None = None self._data_dir = pathlib.Path(data_dir) if data_dir else None self._session_maker = None self._session: sqlalchemy.orm.Session | None = None + self._client: HULC_Client | None = None # set the correct file if file: self.set_file(file) @@ -105,7 +110,9 @@ def run_migrations(self, migration_files: list[str]): .data_file_index() method. This first file is the base database, and the following files are migrations. Migrations can also be .js files. In this case, we have to extract and execute queries from the standard JS.""" conn = sqlite3.connect(self._data_dir / "db.sqlite") - for file in migration_files[1:]: + for file in migration_files: + if not file.startswith("migrations"): + continue # in case db.sqlite file gets passed file_id = file.split("_")[0].split("/")[1] if conn.execute(f"SELECT id FROM __migrations__ WHERE id = '{file_id}';").fetchall(): continue # skip migration as it was already ran @@ -146,6 +153,11 @@ def create_budget(self, budget_name: str): self._file = RemoteFileListDTO(name=budget_name, fileId=file_id, groupId=None, deleted=0, encryptKeyId=None) # create engine for downloaded database and run migrations self.run_migrations(migration_files[1:]) + # generate a session + engine = sqlalchemy.create_engine(f"sqlite:///{self._data_dir}/db.sqlite") + self._session_maker = sqlalchemy.orm.sessionmaker(engine) + # create a clock + self.load_clock() def upload_budget(self): """Uploads the current file to the Actual server.""" @@ -158,6 +170,10 @@ def upload_budget(self): binary_data.seek(0) return self.upload_user_file(binary_data.getvalue(), self._file.file_id, self._file.name) + def reupload_budget(self): + self.reset_user_file(self._file.file_id) + self.upload_budget() + def apply_changes(self, messages: list[Message]): """Applies a list of sync changes, based on what the sync method returned on the remote.""" if not self._session_maker: @@ -197,17 +213,33 @@ def download_budget(self): self._session_maker = sqlalchemy.orm.sessionmaker(engine) # actual js always calls validation self.validate() + # load the client id + self.load_clock() # after downloading the budget, some pending transactions still need to be retrieved using sync request = SyncRequest({"messages": [], "fileId": self._file.file_id, "groupId": self._file.group_id}) - request.set_null_timestamp() # using 0 timestamp to retrieve all changes + request.set_null_timestamp(client_id=self._client.client_id) # using 0 timestamp to retrieve all changes changes = self.sync(request) self.apply_changes(changes.get_messages()) + if changes.messages: + self._client = HULC_Client.from_timestamp(changes.messages[-1].timestamp) - def load_clock(self): + def load_clock(self) -> MessagesClock: """See implementation at: https://github.com/actualbudget/actual/blob/5bcfc71be67c6e7b7c8b444e4c4f60da9ea9fdaa/packages/loot-core/src/server/db/index.ts#L81-L98 """ - pass + with self.with_session() as session: + clock = session.query(MessagesClock).first() + if not clock: + clock_message = { + "timestamp": HULC_Client().timestamp(now=datetime.datetime(1970, 1, 1, 0, 0, 0, 0)), + "merkle": {}, + } + clock = MessagesClock(id=0, clock=json.dumps(clock_message)) + session.add(clock) + session.commit() + # add clock id to client id + self._client = HULC_Client.from_timestamp(json.loads(clock.clock)["timestamp"]) + return clock def get_transactions(self) -> List[Transactions]: with self._session_maker() as s: @@ -243,8 +275,8 @@ def add(self, model: BaseModel): s.add(model) # generate a sync request and sync it to the server req = SyncRequest({"fileId": self._file.file_id, "groupId": self._file.group_id}) - req.set_timestamp() - req.set_messages(model.convert()) + req.set_null_timestamp(client_id=self._client.client_id) + req.set_messages(model.convert(), self._client) self.sync(req) s.commit() if not self._session: diff --git a/actual/api.py b/actual/api.py index cebcb4a..fe7f174 100644 --- a/actual/api.py +++ b/actual/api.py @@ -15,6 +15,7 @@ class Endpoints(enum.Enum): INFO = "info" ACCOUNT_VALIDATE = "account/validate" NEEDS_BOOTSTRAP = "account/needs-bootstrap" + BOOTSTRAP = "account/bootstrap" SYNC = "sync/sync" LIST_USER_FILES = "sync/list-user-files" GET_USER_FILE_INFO = "sync/get-user-file-info" @@ -122,14 +123,19 @@ def __init__( base_url: str = "http://localhost:5006", token: str = None, password: str = None, + bootstrap: bool = False, ): self.api_url = base_url self._token = token if token is None and password is None: raise ValueError("Either provide a valid token or a password.") # already try to login if password was provided - if password: + if password and bootstrap and not self.needs_bootstrap().data.bootstrapped: + self.bootstrap(password) + elif password: self.login(password) + # finally call validate + self.validate() def login(self, password: str) -> LoginDTO: """Logs in on the Actual server using the password provided. Raises `AuthorizationError` if it fails to @@ -173,6 +179,13 @@ def needs_bootstrap(self) -> BootstrapInfoDTO: response.raise_for_status() return BootstrapInfoDTO.parse_obj(response.json()) + def bootstrap(self, password: str) -> LoginDTO: + response = requests.post(f"{self.api_url}/{Endpoints.BOOTSTRAP}", json={"password": password}) + response.raise_for_status() + login_response = LoginDTO.parse_obj(response.json()) + self._token = login_response.data.token + return login_response + def data_file_index(self) -> List[str]: """Gets all the migration file references for the actual server.""" response = requests.get(f"{self.api_url}/{Endpoints.DATA_FILE_INDEX}") diff --git a/actual/exceptions.py b/actual/exceptions.py index d268bb1..dec5aa8 100644 --- a/actual/exceptions.py +++ b/actual/exceptions.py @@ -1,3 +1,25 @@ +import requests + + +def get_exception_from_response(response: requests.Response): + text = response.content.decode() + if text == "internal-error" or response.status_code == 500: + return ActualError(text) + # taken from + # https://github.com/actualbudget/actual-server/blob/6e9eddeb561b0d9f2bbb6301c3e2c30b4effc522/src/app-sync.js#L107 + elif text == "file-has-new-key": + return ActualError(f"{text}: The data is encrypted with a different key") + elif text == "file-has-reset": + return InvalidFile( + f"{text}: The changes being synced are part of an old group, which means the file has been reset. " + f"User needs to re-download." + ) + elif text in ("file-not-found", "file-needs-upload"): + raise UnknownFileId(text) + elif text == "file-old-version": + raise InvalidFile(f"{text}: SYNC_FORMAT_VERSION was generated with an old format") + + class ActualError(Exception): pass @@ -12,3 +34,7 @@ class UnknownFileId(ActualError): class InvalidZipFile(ActualError): pass + + +class InvalidFile(ActualError): + pass diff --git a/actual/protobuf_models.py b/actual/protobuf_models.py index 69f0c7d..bcae718 100644 --- a/actual/protobuf_models.py +++ b/actual/protobuf_models.py @@ -16,29 +16,38 @@ """ -def timestamp(client_id: str = None, now: datetime.datetime = None) -> str: - """Actual uses Hybrid Unique Logical Clock (HULC) timestamp generator. +class HULC_Client: + def __init__(self, client_id: str = None, initial_count: int = 0): + self.client_id = client_id + self.initial_count = initial_count - Timestamps serialize into a 46-character collatable string - * example: 2015-04-24T22:23:42.123Z-1000-0123456789ABCDEF - * example: 2015-04-24T22:23:42.123Z-1000-A219E7A71CC18912 + @classmethod + def from_timestamp(cls, ts: str) -> HULC_Client: + segments = ts.split("-") + return cls(segments[-1], int(segments[-2])) - See https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts - for reference. - """ - if not now: - now = datetime.datetime.utcnow() - if not client_id: - client_id = get_client_id() - return f"{now.isoformat(timespec='milliseconds')}Z-0000-{client_id}" + def timestamp(self, now: datetime.datetime = None) -> str: + """Actual uses Hybrid Unique Logical Clock (HULC) timestamp generator. + Timestamps serialize into a 46-character collatable string + * example: 2015-04-24T22:23:42.123Z-1000-0123456789ABCDEF + * example: 2015-04-24T22:23:42.123Z-1000-A219E7A71CC18912 -def get_client_id(): - """Creates a client id for the HULC request. Copied implementation from: + See https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts + for reference. + """ + if not now: + now = datetime.datetime.utcnow() + count = str(self.initial_count).zfill(4) + self.initial_count += 1 + return f"{now.isoformat(timespec='milliseconds')}Z-{count}-{self.client_id}" + + def get_client_id(self): + """Creates a client id for the HULC request. Copied implementation from: - https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts#L80 - """ - return str(uuid.uuid4()).replace("-", "")[-16:] + https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts#L80 + """ + return self.client_id if self.client_id is not None else str(uuid.uuid4()).replace("-", "")[-16:] class EncryptedData(proto.Message): @@ -62,7 +71,7 @@ def get_value(self) -> str | int | float | None: if datatype == "S": return value elif datatype == "N": - return int(value) + return float(value) elif datatype == "0": return None else: @@ -71,7 +80,7 @@ def get_value(self) -> str | int | float | None: def set_value(self, value: str | int | float | None) -> str: if isinstance(value, str): datatype = "S" - elif isinstance(value, int): + elif isinstance(value, int) or isinstance(value, float): datatype = "N" elif value is None: datatype = "0" @@ -87,7 +96,7 @@ class MessageEnvelope(proto.Message): content = proto.Field(proto.BYTES, number=3) def set_timestamp(self, client_id: str = None, now: datetime.datetime = None) -> str: - self.timestamp = timestamp(client_id, now) + self.timestamp = HULC_Client(client_id).timestamp(now) return self.timestamp @@ -99,17 +108,17 @@ class SyncRequest(proto.Message): since = proto.Field(proto.STRING, number=6) def set_timestamp(self, client_id: str = None, now: datetime.datetime = None) -> str: - self.since = timestamp(client_id, now) + self.since = HULC_Client(client_id).timestamp(now) return self.since - def set_null_timestamp(self) -> str: - return self.set_timestamp(None, datetime.datetime(1970, 1, 1, 0, 0, 0, 0)) + def set_null_timestamp(self, client_id: str = None) -> str: + return self.set_timestamp(client_id, datetime.datetime(1970, 1, 1, 0, 0, 0, 0)) - def set_messages(self, messages: list[Message]): + def set_messages(self, messages: list[Message], client: HULC_Client): _messages = [] for message in messages: m = MessageEnvelope({"content": Message.serialize(message), "isEncrypted": False}) - m.set_timestamp() + m.timestamp = client.timestamp() _messages.append(m) self.messages = _messages diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 440126c..8076a54 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,7 +3,7 @@ version: '3' services: actual: container_name: actual - image: docker.io/actualbudget/actual-server:latest + image: docker.io/actualbudget/actual-server:24.3.0 ports: - '5006:5006' volumes: diff --git a/setup.py b/setup.py index 8c0dc69..e332e90 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ name="actual", version="0.0.1", packages=find_packages(), - description="Implementation of the Actual API to interact with Actual over Python..", + description="Implementation of the Actual API to interact with Actual over Python.", long_description=open("README.md").read(), long_description_content_type="text/markdown", author="Brunno Vanelli", diff --git a/tests/test_models.py b/tests/test_models.py index 6902c79..9a2874d 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,3 +1,6 @@ +import datetime +import decimal + from actual.database import ( Transactions, get_attribute_by_table_name, @@ -15,3 +18,15 @@ def test_get_attribute_by_table_name(): assert get_attribute_by_table_name("transactions", "category") == "category_id" assert get_attribute_by_table_name("transactions", "foo") is None assert get_attribute_by_table_name("foo", "bar") is None + + +def test_conversion(): + t = Transactions.new("foo", decimal.Decimal(10), datetime.date(2024, 3, 17)) + conversion = t.convert() + # conversion should all contain the same row id and same dataset + assert all(c.dataset == "transactions" for c in conversion) + assert all(c.row == conversion[0].row for c in conversion) + # check fields + assert [c for c in conversion if c.column == "acct"][0].get_value() == "foo" + assert [c for c in conversion if c.column == "amount"][0].get_value() == 1000 + assert [c for c in conversion if c.column == "date"][0].get_value() == 20240317 diff --git a/tests/test_protobuf.py b/tests/test_protobuf.py new file mode 100644 index 0000000..3ba653f --- /dev/null +++ b/tests/test_protobuf.py @@ -0,0 +1,47 @@ +import datetime + +import pytest + +from actual.protobuf_models import ( + HULC_Client, + Message, + MessageEnvelope, + SyncRequest, + SyncResponse, +) + + +def test_timestamp(): + now = datetime.datetime(2020, 10, 11, 12, 13, 14, 15 * 1000) + ts = HULC_Client("foo").timestamp(now) + assert ts == "2020-10-11T12:13:14.015Z-0000-foo" + + +def test_message_envelope(): + me = MessageEnvelope() + me.set_timestamp() + assert isinstance(MessageEnvelope.serialize(me), bytes) + + +def test_sync_request(): + m = Message({"dataset": "foo", "row": "bar", "column": "foobar"}) + m.set_value("example") + req = SyncRequest() + req.set_null_timestamp() + req.set_messages([m], HULC_Client()) + # create a sync response from the messages array + sr = SyncResponse({"merkle": "", "messages": req.messages}) + messages_decoded = sr.get_messages() + assert messages_decoded == [m] + + +def test_message_set_value(): + m = Message() + for data in ["foo", 1, 1.5, None]: + m.set_value(data) + assert m.get_value() == data + with pytest.raises(ValueError): + m.set_value(object()) # noqa + with pytest.raises(ValueError): + m.value = "T:foo" + m.get_value()