feat: Add basic documentation and testing

README.md

@@ -1,2 +1,89 @@
 # actualpy
-Python API implementation for Actual server - reference https://actualbudget.org/
+
+Python API implementation for Actual server.
+
+[Actual Budget](https://actualbudget.org/) is a super fast and privacy-focused app for managing your finances.
+
+> **WARNING:** The [Javascript API](https://actualbudget.org/docs/api/) to interact with Actual server already exists,
+> and is battle-tested as it is the core of the Actual frontend libraries. If you intend to use a reliable and well
+> tested library, that is the way to go.
+
+# Installation
+
+Install it via Pip using the repository url:
+
+```bash
+pip install https://github.com/bvanelli/actualpy
+```
+
+# Basic usage
+
+The most common usage would be downloading a budget to more easily build queries. This would you could handle the
+Actual database using SQLAlchemy instead of having to retrieve the data via the export. The following script will print
+every single transaction registered on the Actual budget file:
+
+```python
+from actual import Actual
+
+with Actual(
+    base_url="http://localhost:5006",  # Url of the Actual Server
+    password="<your_password>",  # Password for authentication
+    file="<file_id_or_name>",  # Set the file to work with. Can be either the file id or file name, if name is unique
+    data_dir="<path_to_data_directory>"  # Optional: Directory to store downloaded files. Will use a temporary if not provided
+) as actual:
+    transactions = actual.get_transactions()
+    for t in transactions:
+        account_name = t.account.name if t.account else None
+        category = t.category.name if t.category else None
+        print(t.date, account_name, t.notes, t.amount, category)
+```
+
+# Experimental features
+
+> **WARNING:** Experimental features do not have all the testing necessary to ensure correctness in comparison to the
+> files generated by the Javascript API. This means that this operations could in theory corrupt the data. Make sure
+> you have backups of your data before trying any of those operations.
+
+## Bootstraping a new server and uploading a first file
+
+The following script would generate a new empty budget on the Actual server, even if the server was not bootstrapped
+with an initial password.
+
+```python
+from actual import Actual
+
+with Actual(base_url="http://localhost:5006", password="mypass", bootstrap=True) as actual:
+    actual.create_budget("My budget")
+    actual.upload_budget()
+```
+
+You will then have a freshly created new budget to use:
+
+![created-budget](./docs/static/new-budget.png)
+
+# Adding new transactions
+
+After you created your first budget (or when updating an existing budget), you can add new transactions by adding them
+using the `actual.add()` method. You cannot use the SQLAlchemy session directly because that adds the entries to your
+local database, but will not sync the results back to the server (that is only possible when reuploading the file).
+
+The method will make sure the local database is updated, but will also send a SYNC request with the added data so that
+it will be immediately available on the frontend:
+
+```python
+import decimal
+import datetime
+from actual import Actual
+from actual.database import Transactions
+
+with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual:
+    act = actual.get_accounts()[0]  # get first account
+    t = Transactions.new(act.id, decimal.Decimal(10.5), datetime.date.today(), notes="My first transaction")
+    actual.add(t)
+```
+
+![added-transaction](./docs/static/added-transaction.png)
+
+# Contributing
+
+The goal is to have more features implemented and tested on the Actual API.

actual/__init__.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import contextlib
+import datetime
 import io
 import json
 import pathlib
@@ -19,13 +20,14 @@
 from actual.database import (
     Accounts,
     Categories,
+    MessagesClock,
     Payees,
     Transactions,
     get_attribute_by_table_name,
     get_class_by_table_name,
 )
 from actual.exceptions import InvalidZipFile, UnknownFileId
-from actual.protobuf_models import Message, SyncRequest
+from actual.protobuf_models import HULC_Client, Message, SyncRequest
 
 if TYPE_CHECKING:
     from actual.database import BaseModel
@@ -39,6 +41,7 @@ def __init__(
         password: str = None,
         file: str = None,
         data_dir: Union[str, pathlib.Path] = None,
+        bootstrap: bool = False,
     ):
         """
         Implements the Python API for the Actual Server in order to be able to read and modify information on Actual
@@ -53,12 +56,14 @@ def __init__(
         :param file: the name or id of the file to be set
         :param data_dir: where to store the downloaded files from the server. If not specified, a temporary folder will
             be created instead.
+        :param bootstrap: if the server is not bootstrapped, bootstrap it with the password.
         """
-        super().__init__(base_url, token, password)
+        super().__init__(base_url, token, password, bootstrap)
         self._file: RemoteFileListDTO | None = None
         self._data_dir = pathlib.Path(data_dir) if data_dir else None
         self._session_maker = None
         self._session: sqlalchemy.orm.Session | None = None
+        self._client: HULC_Client | None = None
         # set the correct file
         if file:
             self.set_file(file)
@@ -105,7 +110,9 @@ def run_migrations(self, migration_files: list[str]):
         .data_file_index() method. This first file is the base database, and the following files are migrations.
         Migrations can also be .js files. In this case, we have to extract and execute queries from the standard JS."""
         conn = sqlite3.connect(self._data_dir / "db.sqlite")
-        for file in migration_files[1:]:
+        for file in migration_files:
+            if not file.startswith("migrations"):
+                continue  # in case db.sqlite file gets passed
             file_id = file.split("_")[0].split("/")[1]
             if conn.execute(f"SELECT id FROM __migrations__ WHERE id = '{file_id}';").fetchall():
                 continue  # skip migration as it was already ran
@@ -146,6 +153,11 @@ def create_budget(self, budget_name: str):
         self._file = RemoteFileListDTO(name=budget_name, fileId=file_id, groupId=None, deleted=0, encryptKeyId=None)
         # create engine for downloaded database and run migrations
         self.run_migrations(migration_files[1:])
+        # generate a session
+        engine = sqlalchemy.create_engine(f"sqlite:///{self._data_dir}/db.sqlite")
+        self._session_maker = sqlalchemy.orm.sessionmaker(engine)
+        # create a clock
+        self.load_clock()
 
     def upload_budget(self):
         """Uploads the current file to the Actual server."""
@@ -158,6 +170,10 @@ def upload_budget(self):
             binary_data.seek(0)
         return self.upload_user_file(binary_data.getvalue(), self._file.file_id, self._file.name)
 
+    def reupload_budget(self):
+        self.reset_user_file(self._file.file_id)
+        self.upload_budget()
+
     def apply_changes(self, messages: list[Message]):
         """Applies a list of sync changes, based on what the sync method returned on the remote."""
         if not self._session_maker:
@@ -197,17 +213,33 @@ def download_budget(self):
         self._session_maker = sqlalchemy.orm.sessionmaker(engine)
         # actual js always calls validation
         self.validate()
+        # load the client id
+        self.load_clock()
         # after downloading the budget, some pending transactions still need to be retrieved using sync
         request = SyncRequest({"messages": [], "fileId": self._file.file_id, "groupId": self._file.group_id})
-        request.set_null_timestamp()  # using 0 timestamp to retrieve all changes
+        request.set_null_timestamp(client_id=self._client.client_id)  # using 0 timestamp to retrieve all changes
         changes = self.sync(request)
         self.apply_changes(changes.get_messages())
+        if changes.messages:
+            self._client = HULC_Client.from_timestamp(changes.messages[-1].timestamp)
 
-    def load_clock(self):
+    def load_clock(self) -> MessagesClock:
         """See implementation at:
         https://github.com/actualbudget/actual/blob/5bcfc71be67c6e7b7c8b444e4c4f60da9ea9fdaa/packages/loot-core/src/server/db/index.ts#L81-L98
         """
-        pass
+        with self.with_session() as session:
+            clock = session.query(MessagesClock).first()
+            if not clock:
+                clock_message = {
+                    "timestamp": HULC_Client().timestamp(now=datetime.datetime(1970, 1, 1, 0, 0, 0, 0)),
+                    "merkle": {},
+                }
+                clock = MessagesClock(id=0, clock=json.dumps(clock_message))
+                session.add(clock)
+            session.commit()
+            # add clock id to client id
+            self._client = HULC_Client.from_timestamp(json.loads(clock.clock)["timestamp"])
+        return clock
 
     def get_transactions(self) -> List[Transactions]:
         with self._session_maker() as s:
@@ -243,8 +275,8 @@ def add(self, model: BaseModel):
             s.add(model)
             # generate a sync request and sync it to the server
             req = SyncRequest({"fileId": self._file.file_id, "groupId": self._file.group_id})
-            req.set_timestamp()
-            req.set_messages(model.convert())
+            req.set_null_timestamp(client_id=self._client.client_id)
+            req.set_messages(model.convert(), self._client)
             self.sync(req)
             s.commit()
             if not self._session:

actual/api.py

@@ -15,6 +15,7 @@ class Endpoints(enum.Enum):
     INFO = "info"
     ACCOUNT_VALIDATE = "account/validate"
     NEEDS_BOOTSTRAP = "account/needs-bootstrap"
+    BOOTSTRAP = "account/bootstrap"
     SYNC = "sync/sync"
     LIST_USER_FILES = "sync/list-user-files"
     GET_USER_FILE_INFO = "sync/get-user-file-info"
@@ -122,14 +123,19 @@ def __init__(
         base_url: str = "http://localhost:5006",
         token: str = None,
         password: str = None,
+        bootstrap: bool = False,
     ):
         self.api_url = base_url
         self._token = token
         if token is None and password is None:
             raise ValueError("Either provide a valid token or a password.")
         # already try to login if password was provided
-        if password:
+        if password and bootstrap and not self.needs_bootstrap().data.bootstrapped:
+            self.bootstrap(password)
+        elif password:
             self.login(password)
+        # finally call validate
+        self.validate()
 
     def login(self, password: str) -> LoginDTO:
         """Logs in on the Actual server using the password provided. Raises `AuthorizationError` if it fails to
@@ -173,6 +179,13 @@ def needs_bootstrap(self) -> BootstrapInfoDTO:
         response.raise_for_status()
         return BootstrapInfoDTO.parse_obj(response.json())
 
+    def bootstrap(self, password: str) -> LoginDTO:
+        response = requests.post(f"{self.api_url}/{Endpoints.BOOTSTRAP}", json={"password": password})
+        response.raise_for_status()
+        login_response = LoginDTO.parse_obj(response.json())
+        self._token = login_response.data.token
+        return login_response
+
     def data_file_index(self) -> List[str]:
         """Gets all the migration file references for the actual server."""
         response = requests.get(f"{self.api_url}/{Endpoints.DATA_FILE_INDEX}")

actual/exceptions.py

@@ -1,3 +1,25 @@
+import requests
+
+
+def get_exception_from_response(response: requests.Response):
+    text = response.content.decode()
+    if text == "internal-error" or response.status_code == 500:
+        return ActualError(text)
+    # taken from
+    # https://github.com/actualbudget/actual-server/blob/6e9eddeb561b0d9f2bbb6301c3e2c30b4effc522/src/app-sync.js#L107
+    elif text == "file-has-new-key":
+        return ActualError(f"{text}: The data is encrypted with a different key")
+    elif text == "file-has-reset":
+        return InvalidFile(
+            f"{text}: The changes being synced are part of an old group, which means the file has been reset. "
+            f"User needs to re-download."
+        )
+    elif text in ("file-not-found", "file-needs-upload"):
+        raise UnknownFileId(text)
+    elif text == "file-old-version":
+        raise InvalidFile(f"{text}: SYNC_FORMAT_VERSION was generated with an old format")
+
+
 class ActualError(Exception):
     pass
 
@@ -12,3 +34,7 @@ class UnknownFileId(ActualError):
 
 class InvalidZipFile(ActualError):
     pass
+
+
+class InvalidFile(ActualError):
+    pass