diff --git a/README.md b/README.md index 73bab63..94084fe 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Python API implementation for Actual server. -[Actual Budget](https://actualbudget.org/) is a super fast and privacy-focused app for managing your finances. +[Actual Budget](https://actualbudget.org/) is a superfast and privacy-focused app for managing your finances. > **WARNING:** The [Javascript API](https://actualbudget.org/docs/api/) to interact with Actual server already exists, > and is battle-tested as it is the core of the Actual frontend libraries. If you intend to use a reliable and well @@ -29,6 +29,7 @@ from actual.queries import get_transactions with Actual( base_url="http://localhost:5006", # Url of the Actual Server password="", # Password for authentication + encryption_password=None, # Optional: Password for the file encryption. Will not use it if set to None. file="", # Set the file to work with. Can be either the file id or file name, if name is unique data_dir="" # Optional: Directory to store downloaded files. Will use a temporary if not provided ) as actual: @@ -62,11 +63,13 @@ You will then have a freshly created new budget to use: ![created-budget](./docs/static/new-budget.png) +If the `encryption_password` is set, the budget will additionally also be encrypted on the upload step to the server. + # Adding new transactions After you created your first budget (or when updating an existing budget), you can add new transactions by adding them using the `actual.session.add()` method. You cannot use the SQLAlchemy session directly because that adds the entries to your -local database, but will not sync the results back to the server (that is only possible when reuploading the file). +local database, but will not sync the results back to the server (that is only possible when re-uploading the file). The method will make sure the local database is updated, but will also send a SYNC request with the added data so that it will be immediately available on the frontend: @@ -75,12 +78,17 @@ it will be immediately available on the frontend: import decimal import datetime from actual import Actual -from actual.queries import get_accounts, create_transaction_from_ids +from actual.queries import create_transaction, create_account with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: - act = get_accounts(actual.session)[0] # get first account - t = create_transaction_from_ids( - actual.session, datetime.date.today(), act.id, None, notes="My first transaction", amount=decimal.Decimal(10.5) + act = create_account(actual.session, "My account") + t = create_transaction( + actual.session, + datetime.date.today(), + act, + "My payee", + notes="My first transaction", + amount=decimal.Decimal(-10.5), ) actual.session.add(t) actual.commit() # use the actual.commit() instead of session.commit()! @@ -88,6 +96,38 @@ with Actual(base_url="http://localhost:5006", password="mypass", file="My budget ![added-transaction](./docs/static/added-transaction.png) +# Understanding how Actual handles changes + +The Actual budget is stored in a sqlite database hosted on the user's browser. This means all your data is fully local +and can be encrypted with a local key, so that not even the server can read your statements. + +The Actual Server is a way of only hosting files and changes. Since re-uploading the full database on every single +change is too heavy, Actual only stores one state of the database and everything added by the user via frontend +or via the APIs are individual changes above the base stored database on the server, stored via separate endpoint. +This means that on every change, done locally, a SYNC request is sent to the server with a list of the following string +parameters: + +- `dataset`: the name of the table where the change happened. +- `row`: the row identifier for the entry that was added/update. This would be the primary key of the row (a uuid value) +- `column`: the column that had the value changed +- `value`: the new value. Since it's a string, the values are either prefixed by `S:` to denote a string, `N:` to denote +a numeric value and `0:` to denote a null value. + +All individual column changes are computed on an insert, serialized with protobuf and sent to the server to be stored. +New clients can use this individual changes to then sync their local copies and add the changes executed on other users. + +But this also means that new users need to download a long list of changes, possibly making the initialization slow. +Thankfully, user is also allowed to reset the sync. This would make sure all changes are actually stored in the +database. This is done on the frontend under *Settings > Reset sync*, and causes the current file to be reset (removed +from the server) and re-uploaded again, with all changes already in place. In this case, the sync list of changes is +reset because the server already has the latest version of the database. + +This means that, when using this library to operate changes on the database, you have to make sure that either: + +- do a sync request is made using the `actual.commit()` method. This only handles pending operations that haven't yet +been committed, generates a change list with them and posts them on the sync endpoint. +- do a full re-upload of the database is done. + # Contributing The goal is to have more features implemented and tested on the Actual API. If you have ideas, comments, bug fixes or diff --git a/actual/__init__.py b/actual/__init__.py index 9d01054..b213aa4 100644 --- a/actual/__init__.py +++ b/actual/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import datetime import io import json @@ -9,12 +10,13 @@ import tempfile import uuid import zipfile -from typing import TYPE_CHECKING, Union +from typing import Union import sqlalchemy import sqlalchemy.orm from actual.api import ActualServer, RemoteFileListDTO +from actual.crypto import create_key_buffer, decrypt_from_meta, encrypt, make_salt from actual.database import ( MessagesClock, get_attribute_by_table_name, @@ -28,9 +30,6 @@ ) from actual.protobuf_models import HULC_Client, Message, SyncRequest -if TYPE_CHECKING: - pass - class Actual(ActualServer): def __init__( @@ -39,6 +38,7 @@ def __init__( token: str = None, password: str = None, file: str = None, + encryption_password: str = None, data_dir: Union[str, pathlib.Path] = None, bootstrap: bool = False, ): @@ -53,6 +53,7 @@ def __init__( :param token: the token for authentication, if this is available (optional) :param password: the password for authentication. It will be used on the .login() method to retrieve the token. :param file: the name or id of the file to be set + :param encryption_password: password used to configure encryption, if existing :param data_dir: where to store the downloaded files from the server. If not specified, a temporary folder will be created instead. :param bootstrap: if the server is not bootstrapped, bootstrap it with the password. @@ -66,11 +67,13 @@ def __init__( # set the correct file if file: self.set_file(file) + self._encryption_password = encryption_password + self._master_key = None self._in_context = False def __enter__(self) -> Actual: if self._file: - self.download_budget() + self.download_budget(self._encryption_password) self._session = self.session_maker() self._in_context = True return self @@ -126,8 +129,9 @@ def run_migrations(self, migration_files: list[str]): conn.commit() conn.close() - def create_budget(self, budget_name: str): - """Creates a budget using the remote server default database and migrations.""" + def create_budget(self, budget_name: str, encryption_password: str = None): + """Creates a budget using the remote server default database and migrations. If password is provided, the + budget will be encrypted.""" migration_files = self.data_file_index() # create folder for the files if not self._data_dir: @@ -138,16 +142,14 @@ def create_budget(self, budget_name: str): # also write the metadata file with default fields random_id = str(uuid.uuid4()).replace("-", "")[:7] file_id = str(uuid.uuid4()) - (self._data_dir / "metadata.json").write_text( - json.dumps( - { - "id": f"My-Finances-{random_id}", - "budgetName": budget_name, - "userId": self._token, - "cloudFileId": file_id, - "resetClock": True, - } - ) + self.update_metadata( + { + "id": f"My-Finances-{random_id}", + "budgetName": budget_name, + "userId": self._token, + "cloudFileId": file_id, + "resetClock": True, + } ) self._file = RemoteFileListDTO(name=budget_name, fileId=file_id, groupId=None, deleted=0, encryptKeyId=None) # create engine for downloaded database and run migrations @@ -160,6 +162,40 @@ def create_budget(self, budget_name: str): # create a clock self.load_clock() + def _gen_zip(self) -> bytes: + binary_data = io.BytesIO() + with zipfile.ZipFile(binary_data, "a", zipfile.ZIP_DEFLATED, False) as z: + z.write(self._data_dir / "db.sqlite", "db.sqlite") + z.write(self._data_dir / "metadata.json", "metadata.json") + return binary_data.getvalue() + + def encrypt(self, encryption_password: str): + """Encrypts the local database using a new key, and re-uploads to the server. + + WARNING: this resets the file on the server. Make sure you have a copy of the database before attempting this + operation. + """ + if encryption_password and not self._file.encrypt_key_id: + # password was provided, but encryption key not, create one + key_id = str(uuid.uuid4()) + salt = make_salt() + self.user_create_key(self._file.file_id, key_id, encryption_password, salt) + self.update_metadata({"encryptKeyId": key_id}) + self._file.encrypt_key_id = key_id + elif self._file.encrypt_key_id: + key_info = self.user_get_key(self._file.file_id) + salt = key_info.data.salt + else: + raise ActualError("Budget is encrypted but password was not provided") + self._master_key = create_key_buffer(encryption_password, salt) + # encrypt binary data with + encrypted = encrypt(self._file.encrypt_key_id, self._master_key, self._gen_zip()) + binary_data = io.BytesIO(base64.b64decode(encrypted["value"])) + encryption_meta = encrypted["meta"] + self.reset_user_file(self._file.file_id) + self.upload_user_file(binary_data.getvalue(), self._file.file_id, self._file.name, encryption_meta) + self.set_file(self._file.file_id) + def upload_budget(self): """Uploads the current file to the Actual server.""" if not self._data_dir: @@ -168,8 +204,11 @@ def upload_budget(self): with zipfile.ZipFile(binary_data, "a", zipfile.ZIP_DEFLATED, False) as z: z.write(self._data_dir / "db.sqlite", "db.sqlite") z.write(self._data_dir / "metadata.json", "metadata.json") - binary_data.seek(0) - return self.upload_user_file(binary_data.getvalue(), self._file.file_id, self._file.name) + # we have to first upload the user file so the reference id can be used to generate a new encryption key + self.upload_user_file(binary_data.getvalue(), self._file.file_id, self._file.name) + # encrypt the file and re-upload + if self._encryption_password or self._master_key or self._file.encrypt_key_id: + self.encrypt(self._encryption_password) def reupload_budget(self): self.reset_user_file(self._file.file_id) @@ -183,9 +222,7 @@ def apply_changes(self, messages: list[Message]): for message in messages: if message.dataset == "prefs": # write it to metadata.json instead - config = json.loads((self._data_dir / "metadata.json").read_text() or "{}") - config[message.row] = message.get_value() - (self._data_dir / "metadata.json").write_text(json.dumps(config)) + self.update_metadata({message.row: message.get_value()}) continue table = get_class_by_table_name(message.dataset) column = get_attribute_by_table_name(message.dataset, message.column) @@ -198,16 +235,38 @@ def apply_changes(self, messages: list[Message]): s.flush() s.commit() - def download_budget(self): + def update_metadata(self, patch: dict): + """Updates the metadata.json from the Actual file with the patch fields. The patch is a dictionary that will + then be merged on the metadata and written again to a file.""" + config = json.loads((self._data_dir / "metadata.json").read_text() or "{}") | patch + (self._data_dir / "metadata.json").write_text(json.dumps(config)) + + def download_budget(self, encryption_password: str = None): """Downloads the budget file from the remote. After the file is downloaded, the sync endpoint is queries for the list of pending changes. The changes are individual row updates, that are then applied on by one to - the downloaded database state.""" + the downloaded database state. + + If the budget is password protected, the password needs to be present to download the budget, otherwise it will + fail. + """ file_bytes = self.download_user_file(self._file.file_id) + + if self._file.encrypt_key_id and encryption_password is None: + raise ActualError("File is encrypted but no encryption password provided.") + if encryption_password is not None: + file_info = self.get_user_file_info(self._file.file_id) + key_info = self.user_get_key(self._file.file_id) + self._master_key = create_key_buffer(encryption_password, key_info.data.salt) + # decrypt file bytes + file_bytes = decrypt_from_meta(self._master_key, file_bytes, file_info.data.encrypt_meta) + self._load_zip(file_bytes) + + def _load_zip(self, file_bytes: bytes): f = io.BytesIO(file_bytes) try: zip_file = zipfile.ZipFile(f) except zipfile.BadZipfile as e: - raise InvalidZipFile(f"Invalid zip file: {e}") + raise InvalidZipFile(f"Invalid zip file: {e}") from None if not self._data_dir: self._data_dir = pathlib.Path(tempfile.mkdtemp()) # this should extract 'db.sqlite' and 'metadata.json' to the folder @@ -219,10 +278,17 @@ def download_budget(self): # load the client id self.load_clock() # after downloading the budget, some pending transactions still need to be retrieved using sync - request = SyncRequest({"messages": [], "fileId": self._file.file_id, "groupId": self._file.group_id}) + request = SyncRequest( + { + "messages": [], + "fileId": self._file.file_id, + "groupId": self._file.group_id, + "keyId": self._file.encrypt_key_id, + } + ) request.set_null_timestamp(client_id=self._client.client_id) # using 0 timestamp to retrieve all changes changes = self.sync(request) - self.apply_changes(changes.get_messages()) + self.apply_changes(changes.get_messages(self._master_key)) if changes.messages: self._client = HULC_Client.from_timestamp(changes.messages[-1].timestamp) @@ -257,14 +323,16 @@ def commit(self): ) # create sync request based on the session reference that is tracked req = SyncRequest({"fileId": self._file.file_id, "groupId": self._file.group_id}) + if self._file.encrypt_key_id: + req.keyId = self._file.encrypt_key_id req.set_null_timestamp(client_id=self._client.client_id) # first we add all new entries and modify is required for model in self._session.new: - req.set_messages(model.convert(is_new=True), self._client) + req.set_messages(model.convert(is_new=True), self._client, master_key=self._master_key) # modify if required for model in self._session.dirty: if self._session.is_modified(model): - req.set_messages(model.convert(is_new=False), self._client) + req.set_messages(model.convert(is_new=False), self._client, master_key=self._master_key) # make sure changes are valid before syncing self._session.commit() # sync all changes to the server diff --git a/actual/api.py b/actual/api.py index fe7f174..1cc5bfc 100644 --- a/actual/api.py +++ b/actual/api.py @@ -1,11 +1,13 @@ from __future__ import annotations import enum +import json from typing import List, Optional import requests from pydantic import BaseModel, Field +from actual.crypto import create_key_buffer, make_test_message from actual.exceptions import AuthorizationError, UnknownFileId from actual.protobuf_models import SyncRequest, SyncResponse @@ -43,21 +45,6 @@ class StatusDTO(BaseModel): status: StatusCode -class FileDTO(BaseModel): - deleted: Optional[int] - file_id: Optional[str] = Field(..., alias="fileId") - group_id: Optional[str] = Field(..., alias="groupId") - name: Optional[str] - - -class RemoteFileListDTO(FileDTO): - encrypt_key_id: Optional[str] = Field(..., alias="encryptKeyId") - - -class RemoteFileDTO(FileDTO): - encrypt_meta: Optional[EncryptMetaDTO] = Field(..., alias="encryptMeta") - - class TokenDTO(BaseModel): token: Optional[str] @@ -70,10 +57,6 @@ class UploadUserFileDTO(StatusDTO): group_id: str = Field(..., alias="groupId") -class ListUserFilesDTO(StatusDTO): - data: List[RemoteFileListDTO] - - class IsValidatedDTO(BaseModel): validated: Optional[bool] @@ -82,17 +65,48 @@ class ValidateDTO(StatusDTO): data: IsValidatedDTO +class EncryptMetaDTO(BaseModel): + key_id: Optional[str] = Field(..., alias="keyId") + algorithm: Optional[str] + iv: Optional[str] + auth_tag: Optional[str] = Field(..., alias="authTag") + + +class EncryptionTestDTO(BaseModel): + value: str + meta: EncryptMetaDTO + + class EncryptionDTO(BaseModel): id: Optional[str] salt: Optional[str] test: Optional[str] + def meta(self) -> EncryptionTestDTO: + return EncryptionTestDTO.parse_raw(self.test) -class EncryptMetaDTO(BaseModel): - key_id: Optional[str] = Field(..., alias="keyId") - algorithm: Optional[str] - iv: Optional[str] - auth_tag: Optional[str] = Field(..., alias="authTag") + +class FileDTO(BaseModel): + deleted: Optional[int] + file_id: Optional[str] = Field(..., alias="fileId") + group_id: Optional[str] = Field(..., alias="groupId") + name: Optional[str] + + +class RemoteFileListDTO(FileDTO): + encrypt_key_id: Optional[str] = Field(..., alias="encryptKeyId") + + +class RemoteFileDTO(FileDTO): + encrypt_meta: Optional[EncryptMetaDTO] = Field(..., alias="encryptMeta") + + +class GetUserFileInfoDTO(StatusDTO): + data: RemoteFileDTO + + +class ListUserFilesDTO(StatusDTO): + data: List[RemoteFileListDTO] class UserGetKeyDTO(StatusDTO): @@ -217,19 +231,24 @@ def download_user_file(self, file_id: str) -> bytes: db.raise_for_status() return db.content - def upload_user_file(self, binary_data: bytes, file_id: str, file_name: str = "My Finances") -> UploadUserFileDTO: - """Uploads the binary data, which is a zip folder containing the `db.sqlite` and the `metadata.json`.""" + def upload_user_file( + self, binary_data: bytes, file_id: str, file_name: str = "My Finances", encryption_meta: dict = None + ) -> UploadUserFileDTO: + """Uploads the binary data, which is a zip folder containing the `db.sqlite` and the `metadata.json`. If the + file is encrypted, the encryption_meta has to be provided with fields `keyId`, `algorithm`, `iv` and `authTag` + """ + base_headers = { + "X-ACTUAL-FORMAT": "2", + "X-ACTUAL-FILE-ID": file_id, + "X-ACTUAL-NAME": file_name, + "Content-Type": "application/encrypted-file", + } + if encryption_meta: + base_headers["X-ACTUAL-ENCRYPT-META"] = json.dumps(encryption_meta) request = requests.post( f"{self.api_url}/{Endpoints.UPLOAD_USER_FILE}", data=binary_data, - headers=self.headers( - extra_headers={ - "X-ACTUAL-FORMAT": "2", - "X-ACTUAL-FILE-ID": file_id, - "X-ACTUAL-NAME": file_name, - "Content-Type": "application/encrypted-file", - } - ), + headers=self.headers(extra_headers=base_headers), ) request.raise_for_status() return UploadUserFileDTO.parse_obj(request.json()) @@ -241,11 +260,11 @@ def list_user_files(self) -> ListUserFilesDTO: response.raise_for_status() return ListUserFilesDTO.parse_obj(response.json()) - def get_user_file_info(self, file_id: str) -> RemoteFileDTO: + def get_user_file_info(self, file_id: str) -> GetUserFileInfoDTO: """Gets the user file information, including the encryption metadata.""" response = requests.get(f"{self.api_url}/{Endpoints.GET_USER_FILE_INFO}", headers=self.headers(file_id)) response.raise_for_status() - return RemoteFileDTO.parse_obj(response.json()) + return GetUserFileInfoDTO.parse_obj(response.json()) def update_user_file_name(self, file_id: str, file_name: str) -> StatusDTO: """Updates the file name for the budget on the remote server.""" @@ -259,10 +278,10 @@ def update_user_file_name(self, file_id: str, file_name: str) -> StatusDTO: def user_get_key(self, file_id: str) -> UserGetKeyDTO: """Gets the key information associated with a user file, including the algorithm, key, salt and iv.""" - response = requests.get( + response = requests.post( f"{self.api_url}/{Endpoints.USER_GET_KEY}", json={ - "file_id": file_id, + "fileId": file_id, "token": self._token, }, headers=self.headers(file_id), @@ -270,10 +289,11 @@ def user_get_key(self, file_id: str) -> UserGetKeyDTO: response.raise_for_status() return UserGetKeyDTO.parse_obj(response.json()) - def user_create_key(self, file_id: str, key_id: str, key_salt: str) -> StatusDTO: + def user_create_key(self, file_id: str, key_id: str, password: str, key_salt: str) -> StatusDTO: """Creates a new key for the user file. The key has to be used then to encrypt the local file, and this file still needs to be uploaded.""" - test_content = "" # todo: see how this is generated + key = create_key_buffer(password, key_salt) + test_content = make_test_message(key_id, key) response = requests.post( f"{self.api_url}/{Endpoints.USER_CREATE_KEY}", headers=self.headers(), @@ -281,7 +301,7 @@ def user_create_key(self, file_id: str, key_id: str, key_salt: str) -> StatusDTO "fileId": file_id, "keyId": key_id, "keySalt": key_salt, - "testContent": test_content, + "testContent": json.dumps(test_content), "token": self._token, }, ) diff --git a/actual/crypto.py b/actual/crypto.py new file mode 100644 index 0000000..82cb20a --- /dev/null +++ b/actual/crypto.py @@ -0,0 +1,68 @@ +import base64 +import os + +import cryptography.exceptions +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + +from actual.exceptions import ActualDecryptionError + + +def random_bytes(size: int = 12) -> str: + return str(os.urandom(size)) + + +def make_salt(length: int = 32) -> str: + # reference generates 32 bytes of random data + # github.com/actualbudget/actual/blob/70e37c0119f4ba95ccf6549f0df4aac770f1bb8f/packages/loot-core/src/server/main.ts#L1489 + return base64.b64encode(os.urandom(length)).decode() + + +def create_key_buffer(password: str, key_salt: str) -> bytes: + if key_salt is None: + key_salt = make_salt() + kdf = PBKDF2HMAC(algorithm=hashes.SHA512(), length=32, salt=key_salt.encode(), iterations=10_000) + return kdf.derive(password.encode()) + + +def encrypt(key_id: str, master_key: bytes, plaintext: bytes) -> dict: + iv = os.urandom(12) + encryptor = Cipher(algorithms.AES(master_key), modes.GCM(iv)).encryptor() + value = encryptor.update(plaintext) + encryptor.finalize() + auth_tag = encryptor.tag + return { + "value": base64.b64encode(value).decode(), + "meta": { + "keyId": key_id, + "algorithm": "aes-256-gcm", + "iv": base64.b64encode(iv).decode(), + "authTag": base64.b64encode(auth_tag).decode(), + }, + } + + +def decrypt(master_key: bytes, iv: bytes, ciphertext: bytes, auth_tag: bytes = None) -> bytes: + decryptor = Cipher(algorithms.AES(master_key), modes.GCM(iv, auth_tag)).decryptor() + try: + return decryptor.update(ciphertext) + decryptor.finalize() + except cryptography.exceptions.InvalidTag: + raise ActualDecryptionError("Error decrypting file. Is the encryption key correct?") from None + + +def decrypt_from_meta(master_key: bytes, ciphertext: bytes, encrypt_meta) -> bytes: + iv = base64.b64decode(encrypt_meta.iv) + auth_tag = base64.b64decode(encrypt_meta.auth_tag) + return decrypt(master_key, iv, ciphertext, auth_tag) + + +def make_test_message(key_id: str, key: bytes) -> dict: + """Reference + https://github.com/actualbudget/actual/blob/70e37c0119f4ba95ccf6549f0df4aac770f1bb8f/packages/loot-core/src/server/sync/make-test-message.ts#L10 + """ + from actual.protobuf_models import Message + + m = Message(dict(dataset=random_bytes(), row=random_bytes(), column=random_bytes(), value=random_bytes())) + binary_message = Message.serialize(m) + # return encrypted binary message + return encrypt(key_id, key, binary_message) diff --git a/actual/exceptions.py b/actual/exceptions.py index 12ff17b..6dc99a7 100644 --- a/actual/exceptions.py +++ b/actual/exceptions.py @@ -42,3 +42,7 @@ class InvalidZipFile(ActualError): class InvalidFile(ActualError): pass + + +class ActualDecryptionError(ActualError): + pass diff --git a/actual/protobuf_models.py b/actual/protobuf_models.py index ef1ce55..091ce82 100644 --- a/actual/protobuf_models.py +++ b/actual/protobuf_models.py @@ -1,10 +1,13 @@ from __future__ import annotations +import base64 import datetime import uuid import proto +from actual.crypto import decrypt, encrypt + """ Protobuf message definitions taken from: @@ -18,7 +21,7 @@ class HULC_Client: def __init__(self, client_id: str = None, initial_count: int = 0): - self.client_id = client_id + self.client_id = client_id or self.get_client_id() self.initial_count = initial_count @classmethod @@ -47,7 +50,9 @@ def get_client_id(self): https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts#L80 """ - return self.client_id if self.client_id is not None else str(uuid.uuid4()).replace("-", "")[-16:] + return ( + self.client_id if getattr(self, "client_id", None) is not None else str(uuid.uuid4()).replace("-", "")[-16:] + ) class EncryptedData(proto.Message): @@ -114,11 +119,24 @@ def set_timestamp(self, client_id: str = None, now: datetime.datetime = None) -> def set_null_timestamp(self, client_id: str = None) -> str: return self.set_timestamp(client_id, datetime.datetime(1970, 1, 1, 0, 0, 0, 0)) - def set_messages(self, messages: list[Message], client: HULC_Client): + def set_messages(self, messages: list[Message], client: HULC_Client, master_key: bytes = None): if not self.messages: self.messages = [] for message in messages: - m = MessageEnvelope({"content": Message.serialize(message), "isEncrypted": False}) + content = Message.serialize(message) + is_encrypted = False + if master_key is not None: + encrypted_content = encrypt("", master_key, content) + encrypted_data = EncryptedData( + { + "iv": base64.b64decode(encrypted_content["meta"]["iv"]), + "authTag": base64.b64decode(encrypted_content["meta"]["authTag"]), + "data": base64.b64decode(encrypted_content["value"]), + } + ) + content = EncryptedData.serialize(encrypted_data) + is_encrypted = True + m = MessageEnvelope({"content": content, "isEncrypted": is_encrypted}) m.timestamp = client.timestamp() self.messages.append(m) @@ -127,8 +145,15 @@ class SyncResponse(proto.Message): messages = proto.RepeatedField(MessageEnvelope, number=1) merkle = proto.Field(proto.STRING, number=2) - def get_messages(self) -> list[Message]: + def get_messages(self, master_key: bytes = None) -> list[Message]: messages = [] for message in self.messages: # noqa - messages.append(Message.deserialize(message.content)) + if message.isEncrypted: + if not master_key: + raise ValueError("Master key not provided and data is encrypted.") + encrypted = EncryptedData.deserialize(message.content) + content = decrypt(master_key, encrypted.iv, encrypted.data, encrypted.authTag) + else: + content = message.content + messages.append(Message.deserialize(content)) return messages diff --git a/docs/static/added-transaction.png b/docs/static/added-transaction.png index 2e63726..b924230 100644 Binary files a/docs/static/added-transaction.png and b/docs/static/added-transaction.png differ diff --git a/requirements.txt b/requirements.txt index 942334d..b567116 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ pydantic>=2 sqlalchemy>=1.4 proto-plus>=1 protobuf>=4 +cryptography>=42 diff --git a/tests/test_crypto.py b/tests/test_crypto.py new file mode 100644 index 0000000..656874c --- /dev/null +++ b/tests/test_crypto.py @@ -0,0 +1,40 @@ +import base64 + +import pytest + +from actual.api import EncryptMetaDTO +from actual.crypto import create_key_buffer, decrypt_from_meta, encrypt, random_bytes +from actual.exceptions import ActualDecryptionError +from actual.protobuf_models import HULC_Client, Message, SyncRequest, SyncResponse + + +def test_create_key_buffer(): + # Tested based on: + # const crypto = require('crypto'); + # console.log(crypto.pbkdf2Sync('foo', 'bar', 10000, 32, 'sha512').toString("base64")) + buffer = create_key_buffer("foo", base64.b64encode(b"bar").decode()) + assert base64.b64encode(buffer).decode() == "+Do1kTWpkRT0w4kl2suJLdbY1BLtyEpRCiImRtslNgQ=" + + +def test_encrypt_decrypt(): + key = create_key_buffer("foo", "bar") + string_to_encrypt = b"foobar" + encrypted = encrypt("foo", key, string_to_encrypt) + decrypted_from_meta = decrypt_from_meta( + key, base64.b64decode(encrypted["value"]), EncryptMetaDTO(**encrypted["meta"]) + ) + assert decrypted_from_meta == string_to_encrypt + with pytest.raises(ActualDecryptionError): + decrypt_from_meta(key[::-1], base64.b64decode(encrypted["value"]), EncryptMetaDTO(**encrypted["meta"])) + + +def test_encrypt_decrypt_message(): + key = create_key_buffer("foo", "bar") + m = Message(dict(dataset=random_bytes(), row=random_bytes(), column=random_bytes(), value=random_bytes())) + req = SyncRequest() + req.set_messages([m], HULC_Client(), master_key=key) + resp = SyncResponse() + resp.messages = req.messages + decrypted_messages = resp.get_messages(master_key=key) + assert len(decrypted_messages) == 1 + assert decrypted_messages[0] == m