diff --git a/README.md b/README.md index 579089a..0b14040 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,39 @@ class Task(KubernetesJobTask): ``` +### Volumes + +You can use volumes in the pods that run your tasks: + +Simple Ephemeral volume example: + +```python +with_ephemeral_volume200 = EphemeralVolume(size_in_gb=200) +class MyTask: + def __init__(self...): + ... + # you can find this volume in your containers under `/mnt/data/` + self.volumes = [with_ephemeral_volume200] +``` + +By leveraging volumes with cloud storage you can read and write data as if it existed locally. For example by mounting CSI drives your tasks can read inputs and write outputs to `/mnt/my_s3_bucket/`, this avoids complicated setups in which tasks have to know cloud specifics to read inputs and outputs + +We provide a base class for Azure blob storage, this pressuposes you installed azure blob CSI driver in your AKS cluster. + +```python +with_azure_blob_volume = AzureBlobStorageVolume(storage_account=AZ_STORAGE_ACCOUNT, + storage_container=AZ_CONTAINER) +class MyTask: + def __init__(self...): + ... + # you can find this volume in your containers under `/mnt/{AZ_STORAGE_ACCOUNT}/{AZ_CONTAINER}` + # you can use this convention to have your containers inputs and outputs params + # read data from this mount point + self.volumes = [with_azure_blob_volume] + +``` + + ## Logs Kubeluigi's task logs include Job, Task, and Pod identifiers: diff --git a/kubeluigi/volumes.py b/kubeluigi/volumes.py index 2d239f8..e7a82ed 100644 --- a/kubeluigi/volumes.py +++ b/kubeluigi/volumes.py @@ -86,7 +86,7 @@ def pod_mount_spec(self): } -class BlobStorageVolume(AttachableVolume): +class AzureBlobStorageVolume(AttachableVolume): """ Returns a volume which mounts an azure storage container. it assumes that the needed secret to access the underlying diff --git a/test/test_volumes.py b/test/test_volumes.py index 419e674..31d6f26 100644 --- a/test/test_volumes.py +++ b/test/test_volumes.py @@ -1,4 +1,4 @@ -from kubeluigi.volumes import BlobStorageVolume, EphemeralVolume +from kubeluigi.volumes import AzureBlobStorageVolume, EphemeralVolume from test.test_kubernetes_job_task import DummyTask from kubeluigi.k8s import attach_volume_to_spec @@ -8,7 +8,7 @@ def test_volumes(): t = DummyTask() # testing attaching volumes - v = BlobStorageVolume("my_account", "my_storage") + v = AzureBlobStorageVolume("my_account", "my_storage") t.add_volume(v) assert t.volumes == [v] # augmenting pod spec @@ -42,10 +42,10 @@ def test_ephemeral_volumes(): def test_blob_storage_volumes(): - v = BlobStorageVolume("my_account", "my_container") + v = AzureBlobStorageVolume("my_account", "my_container") assert v.secret_name() == "storage-sas-my_account" - v2 = BlobStorageVolume("my_account", "my_container") + v2 = AzureBlobStorageVolume("my_account", "my_container") assert v.volume_hash() == v2.volume_hash() vol_spec = v.pod_volume_spec() @@ -62,7 +62,7 @@ def test_blob_storage_volumes(): def test_attach_volume_to_spec(): dummy_spec = {"containers": [{"volume_mounts": []}], "volumes": []} - v = BlobStorageVolume("my_account", "my_container") + v = AzureBlobStorageVolume("my_account", "my_container") updated_spec = attach_volume_to_spec(dummy_spec, v) assert len(updated_spec["volumes"]) == 1 assert len(updated_spec["containers"][0]["volume_mounts"]) == 1