Merge pull request #141 from nebari-dev/upgrade-components

Upgrade components
nebari-dev · Feb 21, 2024 · 01753e7 · 01753e7
2 parents cc53932 + 1e0392f
commit 01753e7
Show file tree

Hide file tree

Showing 21 changed files with 186 additions and 114 deletions.
diff --git a/inventory.template/group_vars/all.yaml b/inventory.template/group_vars/all.yaml
@@ -37,7 +37,12 @@ jupyterhub_services:
 mysql_users:
   - username: slurm
     password: W9T0N4ejQBR4RmQCU6GmbbZa
-    privileges: '*.*:ALL'
+    privileges: "*.*:ALL"
   - username: conda-store
     password: eIbmUditL4RbQm0YPeLozRme
-    privileges: '*.*:ALL'
+    privileges: "*.*:ALL"
+
+postgres_users:
+  - username: conda-store
+    password: eIbmUditL4RbQm0YPeLozRme
+    role: 'CREATEDB,CREATEROLE'
diff --git a/playbook.yaml b/playbook.yaml
@@ -23,6 +23,7 @@
      - cifs
      - nfs
      - mysql
+     - postgresql
      - minio
      - backups
      - traefik

diff --git a/roles/conda_store/defaults/main.yaml b/roles/conda_store/defaults/main.yaml
@@ -1,5 +1,5 @@
 conda_store_enabled: false
-conda_store_version: "0.3.11"
+conda_store_version: "2024.1.1"
 conda_store_port: "5000"
 conda_store_environment: "environments/conda-store.yaml"
 conda_store_prefix: "/conda-store"
@@ -17,13 +17,21 @@ mysql_databases:
   - slurm
   - conda-store
 
+postgresql_databases:
+  - conda-store
+
 mysql_users:
   - username: slurm
     password: W9T0N4ejQBR4RmQCU6GmbbZa
-    privileges: '*.*:ALL'
+    privileges: "*.*:ALL"
+  - username: conda-store
+    password: eIbmUditL4RbQm0YPeLozRme
+    privileges: "*.*:ALL"
+
+postgres_users:
   - username: conda-store
     password: eIbmUditL4RbQm0YPeLozRme
-    privileges: '*.*:ALL'
+    role: 'CREATEDB,CREATEROLE'
 
 # role: keycloak
 keycloak_port: "30020"

diff --git a/roles/conda_store/templates/conda_store_config.py b/roles/conda_store/templates/conda_store_config.py
@@ -1,5 +1,8 @@
 import logging
 
+import requests
+from conda_store_server import schema, api, orm
+from conda_store_server.server.dependencies import get_conda_store
 from conda_store_server.storage import S3Storage
 from conda_store_server.server.auth import GenericOAuthAuthentication
 
@@ -8,8 +11,7 @@
 # ==================================
 c.CondaStore.storage_class = S3Storage
 c.CondaStore.store_directory = "/opt/conda-store/conda-store/"
-c.CondaStore.conda_command = "conda"
-c.CondaStore.database_url = "mysql+pymysql://{{ mysql_users[1].username }}:{{ mysql_users[1].password }}@localhost/{{ mysql_databases[1] }}"
+c.CondaStore.database_url = "postgresql+psycopg2://{{ postgres_users[0].username }}:{{ postgres_users[0].password }}@localhost/{{ postgres_databases[0] }}"
 c.CondaStore.default_uid = 1000
 c.CondaStore.default_gid = 100
 c.CondaStore.default_permissions = "775"
@@ -36,6 +38,7 @@
 c.CondaStoreServer.enable_metrics = True
 c.CondaStoreServer.address = "0.0.0.0"
 c.CondaStoreServer.port = {{ conda_store_port }}
+c.CondaStoreServer.behind_proxy = True
 c.CondaStoreServer.url_prefix = "{{ conda_store_prefix }}"
 
 
@@ -52,12 +55,8 @@
 c.GenericOAuthAuthentication.user_data_key = "preferred_username"
 c.GenericOAuthAuthentication.tls_verify = False
 
-import requests
-from conda_store_server import schema, api, orm
-from conda_store_server.server.utils import get_conda_store
-
 class KeyCloakAuthentication(GenericOAuthAuthentication):
-    def authenticate(self, request):
+    async def authenticate(self, request):
         # 1. using the callback_url code and state in request
         oauth_access_token = self._get_oauth_token(request)
         if oauth_access_token is None:
@@ -71,30 +70,43 @@ def authenticate(self, request):
         response.raise_for_status()
         user_data = response.json()
 
+        username = user_data["preferred_username"]
+
+        # superadmin gets access to everything
+        if "conda_store_superadmin" in user_data.get("roles", []):
+            return schema.AuthenticationToken(
+                primary_namespace=username,
+                role_bindings={"*/*": {"admin"}},
+            )
+
         role_mappings = {
             'conda_store_admin': 'admin',
             'conda_store_developer': 'developer',
             'conda_store_viewer': 'viewer',
         }
-        roles = {role_mappings[role] for role in user_data.get('roles', []) if role in role_mappings}
-        username = user_data['preferred_username']
+        roles = {
+            role_mappings[role]
+            for role in user_data.get('roles', [])
+            if role in role_mappings
+        }
         namespaces = {username, 'default', 'filesystem'}
         role_bindings = {
             f'{username}/*': {'admin'},
-            f'filesystem/*': {'reader'},
+            f'filesystem/*': {'viewer'},
             f'default/*': roles,
         }
 
         for group in user_data.get('groups', []):
             namespaces.add(group)
             role_bindings[f'{group}/*'] = roles
 
-        conda_store = get_conda_store()
-        for namespace in namespaces:
-            _namespace = api.get_namespace(conda_store.db, name=namespace)
-            if _namespace is None:
-                conda_store.db.add(orm.Namespace(name=namespace))
-                conda_store.db.commit()
+        conda_store = await get_conda_store(request)
+        with conda_store.session_factory() as db:
+            for namespace in namespaces:
+                _namespace = api.get_namespace(db, name=namespace)
+                if _namespace is None:
+                    db.add(orm.Namespace(name=namespace))
+                    db.commit()
 
         return schema.AuthenticationToken(
             primary_namespace=username,
@@ -109,3 +121,4 @@ def authenticate(self, request):
 c.CondaStoreWorker.log_level = logging.INFO
 c.CondaStoreWorker.watch_paths = ["/opt/environments"]
 c.CondaStoreWorker.concurrency = 4
+c.CondaStore.environment_directory = "/opt/conda-store/conda-store/{namespace}/envs/{namespace}-{name}"
diff --git a/roles/conda_store/templates/environments/conda-store.yaml b/roles/conda_store/templates/environments/conda-store.yaml
@@ -2,6 +2,37 @@ name: conda-store
 channels:
  - conda-forge
 dependencies:
- - conda-store-server=={{ conda_store_version }}
- - pymysql==1.0.2
- - pydantic<2.0
+  - python ==3.10
+  - conda-store-server=={{ conda_store_version }}
+  # conda environment builds
+  - conda ==23.5.2
+  - python-docker
+  - conda-docker >= 0.1.2
+  - conda-pack
+  - conda-lock >=1.0.5
+  - conda-package-handling
+  - conda-package-streaming
+  # web server
+  - celery
+  - flower
+  - redis-py
+  - sqlalchemy<=1.4.47
+  - alembic
+  - psycopg2
+  - pymysql
+  - requests
+  - pyyaml
+  - uvicorn
+  - fastapi
+  - pydantic < 2.0
+  - traitlets
+  - yarl
+  - pyjwt
+  - filelock
+  - itsdangerous
+  - jinja2
+  - python-multipart
+  # artifact storage
+  - minio
+  # installer
+  - constructor
diff --git a/roles/jupyterhub/defaults/main.yml b/roles/jupyterhub/defaults/main.yml
@@ -9,7 +9,7 @@ jupyterhub_proxy_auth_token: "m8dfdKvyVJ0bWTNpbVCQyoCt"
 cdsdashboards_enabled: true
 
 jupyterhub_enabled: false
-jupyterhub_version: "==2.3.0"
+jupyterhub_version: "==4.0.2"
 jupyterhub_port: "15001"
 jupyterhub_base_url: "/"
 jupyterhub_hub_environment: "environments/jupyterhub.yaml"
@@ -21,15 +21,15 @@ jupyterhub_qhub_options_form: true
 
 jupyterhub_config:
   spawner:
-    start_timeout: 60  # seconds
+    start_timeout: 180 # seconds
 
 jupyterhub_services:
   dask_gateway: CStgn1NN8DogQR1KajuoQfye1qNRqx6zsh
 
 jupyterhub_theme:
   template_vars:
-    hub_title: "This is QHub HPC"
-    hub_subtitle: "your scalable open source data science laboratory."
+    hub_title: "This is Nebari Slurm"
+    hub_subtitle: "Your scalable open source data science laboratory."
     welcome: "have fun."
     logo: "/hub/custom/images/jupyter_qhub_logo.svg"
     primary_color: '#4f4173'
@@ -49,7 +49,7 @@ jupyterhub_additional_config: {}
 
 idle_culler:
   enabled: true
-  timeout: 86400   # 1 day
+  timeout: 86400 # 1 day
   cull_every: 3600 # 1 hour
 
 # role: miniforge

diff --git a/roles/jupyterhub/templates/environments/dashboards.yaml b/roles/jupyterhub/templates/environments/dashboards.yaml
@@ -5,9 +5,9 @@ dependencies:
   - pip
   - nb_conda_kernels
   # required to launch jupterlab from jupyterhub
-  - jupyterhub==2.3.0
+  - jupyterhub{{ jupyterhub_version }}
   # jupyterhub menu https://github.com/jupyterlab/jupyterlab/issues/9428
-  - jupyterlab>=3.4.0
+  - jupyterlab >=4
   - ipywidgets>=7.6.0
   - ipyparallel
   # dashboards (versions specified to narrow conda solve space)
@@ -16,6 +16,6 @@ dependencies:
   - voila >= 0.2.7
   - streamlit >= 0.76
   - dash >= 1.19
-  - cdsdashboards-singleuser==0.6.3
   - pip:
-      - batchspawner==1.1.0
+      - git+https://github.com/jupyterhub/batchspawner.git
+      - jhub-apps==2024.2.1rc1
diff --git a/roles/jupyterhub/templates/environments/jupyterhub.yaml b/roles/jupyterhub/templates/environments/jupyterhub.yaml
@@ -4,16 +4,16 @@ channels:
 dependencies:
   - pip==21.1.2
   - jupyterhub{{ jupyterhub_version }}
-  - jupyterhub-kubespawner==1.1.0
-  - oauthenticator==14.1.0
+  - jupyterhub-kubespawner==4.2.0
+  - oauthenticator==15.1.0
   - escapism==1.0.1
-  - cdsdashboards==0.6.3
-  - jupyterhub-idle-culler==1.0
+  - jupyterhub-idle-culler==1.2.1
+  - sqlalchemy==1.4.46
   - pip:
       - nebari_jupyterhub_theme==2023.4.1
-      - sqlalchemy==1.4.4
       - python-keycloak==0.26.1
-      - jupyterhub-traefik-proxy==0.3.0
-      - batchspawner==1.1.0
+      - jupyterhub-traefik-proxy==1.1.0
       # jupyterhub-ssh has not made a release yet
       - git+https://github.com/yuvipanda/jupyterhub-ssh.git
+      - git+https://github.com/jupyterhub/batchspawner.git
+      - jhub-apps==2024.2.1rc1
diff --git a/roles/jupyterhub/templates/environments/jupyterlab.yaml b/roles/jupyterhub/templates/environments/jupyterlab.yaml
@@ -2,34 +2,32 @@ name: jupyterlab
 channels:
   - conda-forge
 dependencies:
-# general
   - pip
   # jupyterhub/jupyterlab
   - nb_conda_kernels
   - ipython > 7
   - jupyter-server-proxy
-  - jupyter_server==1.17.1
-  - jupyterlab==3.4.0
+  - jupyter_server >=2.4.0
+  - jupyterlab >=4
   - jupyter_client
   - jupyter_console
-  - jupyterhub==2.3.0
+  - jupyterhub{{ jupyterhub_version }}
   - nbconvert
   - nbval
-  - retrolab
 
   # jupyterlab extensions
   - dask_labextension >= 5.3.0
   - jupyterlab-git >=0.30.0
   - sidecar >=0.5.0
-  - ipywidgets ==7.7.1
+  - ipywidgets >= 8.0.0
   - ipyleaflet >=0.13.5
   - pyviz_comms >=2.0.1
   - jupyter-resource-usage >=0.6.0
   - nbgitpuller
   - jupyterlab_code_formatter
-
-  # cds dashboards
-  - cdsdashboards-singleuser==0.6.3
+  - jupyterlab-spellchecker >= 0.7.3
+  - jupyterlab-pioneer
+  - jupyter-ai
   - jhsingle-native-proxy==0.8.0
 
   # viz tools
@@ -50,3 +48,5 @@ dependencies:
   - pip:
       # vscode jupyterlab launcher
       - git+https://github.com/betatim/vscode-binder
+      - argo-jupyter-scheduler==2024.1.3
+      - jhub-apps==2024.2.1rc1
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,6 +23,7 @@ @@
          - cifs
          - nfs
          - mysql
+         - postgresql
          - minio
          - backups
          - traefik
@@ Expand Down @@