Allow to deploy WT flavor using local data catalog

Makefile

@@ -1,8 +1,8 @@
-.PHONY: clean dirs dev images gwvolman_src wholetale_src dms_src home_src sources \
+.PHONY: clean dirs dev images gwvolman_src wholetale_src dms_src home_src sources_wt \
 	rebuild_dashboard watch_dashboard \
 	restart_worker restart_girder globus_handler_src status update_src
 
-SUBDIRS = src volumes/ps volumes/workspaces volumes/homes volumes/base volumes/versions volumes/runs volumes/licenses volumes/mountpoints volumes/tmp
+SUBDIRS = src volumes/ps volumes/workspaces volumes/homes volumes/base volumes/versions volumes/runs volumes/licenses volumes/mountpoints volumes/tmp volumes/minio
 TAG = latest
 MEM_LIMIT = 2048
 NODE = node --max_old_space_size=${MEM_LIMIT}
@@ -20,6 +20,15 @@ images:
 	docker pull wholetale/repo2docker_wholetale:$(TAG)
 	docker pull wholetale/ngx-dashboard:$(TAG)
 
+src/sem_viewer:
+	git clone https://github.com/htmdec/sem_viewer src/sem_viewer
+
+src/table_view:
+	git clone https://github.com/htmdec/table_view src/table_view
+
+src/synced_folders:
+	git clone https://github.com/whole-tale/synced_folders src/synced_folders
+
 src/girderfs:
 	git clone https://github.com/whole-tale/girderfs src/girderfs
 
@@ -47,14 +56,14 @@ src/globus_handler:
 src/ngx-dashboard:
 	git clone https://github.com/whole-tale/ngx-dashboard src/ngx-dashboard
 
-sources: src src/gwvolman src/wholetale src/wt_data_manager src/wt_home_dir src/globus_handler src/girderfs src/ngx-dashboard src/virtual_resources src/wt_versioning
+sources_wt: src src/gwvolman src/wholetale src/wt_data_manager src/wt_home_dir src/globus_handler src/girderfs src/ngx-dashboard src/virtual_resources src/wt_versioning src/sem_viewer src/table_view src/synced_folders
 
 dirs: $(SUBDIRS)
 
 $(SUBDIRS):
 	@sudo mkdir -p $@
 
-services: dirs sources
+services: dirs sources_wt
 
 dev: services
 	. ./.env && docker stack config --compose-file docker-stack.yml | docker stack deploy --compose-file - wt
@@ -65,8 +74,17 @@ dev: services
 	    cid=$$(docker ps --filter=name=wt_girder -q) ; \
 	done; \
 	true
-	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install plugin plugins/wt_data_manager plugins/wholetale plugins/wt_home_dir plugins/globus_handler plugins/virtual_resources plugins/wt_versioning
-	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install web --dev --plugins=oauth,gravatar,jobs,worker,wt_data_manager,wholetale,wt_home_dir,globus_handler
+	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install plugin \
+		plugins/wt_data_manager \
+		plugins/wholetale \
+		plugins/wt_home_dir \
+		plugins/globus_handler \
+		plugins/virtual_resources \
+		plugins/wt_versioning \
+		plugins/sem_viewer \
+		plugins/table_view \
+		plugins/synced_folders
+	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install web --dev --plugins=oauth,gravatar,jobs,worker,wt_data_manager,wholetale,wt_home_dir,globus_handler,sem_viewer,table_view,synced_folders
 	docker exec --user=root -ti $$(docker ps --filter=name=wt_girder -q) pip install -r /gwvolman/requirements.txt -e /gwvolman
 	docker exec --user=root -ti $$(docker ps --filter=name=wt_girder -q) pip install -e /girderfs
 	./setup_girder.py

README.md

@@ -91,6 +91,14 @@ Run:
 make dev
 ```
 
+or
+
+```
+DATACAT=1 make dev
+```
+
+for WholeTale flavor operating on local data.
+
 To confirm things are working, all `REPLICAS` should show `1/1`
 ```
 $ docker service ls

docker-stack.yml

@@ -21,6 +21,33 @@ services:
       - GODADDY_API_KEY=$WT_GODADDY_API_KEY
       - GODADDY_API_SECRET=$WT_GODADDY_API_SECRET
 
+  minio:
+    image: minio/minio
+    env_file: ./minio.env
+    ports:
+      - 9000:9000
+    volumes:
+      - ./volumes/minio:/data
+    networks:
+      - traefik-net
+    command: minio server /data --console-address ":9001"
+    deploy:
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=wt_traefik-net"
+        - "traefik.http.routers.minio.rule=Host(`minio.${domain}`)"
+        - "traefik.http.routers.minio.entrypoints=websecure"
+        - "traefik.http.routers.minio.tls=true"
+        - "traefik.http.routers.minio.service=minio"
+        - "traefik.http.services.minio.loadbalancer.server.port=9000"
+        - "traefik.http.services.minio.loadbalancer.passhostheader=true"
+        - "traefik.http.routers.minio-console.rule=Host(`minio-console.${domain}`)"
+        - "traefik.http.routers.minio-console.entrypoints=websecure"
+        - "traefik.http.routers.minio-console.tls=true"
+        - "traefik.http.routers.minio-console.service=minio-console"
+        - "traefik.http.services.minio-console.loadbalancer.server.port=9001"
+        - "traefik.http.services.minio-console.loadbalancer.passhostheader=true"
+
   mongo:
     image: mongo:3.6
     networks:
@@ -40,9 +67,11 @@ services:
       - celery
       - mongo
     volumes:
-      - ./volumes:/tmp/data
+      - ./volumes:/srv/data
       - ./girder/girder.local.cfg:/girder/girder/conf/girder.local.cfg
       - ./src/wholetale:/girder/plugins/wholetale
+      - ./src/synced_folders:/girder/plugins/synced_folders
+      - ./src/sem_viewer:/girder/plugins/sem_viewer
       - ./src/wt_data_manager:/girder/plugins/wt_data_manager
       - ./src/wt_home_dir:/girder/plugins/wt_home_dir
       - ./src/wt_versioning:/girder/plugins/wt_versioning
@@ -56,11 +85,12 @@ services:
       - DOMAIN=${domain}
       - DASHBOARD_URL=https://dashboard.${domain}
       - GOSU_USER=girder:girder:${docker_group}
-      - GOSU_CHOWN=/tmp/data
+      - GOSU_CHOWN=/srv/data
       - DATAONE_URL=https://cn-stage-2.test.dataone.org/cn
       - HOSTDIR=/
     extra_hosts:
       - "images.${domain}:host-gateway"
+      - "minio.${domain}:host-gateway"
     deploy:
       replicas: 1
       labels:

setup_girder.py

@@ -51,7 +51,7 @@ def final_msg():
     params={
         "type": 0,
         "name": "Base",
-        "root": "/tmp/data/base",
+        "root": "/srv/data/base",
     },
 )
 
@@ -68,6 +68,12 @@ def final_msg():
     "wt_home_dir",
     "wt_versioning",
 ]
+if os.environ.get("DATACAT"):
+    plugins += [
+        "sem_viewer",
+        "table_view",
+        "synced_folders",
+    ]
 r = requests.put(
     api_url + "/system/plugins",
     headers=headers,
@@ -132,12 +138,18 @@ def final_msg():
         "key": "wholetale.zenodo_extra_hosts",
         "value": ["https://sandbox.zenodo.org/record/"]
     },
-    {"key": "dm.private_storage_path", "value": "/tmp/data/ps"},
-    {"key": "wthome.homedir_root", "value": "/tmp/data/homes"},
-    {"key": "wthome.taledir_root", "value": "/tmp/data/workspaces"},
-    {"key": "wtversioning.runs_root", "value": "/tmp/data/runs"},
-    {"key": "wtversioning.versions_root", "value": "/tmp/data/versions"},
+    {"key": "dm.private_storage_path", "value": "/srv/data/ps"},
+    {"key": "wthome.homedir_root", "value": "/srv/data/homes"},
+    {"key": "wthome.taledir_root", "value": "/srv/data/workspaces"},
+    {"key": "wtversioning.runs_root", "value": "/srv/data/runs"},
+    {"key": "wtversioning.versions_root", "value": "/srv/data/versions"},
 ]
+if os.environ.get("DATACAT"):
+    settings += [
+        {"key": "wholetale.dashboard_link_title", "value": "Tale Dashboard"},
+        {"key": "wholetale.catalog_link_title", "value": "Data Catalog"},
+        {"key": "wholetale.enable_data_catalog", "value": True},
+    ]
 
 r = requests.put(
     api_url + "/system/setting", headers=headers, params={"list": json.dumps(settings)}