Merge pull request #2 from cioos-atlantic/development

Simple & Graph Authorization Model Implementation

README.rst

@@ -41,8 +41,7 @@ ckanext-vitality_prototype
 Requirements
 ------------
 
-For example, you might want to mention here which versions of CKAN this
-extension works with.
+Developed for the CIOOS CKAN fork (https://github.com/cioos-siooc/ckan).
 
 
 ------------
@@ -71,6 +70,26 @@ To install ckanext-vitality_prototype:
 
      sudo service apache2 reload
 
+5. Launch a local instance of Neo4J as a docker container::
+
+     docker run -d -p 7474:7474 -p 7687:7687 -e NEO4J_AUTH=none neo4j:3.5.8
+
+6. Add the following to the production.ini file::
+
+    ckan.vitality.neo4j.host=bolt://localhost:7687
+    ckan.vitality.neo4j.user=neo4j
+    ckan.vitality.neo4j.password=neo4j
+
+7. Seed the CKAN users into the metadata authorization model::
+
+     docker exec ckan /usr/local/bin/ckan-paster --plugin=ckanext-vitality_prototype vitality seed --config=/etc/ckan/production.ini
+
+8. Re-index the datasets in your CKAN instance, this will add them to the authorization model::
+
+     docker exec ckan /usr/local/bin/ckan-paster --plugin=ckan search-index rebuild --config=/etc/ckan/production.ini
+
+
+
 
 ---------------
 Config Settings

azure-pipelines.yml

@@ -30,7 +30,10 @@ steps:
     sudo chmod 777 -R $(ckanexts_path)$(ext_name)
 
     echo "Installing plugin into CKAN venv"
-    sudo docker exec ckan /bin/bash -c "source $(docker_venv_path) && cd $(docker_exts_path)ckanext-vitality_prototype/ && python setup.py develop"
+    sudo docker exec ckan /bin/bash -c "source $(docker_venv_path) && \
+      cd $(docker_exts_path)ckanext-vitality_prototype/ && pip install -r requirements.txt"
+    sudo docker exec ckan /bin/bash -c "source $(docker_venv_path) && \
+      cd $(docker_exts_path)ckanext-vitality_prototype/ && python setup.py develop"
 
     echo "Returning to 755 permissions for the plugin folder" 
     sudo chmod 755 -R $(ckanexts_path)$(ext_name)
@@ -50,13 +53,21 @@ steps:
   env:
     DATASTORE_READONLY_PASSWORD: $(datastore_readonly_password)
     POSTGRES_PASSWORD: $(postgres_password)
-    
+
 - script: |
+    echo "Seeding users into metadata authorization model"
+    sudo docker exec ckan /usr/local/bin/ckan-paster --plugin=ckanext-vitality_prototype \
+      vitality seed --config=/etc/ckan/production.ini
 
+  displayName: 'Seeding Authorization Model'
+
+- script: |
     echo "Re-indexing datasets..."
-    sudo docker exec ckan /usr/local/bin/ckan-paster --plugin=ckan search-index rebuild --config=/etc/ckan/production.ini
+    sudo docker exec ckan /usr/local/bin/ckan-paster --plugin=ckan search-index \
+      rebuild --config=/etc/ckan/production.ini
 
-    sudo docker exec ckan /usr/local/bin/ckan-paster --plugin=ckanext-harvest harvester reindex --config=/etc/ckan/production.ini
+    sudo docker exec ckan /usr/local/bin/ckan-paster --plugin=ckanext-harvest harvester \
+      reindex --config=/etc/ckan/production.ini
 
   displayName: 'Re-Indexing Datasets'
 

ckanext/vitality_prototype/commands/__init__.py

@@ -0,0 +1,8 @@
+try:
+    import pkg_resources
+
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/vitality_prototype/commands/vitality_model.py

@@ -0,0 +1,131 @@
+from ckanext.vitality_prototype.meta_authorize import MetaAuthorize, MetaAuthorizeType
+import logging
+import sys
+
+# CKAN interfacing imports
+from ckan import model
+from ckan.common import config
+from ckan.logic import get_action
+from ckantoolkit import CkanCommand
+
+CMD_ARG = 0
+MIN_ARGS = 1
+
+
+class VitalityModel(CkanCommand):
+    """
+    Utility commands to manage the vitality metadata authorization model.
+
+    ...
+
+    Attributes
+    ----------
+    meta_authorize: GraphMetaAuth
+        Authorization information for Neo4J.
+
+    Methods
+    ------- 
+    seed_users(context)
+        Loads all users from CKAN into the authorization model and creates one 'public' user for anonymous access
+        using the provided context object.
+
+    seed_groups(context)
+        Loads all groups from CKAN into the authorization model using the provided context object.
+
+    seed_orgs(context)
+        Loads all organizations from CKAN into the authorization model using the provided context object.
+        
+    seed(context)
+        Loads users, groups, and organizations into the authorization model using the provided context object.
+    """
+
+    # Authorization Interface
+    meta_authorize = None
+
+    # Required by CKAN Commands
+    summary = __doc__.split("\n")[0]
+    usage = __doc__
+
+    def __init__(self, name):
+        super(VitalityModel, self).__init__(name)
+
+    def command(self):
+        """
+        Ingests command line arguments 
+        """
+        self._load_config()
+
+        # Load neo4j connection parameters from config
+        # Initalize meta_authorize
+        self.meta_authorize = MetaAuthorize.create(MetaAuthorizeType.GRAPH, {
+            'host': config.get('ckan.vitality.neo4j.host', "bolt://localhost:7687"),
+            'user': config.get('ckan.vitality.neo4j.user', "neo4j"),
+            'password': config.get('ckan.vitality.neo4j.password', "password")
+        })
+
+        # We'll need a sysadmin user to perform most of the actions
+        # We will use the sysadmin site user (named as the site_id)
+
+        context = {
+            "model": model,
+            "session": model.Session,
+            "ignore_auth": True
+        }
+        self.admin_user = get_action("get_site_user")(context, {})
+
+        if len(self.args) < MIN_ARGS:
+            print("No args!")
+            self.parser.print_usage()
+            sys.exit(1)
+
+        cmd = self.args[CMD_ARG]
+        print("cmd: {}".format(cmd))
+
+        if cmd == "seed_users":
+            self.seed_users(context)
+        elif cmd == "seed_groups":
+            self.seed_groups(context)
+        elif cmd == "seed_orgs":
+            self.seed_orgs(context)
+        elif cmd == "seed":
+            self.seed_users(context)
+            self.seed_groups(context)
+            self.seed_orgs(context)
+
+
+    def seed_orgs(self, context):
+        org_list = get_action('organization_list')(context, {'all_fields':True, 'include_users':True})
+        print("Got {} organizations".format(len(org_list)))
+
+        print(org_list)
+
+        for o in org_list:
+            self.meta_authorize.add_org(o['id'],o['users'])
+            continue
+
+    def seed_groups(self, context):
+        group_list = get_action('group_list')(context,{'all_fields':True, 'include_users':True})
+        print("Got {} groups".format(len(group_list)))
+
+        for g in group_list:
+
+            self.meta_authorize.add_group(g['id'].decode('utf-8'), g['users'])
+
+        print("group_list")
+        print(group_list)
+
+
+    def seed_users(self, context):
+        user_list = get_action('user_list')(context,{})
+
+        print("Got {} users".format(len(user_list)))
+        for u in user_list:
+            print(u)
+            self.meta_authorize.add_user(u['id'].decode('utf-8'))
+
+        # Create the public user for people not logged in.
+        self.meta_authorize.add_user('public')
+
+
+    def _load_config(self):
+        super(VitalityModel, self)._load_config()

ckanext/vitality_prototype/impl/graph_meta_auth.py

@@ -0,0 +1,180 @@
+import logging
+from os import stat
+from ckanext.vitality_prototype.meta_authorize import MetaAuthorize
+from neo4j import GraphDatabase
+import uuid 
+
+log = logging.getLogger(__name__)
+
+class _GraphMetaAuth(MetaAuthorize):
+    """ Graph database authorization settings.
+    """
+
+    def __init__(self, uri, user, password):
+        self.driver = GraphDatabase.driver(uri, auth=(user, password))
+
+    def __close(self):
+        self.driver.close()
+
+    def add_org(self, org_id, users):
+        with self.driver.session() as session:
+            # Check to see if the org already exists, if so we're done as we don't want to create duplicates.
+            if session.read_transaction(self.__get_org, org_id):
+                return
+
+            session.write_transaction(self.__write_org, org_id)
+
+            for user in users:
+                session.write_transaction(self.__bind_user_to_org, org_id, user['id'])
+
+    def add_group(self, group_id, users):
+        with self.driver.session() as session:
+            # Check to see if the group already exists, if so we're done as we don't want to create duplicates.
+            if session.read_transaction(self.__get_group, group_id):
+                return
+
+            session.write_transaction(self.__write_group, group_id)
+
+            for user in users:
+                session.write_transaction(self.__bind_user_to_group, group_id, user['id'])
+
+    def add_user(self, user_id):
+        with self.driver.session() as session:
+            # Check to see if the user already exists, if so we're done as we don't want to create duplicates.
+            if session.read_transaction(self.__get_user, user_id) != None:
+                return
+
+            session.write_transaction(self.__write_user, user_id)
+
+    def get_users(self):
+        with self.driver.session() as session:
+            return session.read_transaction(self.__read_users)
+
+    def add_dataset(self, dataset_id, fields, owner_id):
+
+        with self.driver.session() as session:
+
+            # Check to see if the dataset already exists, if so we're done as we don't want to create duplicates.
+            if session.read_transaction(self.__get_dataset, dataset_id) != None:
+                return
+
+            session.write_transaction(self.__write_dataset, dataset_id)
+            session.write_transaction(self.__bind_dataset_to_org, owner_id, dataset_id)
+            # create the fields as well
+            for name,id in fields.items():
+                session.write_transaction(self.__write_metadata_field, name, id, dataset_id)
+
+    def get_metadata_fields(self, dataset_id):
+        with self.driver.session() as session:
+            return session.read_transaction(self.__read_elements, dataset_id)
+
+    def get_visible_fields(self, dataset_id, user_id):
+        with self.driver.session() as session:
+            return session.read_transaction(self.__read_visible_fields, dataset_id, user_id)
+
+    def set_visible_fields(self, dataset_id, user_id, whitelist):
+        with self.driver.session() as session:
+            session.write_transaction(self.__write_visible_fields, dataset_id, user_id, whitelist)
+
+    @staticmethod
+    def __write_visible_fields(tx, dataset_id, user_id, whitelist):      
+        for name,id in whitelist.items():
+            result = tx.run("MATCH (e:element {id:'"+id+"'}), (u:user {id:'"+user_id+"'}) CREATE (u)-[:can_see]->(e)")
+        return
+
+    @staticmethod
+    def __write_dataset(tx,id):
+        result = tx.run("CREATE (:dataset { id: '"+id+"'})")
+        return
+
+    @staticmethod
+    def __write_metadata_field(tx, name, id, dataset_id):
+        result = tx.run("MATCH (d:dataset {id:'"+dataset_id+"'}) CREATE (d)-[:has]->(:element {name:'"+name+"',id:'"+id+"'})")
+        return
+
+    @staticmethod
+    def __write_user(tx, id):
+        result = tx.run("CREATE (u:user {id:'"+id+"'})")
+        return
+
+    @staticmethod
+    def __write_org(tx, id):
+        result = tx.run("CREATE (o:organization {id:'"+id+"'})")
+        return
+
+    @staticmethod
+    def __write_group(tx, id):
+        result = tx.run("CREATE (g:group {id:'"+id+"'})")
+        return
+
+    @staticmethod
+    def __bind_user_to_org(tx, org_id, user_id):
+        result = tx.run("MATCH (o:organization {id:'"+org_id+"'}), (u:user {id:'"+user_id+"'}) CREATE (o)-[:has_member]->(u)")
+        return
+
+    @staticmethod
+    def __bind_user_to_group(tx, group_id, user_id):
+        result = tx.run("MATCH (g:group {id:'"+group_id+"'}), (u:user {id:'"+user_id+"'}) CREATE (g)-[:has_member]->(u)")
+        return
+
+    @staticmethod
+    def __bind_dataset_to_org(tx, org_id, dataset_id):
+        result = tx.run("MATCH (o:organization {id:'"+org_id+"'}), (d:dataset {id:'"+dataset_id+"'}) CREATE (o)-[:owns]->(d)")
+        return
+
+    @staticmethod
+    def __get_org(tx, id):
+        records = tx.run("MATCH (o:organization {id:'"+id+"'}) return o.id as id")      
+        for record in records:
+            return record['id']    
+        return None
+
+    @staticmethod 
+    def __get_user(tx, id):
+        records = tx.run("MATCH (u:user {id:'"+id+"'}) return u.id as id")   
+        for record in records:
+            return record['id']
+        return None
+
+    @staticmethod
+    def __get_group(tx, id):
+        records = tx.run("MATCH (g:group {id:'"+id+"'}) return g.id as id")   
+        for record in records:
+            return record['id']
+        return None
+
+    @staticmethod
+    def __get_dataset(tx, id):
+        records = tx.run("MATCH (d:dataset {id:'"+id+"'}) return d.id as id")
+        for record in records:
+            return record['id']
+        return None
+
+    @staticmethod
+    def __read_users(tx):
+        result = []
+        for record in tx.run("MATCH (u:user) RETURN u.id as id"):
+            result.append(record['id'])
+
+        return result
+
+    @staticmethod
+    def __read_visible_fields(tx, dataset_id, user_id):
+        result = []
+        for record in tx.run("MATCH (u:user {id:'"+user_id+"'})-[:can_see]->(e:element)<-[:has]-(d:dataset {id:'"+dataset_id+"'}) return e.id AS id"):
+            result.append(record['id'])
+        return result
+
+    @staticmethod
+    def __read_elements(tx, dataset_id):
+        #log.debug("Getting elements for dataset: %s", dataset_id)
+        result = {}
+        for record in tx.run("MATCH (:dataset {id:'"+dataset_id+"'})-[:has]->(e:element) RETURN e.name AS name, e.id AS id"):
+            #log.debug("record: %s", str(record))
+            result[record['name']] = record['id']
+        return result
+
+if __name__ == "__main__":
+    greeter = _GraphMetaAuth("bolt://localhost:7687", "neo4j", "password")
+    greeter.print_greeting("hello, world")
+    greeter.__close()