run_test_FSI_pop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Version of SciUnit is still the same, but the code has changed. Better to install a specific commit\n",
    "# !pip install -q git+git://github.com/scidash/sciunit.git@9e61c060cd8e5c6d7bce675d37c398987a9491c5\n",
    "# For sciunit 0.2.0.2\n",
    "# !pip install quantities==0.12.1\n",
    "# pip install neurom==1.14.6\n",
    "\n",
    "# !pip install -q neo\n",
    "# !pip install -q elephant\n",
    "# !pip install -q gitpython\n",
    "# !pip install -q efel\n",
    "\n",
    "!pip install scipy seaborn\n",
    "!pip install -U git+https://github.com/appukuttan-shailesh/hbp-validation-client.git\n",
    "!pip install --quiet fpdf PyPDF2 tabulate\n",
    "# First downgrade your pip version to install some packages:\n",
    "# python -m pip install pip==9.0.3\n",
    "# !pip install ./bbp-services-0.0.15/ ./task-types-0.0.10/ \n",
    "# !pip install hbp_service_client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Testing the average values from a whole POPULATION\n",
    "models_path = './models_excerpt-pop'\n",
    "neuroM_stats_file = 'fs_cells_NeuroM_MorphStats_pred.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip uninstall --yes morphounit\n",
    "# !pip install --quiet /home/pedroernesto/Documents/Project/Code/Models_Validation/Validation/ValidationTests/morphounit/\n",
    "!pip install --upgrade -q git+https://github.com/pedroernesto/morphounit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "## Testing the average values from the whole POPULATION\n",
    "from morphounit.utils import NeuroM_MorphStats_pop\n",
    "import morphounit.tests.morph_cells as cells_tests\n",
    "\n",
    "# To run the test Locally\n",
    "import os, json, copy\n",
    "test_obs_path = os.path.expanduser('~/Documents/Project/Code/Models_Validation/Validation/ValidationTests/morphounit_expData')\n",
    "# obs_file = os.path.join(obs_path, 'fs_morpho_stats_NeuroM.json')\n",
    "obs_file = os.path.join(test_obs_path, 'fs_morpho_stats_NeuroM_NO_Field_Diam_test.json')\n",
    "with open(obs_file, 'r+') as fp:\n",
    "    raw_observation = json.load(fp)\n",
    "\n",
    "morph_path = models_path\n",
    "model_name = 'population_'+os.path.basename(morph_path) # Cells-dir\n",
    "morph_model = NeuroM_MorphStats_pop(model_name=model_name, morph_path=morph_path, \n",
    "                                neuroM_pred_file=neuroM_stats_file)\n",
    "\n",
    "print('Testing population directory ----> ', models_path, '\\n\\n')\n",
    "\n",
    "test = cells_tests.NeuroM_MorphStats_pop_Test(observation=copy.deepcopy(raw_observation),\n",
    "                                              base_directory=morph_model.morph_stats_output)\n",
    "\n",
    "# test.generate_prediction(model=morph_model)import morphounit.tests.morph_cells as cells_tests\n",
    "\n",
    "score = test.judge(morph_model, deep_error=True)\n",
    "print(\"Score: \", score.score)\n",
    "print(\"---------------------------------------------- \\n\")\n",
    "\n",
    "if \"figures\" in score.related_data:\n",
    "    print(\"Output files: \")\n",
    "    for item in score.related_data[\"figures\"]:\n",
    "        print(item)\n",
    "print(\"----------------------------------------------\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# On the Collab\n",
    "!pip install --upgrade -q hbp_validation_framework\n",
    "\n",
    "# First login in into the production version of the VF\n",
    "from hbp_validation_framework import utils, TestLibrary, ModelCatalog\n",
    "HBP_USERNAME = \"pedroernesto\"\n",
    "testLibrary = TestLibrary(username=HBP_USERNAME, environment=\"production\")\n",
    "# model_catalog = ModelCatalog.from_existing(testLibrary)\n",
    "\n",
    "# model_catalog=ModelCatalog(username=HBP_USERNAME)\n",
    "\n",
    "from datetime import datetime\n",
    "morph_pop_path = models_path\n",
    "model_pop_name = 'population_'+os.path.basename(morph_path) # Cells-dir\n",
    "morph_model = NeuroM_MorphStats_pop(model_name=model_pop_name, morph_path=morph_pop_path, \n",
    "                                    neuroM_pred_file=neuroM_stats_file)\n",
    "\n",
    "# Creating a new model instance on the Model Catalog for the chosen \n",
    "# cells collection that passed the hard-constraints Test\n",
    "model_catalog = ModelCatalog(username=HBP_USERNAME)\n",
    "model_id = '078d19ae-8107-476e-8efe-bf7b0e0898bd'\n",
    "\n",
    "morph_model.model_uuid = model_id          # uuid for the chosen cell collection\n",
    "morph_model.model_version = datetime.now().strftime(\"%Y%m%d-%H%M%S\")+'_'+ model_pop_name # model instance\n",
    "\n",
    "model_catalog.add_model_instance(model_id=morph_model.model_uuid, alias='NeuroM-MorphStats-Pop', \n",
    "                                 version=morph_model.model_version)\n",
    "from time import sleep\n",
    "sleep(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Running the test and registering the results on the Collab with the raw functions of the Python client\n",
    "# Note: observation path is registered with Test definition on the VF!)\n",
    "\n",
    "import morphounit.tests\n",
    "test_alias = 'morph_stats_pop_Test'\n",
    "\n",
    "neuroM_stats_file = 'fs_cells_NeuroM_MorphStats_pred.json'\n",
    "\n",
    "result_uuids = []\n",
    "\n",
    "print('Testing population directory ----> ', morph_pop_path, '\\n\\n')\n",
    "collab_id=\"76278\"\n",
    "result_id, score = utils.run_test(username=HBP_USERNAME,\n",
    "                                  model=morph_model, test_alias=test_alias, test_version=\"2.0\",\n",
    "                                  storage_collab_id=collab_id, register_result=True, \n",
    "                                  client_obj=testLibrary)\n",
    "result_uuids.append(result_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_alias=\"morph_stats_pop_Test\"\n",
    "test_version=\"1.0\"\n",
    "test = testLibrary.get_validation_test(alias=test_alias, version=test_version)\n",
    "print(test.description)\n",
    "test.observation\n",
    "score = test.judge(morph_model, deep_error=True)\n",
    "\n",
    "print(\"----------------------------------------------\")\n",
    "print(\"Score: \", score.score)\n",
    "if \"figures\" in score.related_data:\n",
    "    print(\"Output files: \")\n",
    "    for item in score.related_data[\"figures\"]:\n",
    "        print(item)\n",
    "print(\"----------------------------------------------\")\n",
    "\n",
    "# Registering the results on the VF\n",
    "from hbp_validation_framework.datastores import CollabDataStore\n",
    "from datetime import datetime\n",
    "\n",
    "# collab_id=\"5165\" # Neuroinfo-Gif (dev) Collab\n",
    "collab_id=\"8123\" # Model validation suites Collab\n",
    "timestamp = datetime.now().strftime(\"%Y-%m-%d\")\n",
    "collab_folder = \"validation_results/{}/{}_{}\".format(timestamp,model_name, datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "collab_storage = CollabDataStore(collab_id=collab_id,\n",
    "                                 base_folder=collab_folder,\n",
    "                                 auth=testLibrary.auth)\n",
    "\n",
    "# HBP authentication token might expire when the test takes too long, \n",
    "# so create a new testLibrary object with a valid token\n",
    "testLibrary = TestLibrary(username=HBP_USERNAME, environment=\"production\")\n",
    "response = testLibrary.register_result(test_result=score, data_store=collab_storage, project=collab_id)\n",
    "\n",
    "\"\"\"\n",
    "result_uuids = []\n",
    "result_id = utils.run_test(hbp_username=HBP_USERNAME, environment=\"production\", \n",
    "                           model=morph_model, test_alias=test_alias, test_version=test_version,\n",
    "                           storage_collab_id=collab_id, register_result=False, client_obj=testLibrary)\n",
    "\n",
    "result_uuids.append(result_id)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_path = None\n",
    "if len(result_uuids) > 0:\n",
    "    print(\"\\nDo you wish to generate a report of the tests executed?\")\n",
    "    print(\"Enter: y/n\")\n",
    "    choice = raw_input().lower()\n",
    "    valid_choices = {\"yes\": True, \"y\": True, \"no\": False, \"n\": False}\n",
    "    if valid_choices[choice]:\n",
    "        valid_uuids, report_path = utils.generate_report(hbp_username=HBP_USERNAME, environment=\"dev\", \n",
    "                                                         result_list=result_uuids, only_combined=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print report_path\n",
    "if report_path:\n",
    "    rel_report_path = os.path.relpath(report_path)\n",
    "    from IPython.display import IFrame    \n",
    "    display(IFrame(rel_report_path, width=900, height=650))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create a new model entry in the ModelCatalog; metadata to be used for registering model\n",
    "!pip install --upgrade -q hbp_validation_framework\n",
    "\n",
    "from hbp_validation_framework import ModelCatalog, TestLibrary\n",
    "HBP_USERNAME = \"pedroernesto\"\n",
    "testLibrary = TestLibrary(username=HBP_USERNAME)\n",
    "model_catalog = ModelCatalog.from_existing(testLibrary)\n",
    "\n",
    "model_uuid = model_catalog.register_model(app_id=\"395557\", name=\"Population of fs_morphologies\",\n",
    "                alias=\"NeuroM-MorphStats-Pop\", author=\"Alexander Kozlov\", organization=\"KTH-UNIC\",\n",
    "                private=False, cell_type=\"Fast spiking interneuron\", model_type=\"Other\",\n",
    "                brain_region=\"Striatum\", species=\"Mouse (Mus musculus)\",\n",
    "                description=\"This model is being used to demonstrate use of MorphoUnit validation test suite!\",\n",
    "                instances=[{\"source\":\"http://collab.link/8123/usecases/models/basal_morphology/fs\",\n",
    "                            \"version\":\"1.0\", \"parameters\":\"\"}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Register a new test on the test library\n",
    "name=\"Basal Ganglia FS-Population Morphology Soft Constraints\"\n",
    "alias=\"morph_stats_pop_Test\"\n",
    "version=\"1.0\"\n",
    "author=\"Pedro Garcia-Rodriguez\"\n",
    "species=\"Undefined\"\n",
    "age=\"Any\"\n",
    "brain_region=\"Striatum\"\n",
    "cell_type=\"Other\"\n",
    "data_modality=\"electron microscopy\"\n",
    "test_type=\"network structure\"\n",
    "score_type=\"Other\"\n",
    "protocol=\"Tests a set of average morphological features describing a population of digitally-reconstructed neurons\"\n",
    "data_location=\"collab://8123/test_observations/morphounit/FSI_morpho_features_NeuroM.json\"\n",
    "data_type=\"Morphology\"\n",
    "publication=\"Cutuli (2011), DOI:10.1016/j.nbd.2010.12.007; Ransome (2005), DOI:10.1016/j.neuroscience.2004.12.041; Tepper (2010), DOI:10.3389/fnana.2010.00150; Gittis (2011), DOI:10.1016/j.neuron.2011.06.035; Plenz and Kitai (1998) J Neurosci. 18(1):266-83; Kawaguchi (1990), PMID: 1698947, J Neurosci. 1990 Oct;10(10):3421-38; Fieblinger (2010), DOI: 10.1038/ncomms6316; Preston (1980), PMID: 7353139, Brain Res. 183(2):253-63; Reinius (2015), DOI: 10.3389/fnbeh.2015.00071\"\n",
    "repository=\"https://github.com/pedroernesto/morphounit.git\"\n",
    "path=\"morphounit.tests.NeuroM_MorphStats_pop_Test\"\n",
    "\n",
    "from hbp_validation_framework import TestLibrary\n",
    "HBP_USERNAME = \"pedroernesto\"\n",
    "test_library = TestLibrary(username=HBP_USERNAME, environment=\"dev\")\n",
    "\n",
    "test = test_library.add_test(name=name, alias=alias, version=version, author=author,\n",
    "                        species=species, age=age, brain_region=brain_region, cell_type=cell_type,\n",
    "                        data_modality=data_modality, test_type=test_type, score_type=score_type,\n",
    "                        data_location=data_location, data_type=data_type, publication=publication,\n",
    "                        repository=repository, path=path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}