ValidationFrameworkUse_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Installing some Python packages needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "!pip install -q sciunit\n",
    "!pip install -q neo\n",
    "!pip install -q elephant\n",
    "!pip install -q gitpython\n",
    "!pip install -q efel\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Installing 'morphounit' and the Python-client library for the Validation Service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# os.chdir(\"/home/jupyter\")\n",
    "# !rm -rf morphounit hbp-validation-client hippocampus_CA1_pyramidal\n",
    "\n",
    "# !git clone https://github.com/pedroernesto/morphounit\n",
    "# !git clone https://github.com/apdavison/hbp-validation-client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "!pip install -U git+https://github.com/appukuttan-shailesh/hbp-validation-client.git\n",
    "!pip install --quiet fpdf PyPDF2 tabulate\n",
    "!pip install ./bbp-services-0.0.15/ ./task-types-0.0.10/ hbp_service_client\n",
    "!pip install ./morphounit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download the models directory, with the corresponding simulation code and some related Python classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !git clone https://github.com/apdavison/hippocampus_CA1_pyramidal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os;\n",
    "os.chdir(\"./models_to_test/VF_test_models\")\n",
    "\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import models, inspect\n",
    "# !pip install requests\n",
    "!ls -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import models\n",
    "print \"Models available: \\n\"\n",
    "cls_members = inspect.getmembers(models, inspect.isclass)\n",
    "\n",
    "for cls_item in cls_members:\n",
    "    if cls_item[1].__module__ == \"models\":\n",
    "        print cls_item[0]\n",
    "\n",
    "print \"\\n\"\n",
    "# Load the model\n",
    "modelName = raw_input(\"Please enter the NAME of the model to be validated: \")\n",
    "model = getattr(models,modelName)()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the test. \n",
    "*Note that you should use your HBP username at the penultimate line "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tests_url = \"https://validation.brainsimulation.eu/tests/\"\n",
    "import requests\n",
    "\n",
    "# List available tests\n",
    "print \"Validation tests available: \\n\"\n",
    "json_list = requests.get(tests_url).json()\n",
    "\n",
    "for list_item in json_list:\n",
    "    path_parts = list_item[\"code\"][\"path\"].split(\".\")\n",
    "    test_number = list_item[\"resource_uri\"][7]\n",
    "    test_name = path_parts[-1]\n",
    "    print \"({}) {}\".format(test_number, test_name)\n",
    "\n",
    "print \"\\n\"\n",
    "# Choose the test\n",
    "testNumber = raw_input(\"Please enter the NUMBER of the test to be run: \")\n",
    "test_uri = tests_url + testNumber\n",
    "\n",
    "# Load the test\n",
    "from hbp_validation_framework import ValidationTestLibrary\n",
    "\n",
    "# test_library = ValidationTestLibrary() # default url for HBP service\n",
    "print \"Please enter your HBP password: \"\n",
    "test_library = ValidationTestLibrary(username=\"pedroernesto\")\n",
    "\n",
    "test = test_library.get_validation_test(test_uri)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = test.judge(model, deep_error=True)\n",
    "\n",
    "print \"----------------------------------------------\"\n",
    "print \"Score: \", score\n",
    "if \"figures\" in score.related_data:\n",
    "    print \"Output files: \"\n",
    "    for item in score.related_data[\"figures\"]:\n",
    "        print item\n",
    "print \"----------------------------------------------\"\n",
    "score.related_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Register the test results at the Validation Service "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "from hbp_validation_framework.datastores import CollabDataStore\n",
    "# Register the result with the HBP Validation service\n",
    "collab_folder = \"{}_{}\".format(model, datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "collab_storage = CollabDataStore(collab_id=\"1771\",\n",
    "                                 base_folder=collab_folder,\n",
    "                                 auth=test_library.auth)\n",
    "# test_library.register(score)\n",
    "test_library.register(score, project=\"1771\", data_store=collab_storage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid\n",
    "uuid.UUID(model.id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.figures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "# Force matplotlib to not use any Xwindows backend.\n",
    "# matplotlib.use('Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "ind = 2\n",
    "width = 0.35\n",
    "score_lf = 2.5\n",
    "plt.bar(ind, score_lf, color='blue')\n",
    "plt.xlim(0, 4)\n",
    "plt.figlegend(ax_score, ('Z-Score',), 'upper right')\n",
    "plt.ylabel(\"Score value\")\n",
    "\n",
    "frame_bars = plt.gca()\n",
    "frame_bars.axes.get_xaxis().set_visible(False)\n",
    "\n",
    "fig_bars = plt.gcf()\n",
    "fig_bars.set_size_inches(8, 6)\n",
    "\n",
    "filename = path_test_output + 'score_plot' + '.pdf'\n",
    "print filename\n",
    "fig_bars.savefig(filename, dpi=600,)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "float(str(score).split()[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os; os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}