del2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyld import jsonld \n",
    "import cmipld\n",
    "import os, re, glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "# requests.get('https://wcrp-cmip.github.io/WCRP-universe/graph').json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# jsonld.frame('https://wcrp-cmip.github.io/WCRP-universe/realm/graph.jsonld',{\"@id\":\"atmos\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# jsonld.frame('https://wcrp-cmip.github.io/WCRP-universe/graph.jsonld',{\"@context\":\"https://wcrp-cmip.github.io/WCRP-universe/realm/atmos\",\"@id\":\"https://wcrp-cmip.github.io/WCRP-universe/realm/atmos\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wcrp-universe\n",
      "mip-variables\n",
      "cmip6plus\n",
      "cmip7\n",
      "cf\n",
      "obs4mips\n",
      "failed https://wcrp-cmip.github.io/obs4MIPs-cmor-tables-ld/\n",
      "('Could not expand input before serialization to RDF.',)\n",
      "Type: jsonld.RdfError\n",
      "Cause: ('Could not retrieve a JSON-LD document from the URL.',)\n",
      "Type: jsonld.LoadDocumentError\n",
      "Code: loading document failed\n",
      "Cause: Expecting value: line 1 column 1 (char 0)  File \"/Users/daniel.ellis/customlib/homebrew/Caskroom/mambaforge/base/lib/python3.10/site-packages/pyld/documentloader/requests.py\", line 72, in loader\n",
      "    'document': response.json()\n",
      "  File \"/Users/daniel.ellis/customlib/homebrew/Caskroom/mambaforge/base/lib/python3.10/site-packages/requests/models.py\", line 978, in json\n",
      "    raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)\n",
      "  File \"/Users/daniel.ellis/customlib/homebrew/Caskroom/mambaforge/base/lib/python3.10/site-packages/pyld/jsonld.py\", line 1219, in to_rdf\n",
      "    expanded = self.expand(input_, options)\n",
      "  File \"/Users/daniel.ellis/customlib/homebrew/Caskroom/mambaforge/base/lib/python3.10/site-packages/pyld/jsonld.py\", line 820, in expand\n",
      "    remote_doc = load_document(input_, options)\n",
      "  File \"/Users/daniel.ellis/customlib/homebrew/Caskroom/mambaforge/base/lib/python3.10/site-packages/pyld/jsonld.py\", line 6583, in load_document\n",
      "    remote_doc = options['documentLoader'](url, options)\n",
      "  File \"/Users/daniel.ellis/customlib/homebrew/Caskroom/mambaforge/base/lib/python3.10/site-packages/pyld/documentloader/requests.py\", line 100, in loader\n",
      "    raise JsonLdError(\n",
      "\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "import cmipld\n",
    "\n",
    "rdfvalues = []\n",
    "for p,v in cmipld.mapping.items():\n",
    "    print(p)\n",
    "    try:\n",
    "        rdf = jsonld.to_rdf(v+'graph.jsonld')\n",
    "        rdfvalues.extend(list(rdf.values()))\n",
    "    except Exception as e:\n",
    "        print('failed',v)\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# jsonld.to_rdf('https://wcrp-cmip.github.io/MIP-variables/graph.jsonld')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "rdfnodes = []\n",
    "rdflinks = []\n",
    "\n",
    "for g in rdfvalues:\n",
    "    for t in g:\n",
    "        if not t:\n",
    "            continue\n",
    "        s = str(t)\n",
    "        if 'literal' not in s:\n",
    "            if 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' in s:\n",
    "                # self._add_type_node(t, type_map, clink, prefix)\n",
    "                # print('node',t)\n",
    "                rdfnodes.append(t['subject']['value'])\n",
    "            else:\n",
    "                # self._add_link(t, clink) \n",
    "                rdflinks.append({'source':t['subject']['value'],'target':t['object']['value'],'type':t['predicate']['value']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rdfvalues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "Counter(rdfnodes)\n",
    "import pandas as pd\n",
    "df = pd.DataFrame(rdflinks)\n",
    "# remove the website part\n",
    "# df = df.map(lambda x: x.split('github.io/')[1])\n",
    "def prefixify(x):\n",
    "    for m in cmipld.mapping.items():\n",
    "        if x.startswith(m[1]):\n",
    "            x = x.replace(m[1],m[0]+':') \n",
    "    return x\n",
    "\n",
    "\n",
    "# df[df.source[0] != '_']\n",
    "keep = [i[1].source[0] != '_' and i[1].target[0] != '_' for i in df.iterrows()]\n",
    "\n",
    "df = df[keep]\n",
    "\n",
    "df = df.map(prefixify)\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "vscode": {
     "languageId": "javascript"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cmip6plus:experiment/activity</th>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:experiment/model_realms</th>\n",
       "      <td>200</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:experiment/parameters</th>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:experiment/parent_experiment</th>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:project/drs</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:project/index</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:project/mip_era</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:project/product</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:project/required</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:project/tables</th>\n",
       "      <td>78</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:source/license</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip7:experiment/activity_id</th>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>variables:tables/product</th>\n",
       "      <td>77</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>variables:tables/variable_entry</th>\n",
       "      <td>2049</td>\n",
       "      <td>2049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>universal:activity/source_activity</th>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>universal:institutions/location</th>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>universal:linktest/realm2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        source  target\n",
       "type                                                  \n",
       "cmip6plus:experiment/activity               28      28\n",
       "cmip6plus:experiment/model_realms          200     200\n",
       "cmip6plus:experiment/parameters             50      50\n",
       "cmip6plus:experiment/parent_experiment      50      50\n",
       "cmip6plus:project/drs                        1       1\n",
       "cmip6plus:project/index                      1       1\n",
       "cmip6plus:project/mip_era                    1       1\n",
       "cmip6plus:project/product                    1       1\n",
       "cmip6plus:project/required                   1       1\n",
       "cmip6plus:project/tables                    78      78\n",
       "cmip6plus:source/license                    12      12\n",
       "cmip7:experiment/activity_id                42      42\n",
       "variables:tables/product                77      77\n",
       "variables:tables/variable_entry       2049    2049\n",
       "universal:activity/source_activity      27      27\n",
       "universal:institutions/location         72      72\n",
       "universal:linktest/realm2                2       2"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('type').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "short = lambda x: x.split('/')[0] if 'https' not in x else x\n",
    "\n",
    "df['source_base'] = df['source'].map(short)\n",
    "df['target_base'] = df['target'].map(short)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>type</th>\n",
       "      <th>target_base</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_base</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cmip6plus:experiment</th>\n",
       "      <td>328</td>\n",
       "      <td>328</td>\n",
       "      <td>328</td>\n",
       "      <td>328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:project</th>\n",
       "      <td>83</td>\n",
       "      <td>83</td>\n",
       "      <td>83</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip6plus:source</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmip7:experiment</th>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>variables:tables</th>\n",
       "      <td>2126</td>\n",
       "      <td>2126</td>\n",
       "      <td>2126</td>\n",
       "      <td>2126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>universal:activity</th>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>universal:institutions</th>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>universal:linktest</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            source  target  type  target_base\n",
       "source_base                                                  \n",
       "cmip6plus:experiment           328     328   328          328\n",
       "cmip6plus:project               83      83    83           83\n",
       "cmip6plus:source                12      12    12           12\n",
       "cmip7:experiment                42      42    42           42\n",
       "variables:tables          2126    2126  2126         2126\n",
       "universal:activity          27      27    27           27\n",
       "universal:institutions      72      72    72           72\n",
       "universal:linktest           2       2     2            2"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('source_base').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning isolates: ['cmip6plus:sub_experiment' 'universal:consortia'\n",
      " 'universal:source_type' 'universal:resolution'\n",
      " 'universal:frequency']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'id': 'cmip6plus:experiment',\n",
       "  'in_weight': 328,\n",
       "  'out_weight': 100,\n",
       "  'total_weight': 428,\n",
       "  'category': 'cmip6plus',\n",
       "  'href': 'https://wcrp-cmip.github.io/CMIP6Plus_CVs/experiment'},\n",
       " {'id': 'cmip6plus:sub_experiment',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 0,\n",
       "  'total_weight': 0,\n",
       "  'category': 'cmip6plus',\n",
       "  'href': 'https://wcrp-cmip.github.io/CMIP6Plus_CVs/sub_experiment'},\n",
       " {'id': 'universal:consortia',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 0,\n",
       "  'total_weight': 0,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/consortia'},\n",
       " {'id': 'variables:tables',\n",
       "  'in_weight': 2126,\n",
       "  'out_weight': 78,\n",
       "  'total_weight': 2204,\n",
       "  'category': 'mip-variables',\n",
       "  'href': 'https://wcrp-cmip.github.io/MIP-variables/tables'},\n",
       " {'id': 'universal:source_type',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 0,\n",
       "  'total_weight': 0,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/source_type'},\n",
       " {'id': 'universal:license',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 12,\n",
       "  'total_weight': 12,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/license'},\n",
       " {'id': 'cmip6plus:source',\n",
       "  'in_weight': 12,\n",
       "  'out_weight': 27,\n",
       "  'total_weight': 39,\n",
       "  'category': 'cmip6plus',\n",
       "  'href': 'https://wcrp-cmip.github.io/CMIP6Plus_CVs/source'},\n",
       " {'id': 'universal:linktest',\n",
       "  'in_weight': 2,\n",
       "  'out_weight': 2,\n",
       "  'total_weight': 4,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/linktest'},\n",
       " {'id': 'universal:mip',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 1,\n",
       "  'total_weight': 1,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/mip'},\n",
       " {'id': 'universal:location',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 72,\n",
       "  'total_weight': 72,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/location'},\n",
       " {'id': 'universal:institutions',\n",
       "  'in_weight': 72,\n",
       "  'out_weight': 0,\n",
       "  'total_weight': 72,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/institutions'},\n",
       " {'id': 'universal:product',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 78,\n",
       "  'total_weight': 78,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/product'},\n",
       " {'id': 'universal:resolution',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 0,\n",
       "  'total_weight': 0,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/resolution'},\n",
       " {'id': 'universal:activity',\n",
       "  'in_weight': 27,\n",
       "  'out_weight': 28,\n",
       "  'total_weight': 55,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/activity'},\n",
       " {'id': 'variables:variables',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 2049,\n",
       "  'total_weight': 2049,\n",
       "  'category': 'mip-variables',\n",
       "  'href': 'https://wcrp-cmip.github.io/MIP-variables/variables'},\n",
       " {'id': 'cmip6plus:project',\n",
       "  'in_weight': 83,\n",
       "  'out_weight': 3,\n",
       "  'total_weight': 86,\n",
       "  'category': 'cmip6plus',\n",
       "  'href': 'https://wcrp-cmip.github.io/CMIP6Plus_CVs/project'},\n",
       " {'id': 'universal:realm',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 200,\n",
       "  'total_weight': 200,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/realm'},\n",
       " {'id': 'cmip7:experiment',\n",
       "  'in_weight': 42,\n",
       "  'out_weight': 42,\n",
       "  'total_weight': 84,\n",
       "  'category': 'cmip7',\n",
       "  'href': 'https://wcrp-cmip.github.io/CMIP7_CVs/experiment'},\n",
       " {'id': 'universal:frequency',\n",
       "  'in_weight': 0,\n",
       "  'out_weight': 0,\n",
       "  'total_weight': 0,\n",
       "  'category': 'wcrp-universe',\n",
       "  'href': 'https://wcrp-cmip.github.io/WCRP-universe/frequency'}]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes = []\n",
    "for n in set(map(short,map(prefixify,rdfnodes))):\n",
    "    in_weight = df[df['source_base']==n].shape[0]\n",
    "    out_weight = df[df['target_base']==n].shape[0]\n",
    "    url = n.split(':')\n",
    "    try:\n",
    "        url[0] = cmipld.mapping[url[0]]\n",
    "    except:\n",
    "        print('failed',url)\n",
    "    \n",
    "    if n[0]=='_':continue\n",
    "    \n",
    "    nodes.append({'id':n,'in_weight':in_weight,'out_weight':out_weight,'total_weight':in_weight+out_weight,'category':n.split(':')[0],'href':''.join(url)})\n",
    "\n",
    "nodesdf = pd.DataFrame(nodes)\n",
    "isolates = nodesdf[nodesdf['total_weight']==0]['id'].values\n",
    "print('Warning isolates:', isolates)\n",
    "\n",
    "\n",
    "\n",
    "nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'source': 'universal:institutions',\n",
       "  'target': 'universal:location',\n",
       "  'type': 'location',\n",
       "  'value': 72},\n",
       " {'source': 'universal:linktest',\n",
       "  'target': 'universal:linktest',\n",
       "  'type': 'realm2',\n",
       "  'value': 2},\n",
       " {'source': 'variables:tables',\n",
       "  'target': 'universal:product',\n",
       "  'type': 'product',\n",
       "  'value': 77},\n",
       " {'source': 'variables:tables',\n",
       "  'target': 'variables:variables',\n",
       "  'type': 'variable_entry',\n",
       "  'value': 2049},\n",
       " {'source': 'cmip6plus:source',\n",
       "  'target': 'universal:license',\n",
       "  'type': 'license',\n",
       "  'value': 12},\n",
       " {'source': 'universal:activity',\n",
       "  'target': 'cmip6plus:source',\n",
       "  'type': 'source_activity',\n",
       "  'value': 27},\n",
       " {'source': 'cmip6plus:experiment',\n",
       "  'target': 'universal:activity',\n",
       "  'type': 'activity',\n",
       "  'value': 28},\n",
       " {'source': 'cmip6plus:experiment',\n",
       "  'target': 'universal:realm',\n",
       "  'type': 'model_realms',\n",
       "  'value': 200},\n",
       " {'source': 'cmip6plus:experiment',\n",
       "  'target': 'cmip6plus:experiment',\n",
       "  'type': 'parameters',\n",
       "  'value': 50},\n",
       " {'source': 'cmip6plus:experiment',\n",
       "  'target': 'cmip6plus:experiment',\n",
       "  'type': 'parent_experiment',\n",
       "  'value': 50},\n",
       " {'source': 'cmip6plus:project',\n",
       "  'target': 'cmip6plus:project',\n",
       "  'type': 'drs',\n",
       "  'value': 1},\n",
       " {'source': 'cmip6plus:project',\n",
       "  'target': 'cmip6plus:project',\n",
       "  'type': 'index',\n",
       "  'value': 1},\n",
       " {'source': 'cmip6plus:project',\n",
       "  'target': 'universal:mip',\n",
       "  'type': 'mip_era',\n",
       "  'value': 1},\n",
       " {'source': 'cmip6plus:project',\n",
       "  'target': 'universal:product',\n",
       "  'type': 'product',\n",
       "  'value': 1},\n",
       " {'source': 'cmip6plus:project',\n",
       "  'target': 'cmip6plus:project',\n",
       "  'type': 'required',\n",
       "  'value': 1},\n",
       " {'source': 'cmip6plus:project',\n",
       "  'target': 'variables:tables',\n",
       "  'type': 'tables',\n",
       "  'value': 78},\n",
       " {'source': 'cmip7:experiment',\n",
       "  'target': 'cmip7:experiment',\n",
       "  'type': 'activity_id',\n",
       "  'value': 42}]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links = []\n",
    "for l,w in Counter(['->'.join(i) for i in df[['source_base','type','target_base']].values]).items():\n",
    "    split = l.split('->')\n",
    "    links.append({'source':split[0],'target':split[-1],'type':split[1].replace(split[0],'')[1:], 'value':w})\n",
    "    \n",
    "links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['wcrp-universe', 'mip-variables', 'cmip6plus', 'cmip7', 'cf']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prefixes = list(cmipld.mapping.keys())\n",
    "prefixes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'nodes': [{'id': 'cmip6plus:experiment',\n",
       "   'in_weight': 328,\n",
       "   'out_weight': 100,\n",
       "   'total_weight': 428,\n",
       "   'category': 'cmip6plus',\n",
       "   'href': 'https://wcrp-cmip.github.io/CMIP6Plus_CVs/experiment'},\n",
       "  {'id': 'cmip6plus:sub_experiment',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 0,\n",
       "   'total_weight': 0,\n",
       "   'category': 'cmip6plus',\n",
       "   'href': 'https://wcrp-cmip.github.io/CMIP6Plus_CVs/sub_experiment'},\n",
       "  {'id': 'universal:consortia',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 0,\n",
       "   'total_weight': 0,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/consortia'},\n",
       "  {'id': 'variables:tables',\n",
       "   'in_weight': 2126,\n",
       "   'out_weight': 78,\n",
       "   'total_weight': 2204,\n",
       "   'category': 'mip-variables',\n",
       "   'href': 'https://wcrp-cmip.github.io/MIP-variables/tables'},\n",
       "  {'id': 'universal:source_type',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 0,\n",
       "   'total_weight': 0,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/source_type'},\n",
       "  {'id': 'universal:license',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 12,\n",
       "   'total_weight': 12,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/license'},\n",
       "  {'id': 'cmip6plus:source',\n",
       "   'in_weight': 12,\n",
       "   'out_weight': 27,\n",
       "   'total_weight': 39,\n",
       "   'category': 'cmip6plus',\n",
       "   'href': 'https://wcrp-cmip.github.io/CMIP6Plus_CVs/source'},\n",
       "  {'id': 'universal:linktest',\n",
       "   'in_weight': 2,\n",
       "   'out_weight': 2,\n",
       "   'total_weight': 4,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/linktest'},\n",
       "  {'id': 'universal:mip',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 1,\n",
       "   'total_weight': 1,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/mip'},\n",
       "  {'id': 'universal:location',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 72,\n",
       "   'total_weight': 72,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/location'},\n",
       "  {'id': 'universal:institutions',\n",
       "   'in_weight': 72,\n",
       "   'out_weight': 0,\n",
       "   'total_weight': 72,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/institutions'},\n",
       "  {'id': 'universal:product',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 78,\n",
       "   'total_weight': 78,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/product'},\n",
       "  {'id': 'universal:resolution',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 0,\n",
       "   'total_weight': 0,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/resolution'},\n",
       "  {'id': 'universal:activity',\n",
       "   'in_weight': 27,\n",
       "   'out_weight': 28,\n",
       "   'total_weight': 55,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/activity'},\n",
       "  {'id': 'variables:variables',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 2049,\n",
       "   'total_weight': 2049,\n",
       "   'category': 'mip-variables',\n",
       "   'href': 'https://wcrp-cmip.github.io/MIP-variables/variables'},\n",
       "  {'id': 'cmip6plus:project',\n",
       "   'in_weight': 83,\n",
       "   'out_weight': 3,\n",
       "   'total_weight': 86,\n",
       "   'category': 'cmip6plus',\n",
       "   'href': 'https://wcrp-cmip.github.io/CMIP6Plus_CVs/project'},\n",
       "  {'id': 'universal:realm',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 200,\n",
       "   'total_weight': 200,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/realm'},\n",
       "  {'id': 'cmip7:experiment',\n",
       "   'in_weight': 42,\n",
       "   'out_weight': 42,\n",
       "   'total_weight': 84,\n",
       "   'category': 'cmip7',\n",
       "   'href': 'https://wcrp-cmip.github.io/CMIP7_CVs/experiment'},\n",
       "  {'id': 'universal:frequency',\n",
       "   'in_weight': 0,\n",
       "   'out_weight': 0,\n",
       "   'total_weight': 0,\n",
       "   'category': 'wcrp-universe',\n",
       "   'href': 'https://wcrp-cmip.github.io/WCRP-universe/frequency'}],\n",
       " 'links': [{'source': 'universal:institutions',\n",
       "   'target': 'universal:location',\n",
       "   'type': 'location',\n",
       "   'value': 72},\n",
       "  {'source': 'universal:linktest',\n",
       "   'target': 'universal:linktest',\n",
       "   'type': 'realm2',\n",
       "   'value': 2},\n",
       "  {'source': 'variables:tables',\n",
       "   'target': 'universal:product',\n",
       "   'type': 'product',\n",
       "   'value': 77},\n",
       "  {'source': 'variables:tables',\n",
       "   'target': 'variables:variables',\n",
       "   'type': 'variable_entry',\n",
       "   'value': 2049},\n",
       "  {'source': 'cmip6plus:source',\n",
       "   'target': 'universal:license',\n",
       "   'type': 'license',\n",
       "   'value': 12},\n",
       "  {'source': 'universal:activity',\n",
       "   'target': 'cmip6plus:source',\n",
       "   'type': 'source_activity',\n",
       "   'value': 27},\n",
       "  {'source': 'cmip6plus:experiment',\n",
       "   'target': 'universal:activity',\n",
       "   'type': 'activity',\n",
       "   'value': 28},\n",
       "  {'source': 'cmip6plus:experiment',\n",
       "   'target': 'universal:realm',\n",
       "   'type': 'model_realms',\n",
       "   'value': 200},\n",
       "  {'source': 'cmip6plus:experiment',\n",
       "   'target': 'cmip6plus:experiment',\n",
       "   'type': 'parameters',\n",
       "   'value': 50},\n",
       "  {'source': 'cmip6plus:experiment',\n",
       "   'target': 'cmip6plus:experiment',\n",
       "   'type': 'parent_experiment',\n",
       "   'value': 50},\n",
       "  {'source': 'cmip6plus:project',\n",
       "   'target': 'cmip6plus:project',\n",
       "   'type': 'drs',\n",
       "   'value': 1},\n",
       "  {'source': 'cmip6plus:project',\n",
       "   'target': 'cmip6plus:project',\n",
       "   'type': 'index',\n",
       "   'value': 1},\n",
       "  {'source': 'cmip6plus:project',\n",
       "   'target': 'universal:mip',\n",
       "   'type': 'mip_era',\n",
       "   'value': 1},\n",
       "  {'source': 'cmip6plus:project',\n",
       "   'target': 'universal:product',\n",
       "   'type': 'product',\n",
       "   'value': 1},\n",
       "  {'source': 'cmip6plus:project',\n",
       "   'target': 'cmip6plus:project',\n",
       "   'type': 'required',\n",
       "   'value': 1},\n",
       "  {'source': 'cmip6plus:project',\n",
       "   'target': 'variables:tables',\n",
       "   'type': 'tables',\n",
       "   'value': 78},\n",
       "  {'source': 'cmip7:experiment',\n",
       "   'target': 'cmip7:experiment',\n",
       "   'type': 'activity_id',\n",
       "   'value': 42}],\n",
       " 'prefixes': ['wcrp-universe', 'mip-variables', 'cmip6plus', 'cmip7', 'cf']}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = dict(nodes=nodes,links=links,prefixes=prefixes)\n",
    "\n",
    "file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "json.dump(file,open('network.json','w'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}