ps-trace-ASN-cannot-reach-dest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connected\n",
      "2022-04-04 04:28 2022-04-04 10:28\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import itertools\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch.helpers import scan\n",
    "from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\n",
    "from multiprocessing import Manager\n",
    "import datetime as dt\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "import numpy as np\n",
    "\n",
    "from data_objects.NodesMetaData import NodesMetaData\n",
    "import utils.helpers as hp\n",
    "from utils.helpers import timer\n",
    "from alarms import alarms\n",
    "\n",
    "# the query\n",
    "def queryPSTrace(dt):\n",
    "    # include=[\"timestamp\",\"src\",\"dest\",\"asns\",\"src_host\",\"dest_host\",\"src_site\",\"dest_site\",\"destination_reached\",'path_complete','looping','hops']\n",
    "    include=[\"timestamp\",\"src\",\"dest\",\"asns\",\"src_host\",\"dest_host\",\"src_site\",\"dest_site\",'hops','destination_reached']\n",
    "    query = {\n",
    "      \"query\" : {\n",
    "        \"bool\" : {\n",
    "          \"must\" : [\n",
    "            {\n",
    "              \"range\" : {\n",
    "                \"timestamp\" : {\n",
    "                  \"gt\" : dt[0],\n",
    "                  \"lte\": dt[1]\n",
    "                }\n",
    "              }\n",
    "            }\n",
    "          ]\n",
    "        }\n",
    "      }\n",
    "    }\n",
    "#     print(str(query).replace(\"\\'\", \"\\\"\"))\n",
    "    try:\n",
    "        return scan_gen(scan(hp.es,index=\"ps_trace\",query=query, filter_path=['_scroll_id', '_shards', 'hits.hits._source']))\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        \n",
    "def scan_gen(scan):\n",
    "    while True:\n",
    "        try:\n",
    "            yield next(scan)['_source']\n",
    "        except:\n",
    "            break\n",
    "\n",
    "# get the data from ES\n",
    "def ps_trace(dt):\n",
    "    scan_gen = queryPSTrace(dt)\n",
    "    items = []\n",
    "\n",
    "    for meta in scan_gen:\n",
    "        items.append(meta)\n",
    "\n",
    "    return items\n",
    "\n",
    "\n",
    "# create a shared variable to store the data\n",
    "manager = Manager()\n",
    "data = manager.list()\n",
    "\n",
    "# query in chunks based on time ranges\n",
    "def getTraceData(dtRange):\n",
    "    traceData = ps_trace(dtRange)\n",
    "    if len(traceData)>0:\n",
    "        data.extend(traceData)\n",
    "\n",
    "# laod the data in parallel\n",
    "def run(dateFrom, dateTo):\n",
    "    # query the past 24 hours and split the period into 8 time ranges\n",
    "    dtList = hp.GetTimeRanges(dateFrom, dateTo, 8)\n",
    "    with ProcessPoolExecutor(max_workers=8) as pool:\n",
    "        result = pool.map(getTraceData, [[dtList[i], dtList[i+1]] for i in range(len(dtList)-1)])\n",
    "\n",
    "\n",
    "dateFrom, dateTo = hp.defaultTimeRange(6)\n",
    "print(dateFrom, dateTo)\n",
    "run(dateFrom, dateTo)\n",
    "df = pd.DataFrame(list(data))\n",
    "\n",
    "df['pair'] = df['src']+'-'+df['dest']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "696816"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: plotly in /home/petya/.local/lib/python3.8/site-packages (5.6.0)\n",
      "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from plotly) (1.14.0)\n",
      "Requirement already satisfied: tenacity>=6.2.0 in /home/petya/.local/lib/python3.8/site-packages (from plotly) (8.0.1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install ipwhois"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install plotly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Isolate only the relevant fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "relDf = df[['src', 'dest', 'asns', 'hops', 'pair', 'destination_reached']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "asn2Hop,hop2ASN = {},{}\n",
    "strange = []\n",
    "\n",
    "pch, ipl = [],[]\n",
    "relDf = df[['src', 'dest', 'asns', 'hops', 'pair', 'destination_reached']].copy()\n",
    "chunk = relDf[['asns','hops', 'pair']].values.tolist()\n",
    "\n",
    "for asns, hops, pair in chunk:\n",
    "    if len(asns)==len(hops):\n",
    "        for i in range(len(asns)):\n",
    "#             print(asns[i], hops[i])\n",
    "            if asns[i] not in asn2Hop.keys():\n",
    "                asn2Hop[asns[i]] = [hops[i]]\n",
    "            else:\n",
    "                temp = asn2Hop[asns[i]]\n",
    "                if hops[i] not in temp:\n",
    "                    temp.append(hops[i])\n",
    "                    asn2Hop[asns[i]] = temp\n",
    "\n",
    "                # print(pair, hops[i])\n",
    "            if hops[i] not in hop2ASN.keys():\n",
    "                hop2ASN[hops[i]] = [asns[i]]\n",
    "            else:\n",
    "                temp = hop2ASN[hops[i]]\n",
    "                if asns[i] not in temp:\n",
    "                    temp.append(asns[i])\n",
    "                    hop2ASN[hops[i]] = temp\n",
    "    else: \n",
    "        # print(pair, len(asns), len(hops))\n",
    "        print('this shouldn not happen')\n",
    "        strange.append([pair,asns,hops])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fix zero ASNs by picking the IP at the same position and checking for it's AS number for previous measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of repaired measures 9577\n"
     ]
    }
   ],
   "source": [
    "fix_asns = []\n",
    "\n",
    "\n",
    "relDf = df[['src', 'dest', 'asns', 'hops', 'pair', 'destination_reached']].copy()\n",
    "\n",
    "# add a flag showing if all measures for a given pair have reached the destination\n",
    "all_reachedDf = relDf.groupby('pair')[['destination_reached']].apply(lambda x: all(x.values)).to_frame().rename(columns={0:'all_dests_reached'}).reset_index()\n",
    "\n",
    "relDf = pd.merge(relDf, all_reachedDf, on='pair', how='left')\n",
    "\n",
    "relDf['asns_updated'] = None\n",
    "relDf['hops_updated'] = None\n",
    "relDf['first'] = None\n",
    "relDf['last'] = None\n",
    "\n",
    "zfix = []\n",
    "\n",
    "\n",
    "try:\n",
    "    for idx, asns, hops, s, d, destination_reached in relDf[['asns','hops','src','dest','all_dests_reached']].itertuples():\n",
    "        asns_updated  = asns.copy()\n",
    "        hops_updated = hops.copy()\n",
    "        # print(idx, asns, hops, s, d, destination_reached)\n",
    "\n",
    "        for pos, n in enumerate(asns):\n",
    "            # when AS number is 0 (unknown) get the IP at this position and find all ASNs for it\n",
    "            if n == 0:\n",
    "                # print(n)\n",
    "                ip = hops[pos]\n",
    "                asns4IP = hop2ASN[ip]\n",
    "                # print(ip,asns4IP)\n",
    "\n",
    "                if 0 in asns4IP: asns4IP.remove(0)\n",
    "                \n",
    "                asns_updated[pos] = n\n",
    "                if len(asns4IP) == 1:\n",
    "                    asns_updated[pos] = asns4IP[0]\n",
    "                    if idx not in zfix: zfix.append(idx)\n",
    "                elif len(asns4IP) > 1:\n",
    "                    for ip in asns4IP:\n",
    "                        if len(asns4IP)==2:\n",
    "                            alt = [el for el in asns4IP if el != n]\n",
    "                            # alternative_asns_dict[asn] = alt[0]\n",
    "                            asns_updated[pos] = alt[0]\n",
    "                            if idx not in zfix: zfix.append(idx)\n",
    "                            # print(n,'.............', alt[0])\n",
    "                        elif len(hop2ASN[ip])>2:\n",
    "                            print('There are more possibilities ........................', idx, asns, pos, alt)\n",
    "\n",
    "        relDf.at[idx, 'asns_updated'] = asns_updated\n",
    "        relDf.at[idx, 'hops_updated'] = hops_updated\n",
    "        \n",
    "        if len(asns_updated)>2:\n",
    "            relDf.at[idx, 'first'] = asns_updated[0]\n",
    "            relDf.at[idx, 'last'] = asns_updated[-1]\n",
    "except Exception as e:\n",
    "    print('error', e)\n",
    "    print(idx, asns, hops, s, d, destination_reached)\n",
    "    \n",
    "print(f'Number of repaired measures {len(zfix)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Find the alternativ AS numbers based on the IP addresses at the same position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(38, [40387]),\n",
       " (62, [32361, 20080]),\n",
       " (231, [237]),\n",
       " (237, [231, 36375]),\n",
       " (680, [58069]),\n",
       " (2907, [7660]),\n",
       " (7660, [2907]),\n",
       " (10680, [19782]),\n",
       " (16905, [22645]),\n",
       " (19782, [10680]),\n",
       " (20080, [62]),\n",
       " (22645, [16905]),\n",
       " (24489, [24490]),\n",
       " (24490, [24489]),\n",
       " (32361, [62]),\n",
       " (34878, [58069]),\n",
       " (36375, [237]),\n",
       " (40387, [38]),\n",
       " (58069, [34878, 680])]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def getAltASNs():\n",
    "    alt_dict = {}\n",
    "    for asn, l in asn2Hop.items():\n",
    "        if asn != 0:\n",
    "            for ip in l:\n",
    "\n",
    "                others = hop2ASN[ip]\n",
    "                if 0 in others:\n",
    "                    others.remove(0)\n",
    "\n",
    "                if len(others)==2:\n",
    "                    # if 0 not in hop2ASN[ip]:\n",
    "                    alt = [el for el in hop2ASN[ip] if el != asn]\n",
    "                    if len(alt)>1:\n",
    "                        print(alt)\n",
    "\n",
    "                    altASN = [alt[0]]\n",
    "                    if asn in alt_dict.keys():\n",
    "                        temp = alt_dict[asn]\n",
    "                        if alt[0] not in temp:\n",
    "                            temp.append(alt[0])\n",
    "                        altASN = temp\n",
    "\n",
    "                    alt_dict[asn] = altASN\n",
    "                elif len(others)>2:\n",
    "                    print(asn, others)\n",
    "                    print('There are more possibilities ........................')\n",
    "    return alt_dict\n",
    "            \n",
    "alternative_asns_dict = getAltASNs()\n",
    "sorted(alternative_asns_dict.items())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Find the alternative AS numbers of the alternatives. Add some manually "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(38, [40387]),\n",
       " (62, [20080, 32361]),\n",
       " (231, [237, 36375]),\n",
       " (237, [36375, 231]),\n",
       " (513, [20969]),\n",
       " (680, [58069, 34878]),\n",
       " (2907, [7660]),\n",
       " (7660, [2907]),\n",
       " (10680, [19782]),\n",
       " (16905, [22645]),\n",
       " (19782, [10680]),\n",
       " (20080, [32361, 62]),\n",
       " (20969, [513]),\n",
       " (22645, [16905]),\n",
       " (24489, [24490]),\n",
       " (24490, [24489]),\n",
       " (32361, [20080, 62]),\n",
       " (34878, [680, 58069]),\n",
       " (36375, [231, 237]),\n",
       " (40387, [38]),\n",
       " (58069, [680, 34878])]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alt_dict = alternative_asns_dict.copy()\n",
    "altsOfAlts = {}\n",
    "allVals = []\n",
    "for key, vals in alt_dict.items():\n",
    "    allVals = vals\n",
    "    # print(key, vals)\n",
    "    for aslist in alt_dict.values():\n",
    "        temp=[]\n",
    "        if key in aslist and len(aslist)>1:\n",
    "            temp = aslist.copy()\n",
    "            allVals.extend(list(set(temp)))\n",
    "            \n",
    "    allVals = list(set(allVals))\n",
    "    if key in allVals:\n",
    "        allVals.remove(key)\n",
    "    altsOfAlts[key] = allVals\n",
    "\n",
    "def addASNsManualy(asn1, asn2):\n",
    "    if asn1 not in altsOfAlts.keys():\n",
    "        altsOfAlts[asn1] = [asn2]\n",
    "    else:\n",
    "        temp = altsOfAlts[asn1]\n",
    "        temp.extend(asn2)\n",
    "        altsOfAlts[asn1]\n",
    "\n",
    "addASNsManualy(513, 20969)\n",
    "addASNsManualy(20969, 513)\n",
    "\n",
    "sorted(altsOfAlts.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reached</th>\n",
       "      <th>not_reached</th>\n",
       "      <th>total</th>\n",
       "      <th>first</th>\n",
       "      <th>middle</th>\n",
       "      <th>last</th>\n",
       "      <th>pair</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2611</th>\n",
       "      <td>0</td>\n",
       "      <td>4637</td>\n",
       "      <td>4637</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4633</td>\n",
       "      <td>[2607:f388:101c:1000::442-2001:6a8:1080::16, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>0</td>\n",
       "      <td>1790</td>\n",
       "      <td>1790</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1790</td>\n",
       "      <td>[192.108.47.6-192.114.101.125, 193.48.99.76-19...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>0</td>\n",
       "      <td>3536</td>\n",
       "      <td>3536</td>\n",
       "      <td>0</td>\n",
       "      <td>142</td>\n",
       "      <td>3394</td>\n",
       "      <td>[134.75.125.241-132.206.245.253, 193.48.83.97-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2259</th>\n",
       "      <td>0</td>\n",
       "      <td>547</td>\n",
       "      <td>547</td>\n",
       "      <td>0</td>\n",
       "      <td>544</td>\n",
       "      <td>3</td>\n",
       "      <td>[134.158.84.140-134.158.150.53, 193.171.188.21...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3257</th>\n",
       "      <td>0</td>\n",
       "      <td>351</td>\n",
       "      <td>351</td>\n",
       "      <td>0</td>\n",
       "      <td>351</td>\n",
       "      <td>0</td>\n",
       "      <td>[129.107.255.30-132.206.245.253, 129.107.255.2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     reached not_reached total first middle  last  \\\n",
       "2611       0        4637  4637     0      4  4633   \n",
       "378        0        1790  1790     0      0  1790   \n",
       "376        0        3536  3536     0    142  3394   \n",
       "2259       0         547   547     0    544     3   \n",
       "3257       0         351   351     0    351     0   \n",
       "\n",
       "                                                   pair  \n",
       "2611  [2607:f388:101c:1000::442-2001:6a8:1080::16, 2...  \n",
       "378   [192.108.47.6-192.114.101.125, 193.48.99.76-19...  \n",
       "376   [134.75.125.241-132.206.245.253, 193.48.83.97-...  \n",
       "2259  [134.158.84.140-134.158.150.53, 193.171.188.21...  \n",
       "3257  [129.107.255.30-132.206.245.253, 129.107.255.2...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def getStats(val_list):\n",
    "    dicti = {}\n",
    "    for idx, val_list, destination_reached, pair in relDf[[val_list,'destination_reached','pair']].itertuples():\n",
    "        aslist = list(set(val_list))\n",
    "\n",
    "        if 0 in aslist:\n",
    "            aslist.remove(0)\n",
    "        # print(idx, val_list, destination_reached)\n",
    "\n",
    "        for n in aslist:\n",
    "            \n",
    "            # flag where the ASN appeared - first, last of middle\n",
    "            for i in val_list:\n",
    "                if n == val_list[0]:\n",
    "                    pos = [1,0,0]\n",
    "                elif n == val_list[-1]:\n",
    "                    pos = [0,0,1]\n",
    "                else:\n",
    "                    pos = [0,1,0]\n",
    "\n",
    "            if n not in dicti.keys():\n",
    "                dicti[n] = {'reached': 0,\n",
    "                            'not_reached':0,\n",
    "                            'total':0,\n",
    "                            'first':0,\n",
    "                            'middle':0,\n",
    "                            'last':0,\n",
    "                            'pair':[]}\n",
    "\n",
    "            if destination_reached:\n",
    "                dicti[n]['reached'] = dicti[n]['reached']+1\n",
    "            else: dicti[n]['not_reached'] = dicti[n]['not_reached']+1\n",
    "\n",
    "            dicti[n]['total'] = dicti[n]['total']+1\n",
    "            dicti[n]['first'] = dicti[n]['first']+pos[0]\n",
    "            dicti[n]['middle'] = dicti[n]['middle']+pos[1]\n",
    "            dicti[n]['last'] = dicti[n]['last']+pos[-1]\n",
    "            \n",
    "            temp = dicti[n]['pair']\n",
    "            if pair not in temp:\n",
    "                temp.append(pair)\n",
    "                dicti[n]['pair'] = temp\n",
    "                        \n",
    "    return dicti\n",
    "\n",
    "drasns = getStats('asns_updated')\n",
    "drasnsDf = pd.DataFrame(drasns).T\n",
    "display(drasnsDf[(drasnsDf['reached']==0)&(drasnsDf['total']>100)])\n",
    "\n",
    "# drhops = getStats('hops')\n",
    "# drhopsDf = pd.DataFrame(drhops).T\n",
    "# display(drhopsDf[(drhopsDf['reached']==0)&(drhopsDf['total']>100)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "3257 129.107.255.30-132.206.245.253\n",
      "[list([18515, 22645, 3257, 3257, 3257, 376, 0])\n",
      " list([18515, 22645, 3257, 3257, 3257, 376, 0])\n",
      " list([18515, 22645, 3257, 3257, 3257, 376, 0])]\n",
      "['UTA_SWT2' 'ps-bandwidth.clumeq.mcgill.ca']\n",
      "\n",
      "3257 129.107.255.29-132.206.245.252\n",
      "[list([18515, 22645, 3257, 3257, 3257, 376, 0])\n",
      " list([18515, 22645, 3257, 3257, 3257, 376, 0])\n",
      " list([18515, 22645, 3257, 3257, 3257, 376, 0])]\n",
      "['UTA_SWT2' 'ps-latency.clumeq.mcgill.ca']\n",
      "\n",
      "3257 129.107.255.29-132.206.245.253\n",
      "[list([18515, 22645, 3257, 3257, 3257, 376, 0])\n",
      " list([18515, 22645, 3257, 3257, 3257, 376, 0])\n",
      " list([18515, 22645, 3257, 3257, 3257, 376, 0])]\n",
      "['UTA_SWT2' 'ps-bandwidth.clumeq.mcgill.ca']\n",
      "\n",
      "3257 144.206.237.142-132.206.245.253\n",
      "[list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])\n",
      " list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])\n",
      " list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])]\n",
      "['RRC-KI' 'ps-bandwidth.clumeq.mcgill.ca']\n",
      "\n",
      "3257 129.107.255.30-132.206.245.252\n",
      "[list([18515, 22645, 3257, 3257, 3257, 376, 0])\n",
      " list([18515, 22645, 3257, 3257, 3257, 376, 0])\n",
      " list([18515, 22645, 3257, 3257, 3257, 376, 0])]\n",
      "['UTA_SWT2' 'ps-latency.clumeq.mcgill.ca']\n",
      "\n",
      "3257 144.206.237.141-132.206.245.252\n",
      "[list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])\n",
      " list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])\n",
      " list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])]\n",
      "['RRC-KI' 'ps-latency.clumeq.mcgill.ca']\n",
      "\n",
      "3257 159.93.225.210-132.206.245.253\n",
      "[list([2875, 0, 0, 59624, 59624, 9002, 9002, 3257, 3257, 3257, 376])\n",
      " list([2875, 0, 0, 59624, 59624, 9002, 9002, 3257, 3257, 3257, 376])\n",
      " list([2875, 0, 0, 59624, 59624, 9002, 9002, 3257, 3257, 3257, 376])]\n",
      "['JINR-LCG2' 'ps-bandwidth.clumeq.mcgill.ca']\n",
      "\n",
      "3257 144.206.237.142-132.206.245.252\n",
      "[list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])\n",
      " list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])\n",
      " list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])]\n",
      "['RRC-KI' 'ps-latency.clumeq.mcgill.ca']\n",
      "\n",
      "3257 144.206.237.141-132.206.245.253\n",
      "[list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])\n",
      " list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])\n",
      " list([59624, 59624, 9002, 9002, 3257, 3257, 3257, 376, 376])]\n",
      "['RRC-KI' 'ps-bandwidth.clumeq.mcgill.ca']\n",
      "\n",
      "3257 159.93.225.210-132.206.245.252\n",
      "[list([2875, 0, 0, 59624, 59624, 9002, 9002, 3257, 3257, 3257, 376])\n",
      " list([2875, 0, 0, 59624, 59624, 9002, 9002, 3257, 3257, 3257, 376])\n",
      " list([2875, 0, 0, 59624, 59624, 9002, 9002, 3257, 3257, 3257, 376])]\n",
      "['JINR-LCG2' 'ps-latency.clumeq.mcgill.ca']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "asn = 3257\n",
    "vals = drasns[asn]['pair']\n",
    "print(len(vals))\n",
    "for p in vals:\n",
    "    print(asn, p)\n",
    "    print(relDf[relDf['pair']==p]['asns_updated'].values[2:5])\n",
    "    print(df[df['pair']==p][['src_site', 'dest_host']].values[0])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_backup = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df_backup.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "707409 25699\n"
     ]
    }
   ],
   "source": [
    "import hashlib\n",
    "allListsRp,pairListRp = [],[]\n",
    "\n",
    "\n",
    "def hashASNs(group):\n",
    "    try:\n",
    "        hex_list = []\n",
    "        if len(group.asns_updated.values)>1:\n",
    "#             print(group.asns.values)\n",
    "            for g in group.asns_updated.values:\n",
    "                # print('---',g)\n",
    "                if g is not None and g==g:\n",
    "                    asnList = list(set(g.copy()))\n",
    "\n",
    "                if 0 in asnList: asnList.remove(0)\n",
    "                \n",
    "                hash_object = hashlib.sha1(str(asnList).encode())\n",
    "                hex_dig = hash_object.hexdigest()\n",
    "                \n",
    "                if hex_dig not in hex_list:\n",
    "                    # print(asnList,hex_dig)\n",
    "                    hex_list.append(hex_dig)\n",
    "\n",
    "                # pathDict[hex_dig] = {'src':group.name[0], 'dest':group.name[1], \n",
    "                #                      'asn':asnList, 'cnt_asn':len(asnList), 'cnt_total_measures':len(group.values)}\n",
    "                \n",
    "                    # print(group.name[0], group.name[1], asnList, len(asnList), len(group.values), hex_dig, g[0], g[-1])\n",
    "                    \n",
    "                    firsts = list(set(group['first'].values))\n",
    "                    lasts = list(set(group['last'].values))\n",
    "                    \n",
    "                    if len(g)>0:\n",
    "                        pairListRp.append([group.name[0], group.name[1], asnList, len(asnList), len(group.values), hex_dig, group.all_dests_reached.values[0], firsts, lasts])\n",
    "                    # print(asnList)\n",
    "                # print([group.name[0], group.name[1], asnList, len(asnList), len(group.values), hex_dig])\n",
    "                \n",
    "                if len(g)>0:\n",
    "                    allListsRp.append([group.name[0], group.name[1], asnList, len(asnList), len(group.values), hex_dig])\n",
    "                \n",
    "    except Exception as e:\n",
    "        print('Issue wtih:',group.name, asnList)\n",
    "        print(e)\n",
    "#         print()\n",
    "\n",
    "\n",
    "# p = '130.246.176.109-148.6.8.251'\n",
    "# relDf[relDf['pair']==p][['src','dest','asns_updated','hops_updated']].groupby(['src','dest']).apply(lambda x: hashASNs(x))\n",
    "relDf[['src','dest','asns_updated','hops', 'all_dests_reached', 'first', 'last']].groupby(['src','dest']).apply(lambda x: hashASNs(x))\n",
    "\n",
    "\n",
    "\n",
    "cleanRp = pd.DataFrame(pairListRp).rename(columns={0:'src', 1:'dest', 2:'asns_updated', 3:'cnt_asn', 4:'cnt_total_measures', 5:'hash', 6:'all_dests_reached', 7:'first', 8:'last'})\n",
    "\n",
    "\n",
    "# cleanRp = pd.DataFrame(pathDict).T.reset_index().rename(columns={'index':'hash'})\n",
    "cleanRp['pair'] = cleanRp['src']+'-'+cleanRp['dest']\n",
    "\n",
    "cleanPathsRp = pd.DataFrame(allListsRp).rename(columns={0:'src', 1:'dest', 2:'asns_updated', 3:'cnt_asn', 4:'cnt_total_measures', 5:'hash'})\n",
    "print(len(cleanPathsRp), len(cleanRp))\n",
    "\n",
    "\n",
    "pathFreq = cleanPathsRp.groupby(['src','dest'])['hash'].apply(lambda x: x.value_counts(normalize=True))\n",
    "pathFreq = pd.DataFrame(pathFreq).reset_index().rename(columns={'hash':'hash_freq','level_2':'hash'})\n",
    "\n",
    "clean = pd.merge(cleanRp, pathFreq, how=\"inner\", on=['src','dest','hash'])\n",
    "\n",
    "sub = df[['dest', 'src_site', 'src', 'dest_site', 'src_host', 'dest_host', 'pair']].drop_duplicates()\n",
    "clean = pd.merge(clean, sub, on=['pair','src', 'dest'], how='left')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3492"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freqAsBaseList = []\n",
    "\n",
    "def getBaseline(group):\n",
    "    try:\n",
    "        cnt_max = max(group.cnt_asn.values)\n",
    "        freq_max = max(group.hash_freq.values)\n",
    "\n",
    "        cnt_max_position = [pos for pos, i in enumerate(group.cnt_asn) if i==cnt_max]\n",
    "        freq_max_position = [pos for pos, i in enumerate(group.hash_freq) if i==freq_max]\n",
    "\n",
    "        # print(group.cnt_asn.values)\n",
    "        # print(cnt_max,freq_max)\n",
    "        \n",
    "        # print(cnt_max_position,freq_max_position)\n",
    "\n",
    "        if freq_max>=0.75:\n",
    "            # print('***',cnt_max, freq_max, cnt_max_position)\n",
    "            diversity = 0\n",
    "            position = freq_max_position[0]\n",
    "            for pos in freq_max_position:\n",
    "                # print(group.hash_freq.values[pos])\n",
    "                if diversity < group.cnt_asn.values[pos]:\n",
    "                    position = pos\n",
    "                    diversity = group.cnt_asn.values[pos]\n",
    "                # value, position = group.hash_freq.values[pos], pos if value < group.hash_freq.values[pos] else value,pos\n",
    "            max_position = [position]\n",
    "            freq_max = group.hash_freq.values[position]\n",
    "\n",
    "        else: \n",
    "            # print('=======',cnt_max, freq_max, cnt_max_position)\n",
    "            if len(cnt_max_position)>1:\n",
    "                path_freq = 0\n",
    "                for pos in cnt_max_position:\n",
    "                    # print(group.hash_freq.values[pos])\n",
    "                    if path_freq < group.hash_freq.values[pos]:\n",
    "                        position = pos\n",
    "                        path_freq = group.hash_freq.values[pos]\n",
    "                    # value, position = group.hash_freq.values[pos], pos if value < group.hash_freq.values[pos] else value,pos\n",
    "                # print(2)\n",
    "                max_position = [position]\n",
    "            else: \n",
    "                # print(3)\n",
    "                max_position = cnt_max_position\n",
    "                # cnt_max = group.cnt_asn.values[position]\n",
    "\n",
    "        # print(freq_max_position, group.index.values[freq_max_position[0]])\n",
    "        freqAsBaseList.append(group.index.values[max_position[0]])\n",
    "        # print()\n",
    "    except Exception as e:\n",
    "        print('Issue wtih:',group,e)\n",
    "\n",
    "    \n",
    "clean[clean['hash_freq']<1].groupby('pair').apply(lambda x: getBaseline(x))\n",
    "# clean[clean['pair']==list(diffFr)[62]].groupby('pair').apply(lambda x: getBaseline(x))\n",
    "\n",
    "# clean[clean['hash_freq']<1].head(50).groupby('pair').apply(lambda x: getMax(x))\n",
    "len(freqAsBaseList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dest</th>\n",
       "      <th>asns_updated</th>\n",
       "      <th>cnt_asn</th>\n",
       "      <th>cnt_total_measures</th>\n",
       "      <th>hash</th>\n",
       "      <th>all_dests_reached</th>\n",
       "      <th>first</th>\n",
       "      <th>last</th>\n",
       "      <th>pair</th>\n",
       "      <th>hash_freq</th>\n",
       "      <th>src_site</th>\n",
       "      <th>dest_site</th>\n",
       "      <th>src_host</th>\n",
       "      <th>dest_host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[291, 20965, 293, 789, 2200, 59]</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>031fa51279113d0e60de6a10bb4b8181245a1bbc</td>\n",
       "      <td>False</td>\n",
       "      <td>[7896, 59, 229]</td>\n",
       "      <td>[786, 43115, 789]</td>\n",
       "      <td>-</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>hcc-ps01.unl.edu</td>\n",
       "      <td>gridpp-ps-band.ecdf.ed.ac.uk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[291, 20965, 293, 789, 2200, 59]</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>031fa51279113d0e60de6a10bb4b8181245a1bbc</td>\n",
       "      <td>False</td>\n",
       "      <td>[7896, 59, 229]</td>\n",
       "      <td>[786, 43115, 789]</td>\n",
       "      <td>-</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>hcc-ps02.unl.edu</td>\n",
       "      <td>picperfsonar-latency.pic.es</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[291, 20965, 293, 789, 2200, 59]</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>031fa51279113d0e60de6a10bb4b8181245a1bbc</td>\n",
       "      <td>False</td>\n",
       "      <td>[7896, 59, 229]</td>\n",
       "      <td>[786, 43115, 789]</td>\n",
       "      <td>-</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>psmsu02.aglt2.org</td>\n",
       "      <td>lpnhe-psl.in2p3.fr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[7896, 786, 20965]</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>f070524c1d64bdd9e153f06b68e2b05c87fc9cd8</td>\n",
       "      <td>False</td>\n",
       "      <td>[7896, 59, 229]</td>\n",
       "      <td>[786, 43115, 789]</td>\n",
       "      <td>-</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>perfsonar01.hep.wisc.edu</td>\n",
       "      <td>llrpsonar2.in2p3.fr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[7896, 786, 20965]</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>f070524c1d64bdd9e153f06b68e2b05c87fc9cd8</td>\n",
       "      <td>False</td>\n",
       "      <td>[7896, 59, 229]</td>\n",
       "      <td>[786, 43115, 789]</td>\n",
       "      <td>-</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>hcc-ps01.unl.edu</td>\n",
       "      <td>gridpp-ps-band.ecdf.ed.ac.uk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26103</th>\n",
       "      <td>85.122.31.119</td>\n",
       "      <td>81.180.86.62</td>\n",
       "      <td>[2614]</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>fa3ba04018aa660e475494097c242d7ecf66ff56</td>\n",
       "      <td>True</td>\n",
       "      <td>[12675, 2614]</td>\n",
       "      <td>[2614]</td>\n",
       "      <td>85.122.31.119-81.180.86.62</td>\n",
       "      <td>0.055556</td>\n",
       "      <td>RO-16-UAIC</td>\n",
       "      <td>RO-07-NIPNE</td>\n",
       "      <td>perfsonar-grid.uaic.ro</td>\n",
       "      <td>perfsonar1.nipne.ro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26105</th>\n",
       "      <td>85.122.31.119</td>\n",
       "      <td>81.180.86.64</td>\n",
       "      <td>[2614]</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>fa3ba04018aa660e475494097c242d7ecf66ff56</td>\n",
       "      <td>True</td>\n",
       "      <td>[12675, 2614]</td>\n",
       "      <td>[2614]</td>\n",
       "      <td>85.122.31.119-81.180.86.64</td>\n",
       "      <td>0.057143</td>\n",
       "      <td>RO-16-UAIC</td>\n",
       "      <td>RO-07-NIPNE</td>\n",
       "      <td>perfsonar-grid.uaic.ro</td>\n",
       "      <td>perfsonar2.nipne.ro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26106</th>\n",
       "      <td>85.122.31.119</td>\n",
       "      <td>85.122.31.120</td>\n",
       "      <td>[12675]</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>8e027d2915563b8ffa10c4358834b43342eb8c2b</td>\n",
       "      <td>True</td>\n",
       "      <td>[None]</td>\n",
       "      <td>[None]</td>\n",
       "      <td>85.122.31.119-85.122.31.120</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>RO-16-UAIC</td>\n",
       "      <td>RO-16-UAIC</td>\n",
       "      <td>perfsonar-grid.uaic.ro</td>\n",
       "      <td>perfsonar2-grid.uaic.ro</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26107</th>\n",
       "      <td>85.122.31.119</td>\n",
       "      <td>90.147.67.252</td>\n",
       "      <td>[137, 12675, 20965, 2614]</td>\n",
       "      <td>4</td>\n",
       "      <td>36</td>\n",
       "      <td>d4dfb5c8c20e8a8cd89c786013687bc0e94e079e</td>\n",
       "      <td>False</td>\n",
       "      <td>[12675]</td>\n",
       "      <td>[137]</td>\n",
       "      <td>85.122.31.119-90.147.67.252</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>RO-16-UAIC</td>\n",
       "      <td>INFN-NAPOLI-ATLAS</td>\n",
       "      <td>perfsonar-grid.uaic.ro</td>\n",
       "      <td>perfsonar2.na.infn.it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26108</th>\n",
       "      <td>85.122.31.119</td>\n",
       "      <td>90.147.67.253</td>\n",
       "      <td>[137, 12675, 20965, 2614]</td>\n",
       "      <td>4</td>\n",
       "      <td>34</td>\n",
       "      <td>d4dfb5c8c20e8a8cd89c786013687bc0e94e079e</td>\n",
       "      <td>False</td>\n",
       "      <td>[12675]</td>\n",
       "      <td>[137]</td>\n",
       "      <td>85.122.31.119-90.147.67.253</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>RO-16-UAIC</td>\n",
       "      <td>INFN-NAPOLI-ATLAS</td>\n",
       "      <td>perfsonar-grid.uaic.ro</td>\n",
       "      <td>perfsonar.na.infn.it</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22617 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 src           dest                      asns_updated  \\\n",
       "1                                    [291, 20965, 293, 789, 2200, 59]   \n",
       "2                                    [291, 20965, 293, 789, 2200, 59]   \n",
       "3                                    [291, 20965, 293, 789, 2200, 59]   \n",
       "4                                                  [7896, 786, 20965]   \n",
       "5                                                  [7896, 786, 20965]   \n",
       "...              ...            ...                               ...   \n",
       "26103  85.122.31.119   81.180.86.62                            [2614]   \n",
       "26105  85.122.31.119   81.180.86.64                            [2614]   \n",
       "26106  85.122.31.119  85.122.31.120                           [12675]   \n",
       "26107  85.122.31.119  90.147.67.252         [137, 12675, 20965, 2614]   \n",
       "26108  85.122.31.119  90.147.67.253         [137, 12675, 20965, 2614]   \n",
       "\n",
       "       cnt_asn  cnt_total_measures                                      hash  \\\n",
       "1            6                   4  031fa51279113d0e60de6a10bb4b8181245a1bbc   \n",
       "2            6                   4  031fa51279113d0e60de6a10bb4b8181245a1bbc   \n",
       "3            6                   4  031fa51279113d0e60de6a10bb4b8181245a1bbc   \n",
       "4            3                   4  f070524c1d64bdd9e153f06b68e2b05c87fc9cd8   \n",
       "5            3                   4  f070524c1d64bdd9e153f06b68e2b05c87fc9cd8   \n",
       "...        ...                 ...                                       ...   \n",
       "26103        1                  72  fa3ba04018aa660e475494097c242d7ecf66ff56   \n",
       "26105        1                  35  fa3ba04018aa660e475494097c242d7ecf66ff56   \n",
       "26106        1                  36  8e027d2915563b8ffa10c4358834b43342eb8c2b   \n",
       "26107        4                  36  d4dfb5c8c20e8a8cd89c786013687bc0e94e079e   \n",
       "26108        4                  34  d4dfb5c8c20e8a8cd89c786013687bc0e94e079e   \n",
       "\n",
       "       all_dests_reached            first               last  \\\n",
       "1                  False  [7896, 59, 229]  [786, 43115, 789]   \n",
       "2                  False  [7896, 59, 229]  [786, 43115, 789]   \n",
       "3                  False  [7896, 59, 229]  [786, 43115, 789]   \n",
       "4                  False  [7896, 59, 229]  [786, 43115, 789]   \n",
       "5                  False  [7896, 59, 229]  [786, 43115, 789]   \n",
       "...                  ...              ...                ...   \n",
       "26103               True    [12675, 2614]             [2614]   \n",
       "26105               True    [12675, 2614]             [2614]   \n",
       "26106               True           [None]             [None]   \n",
       "26107              False          [12675]              [137]   \n",
       "26108              False          [12675]              [137]   \n",
       "\n",
       "                              pair  hash_freq    src_site          dest_site  \\\n",
       "1                                -   0.250000         NaN                NaN   \n",
       "2                                -   0.250000         NaN                NaN   \n",
       "3                                -   0.250000         NaN                NaN   \n",
       "4                                -   0.250000         NaN                NaN   \n",
       "5                                -   0.250000         NaN                NaN   \n",
       "...                            ...        ...         ...                ...   \n",
       "26103   85.122.31.119-81.180.86.62   0.055556  RO-16-UAIC        RO-07-NIPNE   \n",
       "26105   85.122.31.119-81.180.86.64   0.057143  RO-16-UAIC        RO-07-NIPNE   \n",
       "26106  85.122.31.119-85.122.31.120   1.000000  RO-16-UAIC         RO-16-UAIC   \n",
       "26107  85.122.31.119-90.147.67.252   1.000000  RO-16-UAIC  INFN-NAPOLI-ATLAS   \n",
       "26108  85.122.31.119-90.147.67.253   1.000000  RO-16-UAIC  INFN-NAPOLI-ATLAS   \n",
       "\n",
       "                       src_host                     dest_host  \n",
       "1              hcc-ps01.unl.edu  gridpp-ps-band.ecdf.ed.ac.uk  \n",
       "2              hcc-ps02.unl.edu   picperfsonar-latency.pic.es  \n",
       "3             psmsu02.aglt2.org            lpnhe-psl.in2p3.fr  \n",
       "4      perfsonar01.hep.wisc.edu           llrpsonar2.in2p3.fr  \n",
       "5              hcc-ps01.unl.edu  gridpp-ps-band.ecdf.ed.ac.uk  \n",
       "...                         ...                           ...  \n",
       "26103    perfsonar-grid.uaic.ro           perfsonar1.nipne.ro  \n",
       "26105    perfsonar-grid.uaic.ro           perfsonar2.nipne.ro  \n",
       "26106    perfsonar-grid.uaic.ro       perfsonar2-grid.uaic.ro  \n",
       "26107    perfsonar-grid.uaic.ro         perfsonar2.na.infn.it  \n",
       "26108    perfsonar-grid.uaic.ro          perfsonar.na.infn.it  \n",
       "\n",
       "[22617 rows x 15 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseLine = clean[clean.index.isin(freqAsBaseList)].copy()\n",
    "compare2 = clean[~clean.index.isin(baseLine.index)].copy()\n",
    "\n",
    "compare2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dest</th>\n",
       "      <th>asns_updated</th>\n",
       "      <th>cnt_asn</th>\n",
       "      <th>cnt_total_measures</th>\n",
       "      <th>hash</th>\n",
       "      <th>all_dests_reached</th>\n",
       "      <th>first</th>\n",
       "      <th>last</th>\n",
       "      <th>pair</th>\n",
       "      <th>hash_freq</th>\n",
       "      <th>src_site</th>\n",
       "      <th>dest_site</th>\n",
       "      <th>src_host</th>\n",
       "      <th>dest_host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16810</th>\n",
       "      <td>2001:610:108:203a::31</td>\n",
       "      <td>2001:760:4205:254::12</td>\n",
       "      <td>[513, 137, 1103]</td>\n",
       "      <td>3</td>\n",
       "      <td>67</td>\n",
       "      <td>328821b1bcb35a8deb7741425b864da4367088dd</td>\n",
       "      <td>False</td>\n",
       "      <td>[1103]</td>\n",
       "      <td>[137]</td>\n",
       "      <td>2001:610:108:203a::31-2001:760:4205:254::12</td>\n",
       "      <td>0.507463</td>\n",
       "      <td>SARA-MATRIX</td>\n",
       "      <td>INFN-T1</td>\n",
       "      <td>perfsonar-latency.grid.surfsara.nl</td>\n",
       "      <td>perfsonar-ow.cnaf.infn.it</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         src                   dest      asns_updated  \\\n",
       "16810  2001:610:108:203a::31  2001:760:4205:254::12  [513, 137, 1103]   \n",
       "\n",
       "       cnt_asn  cnt_total_measures                                      hash  \\\n",
       "16810        3                  67  328821b1bcb35a8deb7741425b864da4367088dd   \n",
       "\n",
       "       all_dests_reached   first   last  \\\n",
       "16810              False  [1103]  [137]   \n",
       "\n",
       "                                              pair  hash_freq     src_site  \\\n",
       "16810  2001:610:108:203a::31-2001:760:4205:254::12   0.507463  SARA-MATRIX   \n",
       "\n",
       "      dest_site                            src_host                  dest_host  \n",
       "16810   INFN-T1  perfsonar-latency.grid.surfsara.nl  perfsonar-ow.cnaf.infn.it  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dest</th>\n",
       "      <th>asns_updated</th>\n",
       "      <th>cnt_asn</th>\n",
       "      <th>cnt_total_measures</th>\n",
       "      <th>hash</th>\n",
       "      <th>all_dests_reached</th>\n",
       "      <th>first</th>\n",
       "      <th>last</th>\n",
       "      <th>pair</th>\n",
       "      <th>hash_freq</th>\n",
       "      <th>src_site</th>\n",
       "      <th>dest_site</th>\n",
       "      <th>src_host</th>\n",
       "      <th>dest_host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16811</th>\n",
       "      <td>2001:610:108:203a::31</td>\n",
       "      <td>2001:760:4205:254::12</td>\n",
       "      <td>[137, 20965, 1103]</td>\n",
       "      <td>3</td>\n",
       "      <td>67</td>\n",
       "      <td>d58424671825fffb19a8d0d6c7cc2f6ca13f6273</td>\n",
       "      <td>False</td>\n",
       "      <td>[1103]</td>\n",
       "      <td>[137]</td>\n",
       "      <td>2001:610:108:203a::31-2001:760:4205:254::12</td>\n",
       "      <td>0.492537</td>\n",
       "      <td>SARA-MATRIX</td>\n",
       "      <td>INFN-T1</td>\n",
       "      <td>perfsonar-latency.grid.surfsara.nl</td>\n",
       "      <td>perfsonar-ow.cnaf.infn.it</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         src                   dest        asns_updated  \\\n",
       "16811  2001:610:108:203a::31  2001:760:4205:254::12  [137, 20965, 1103]   \n",
       "\n",
       "       cnt_asn  cnt_total_measures                                      hash  \\\n",
       "16811        3                  67  d58424671825fffb19a8d0d6c7cc2f6ca13f6273   \n",
       "\n",
       "       all_dests_reached   first   last  \\\n",
       "16811              False  [1103]  [137]   \n",
       "\n",
       "                                              pair  hash_freq     src_site  \\\n",
       "16811  2001:610:108:203a::31-2001:760:4205:254::12   0.492537  SARA-MATRIX   \n",
       "\n",
       "      dest_site                            src_host                  dest_host  \n",
       "16811   INFN-T1  perfsonar-latency.grid.surfsara.nl  perfsonar-ow.cnaf.infn.it  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = list(diffFr)[84]\n",
    "\n",
    "display(baseLine[baseLine['pair']==p])\n",
    "display(compare2[compare2['pair']==p])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dest</th>\n",
       "      <th>asns_updated</th>\n",
       "      <th>cnt_asn</th>\n",
       "      <th>cnt_total_measures</th>\n",
       "      <th>hash</th>\n",
       "      <th>all_dests_reached</th>\n",
       "      <th>first</th>\n",
       "      <th>last</th>\n",
       "      <th>pair</th>\n",
       "      <th>hash_freq</th>\n",
       "      <th>src_site</th>\n",
       "      <th>dest_site</th>\n",
       "      <th>src_host</th>\n",
       "      <th>dest_host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16915</th>\n",
       "      <td>2001:610:108:203a::32</td>\n",
       "      <td>2001:67c:1148:204::250</td>\n",
       "      <td>[43115, 20965, 766, 1103]</td>\n",
       "      <td>4</td>\n",
       "      <td>67</td>\n",
       "      <td>d189b5f69fa1c4c41e9ad51ff06bbc36c58213b8</td>\n",
       "      <td>False</td>\n",
       "      <td>[1103]</td>\n",
       "      <td>[43115, 766]</td>\n",
       "      <td>2001:610:108:203a::32-2001:67c:1148:204::250</td>\n",
       "      <td>0.492537</td>\n",
       "      <td>SARA-MATRIX</td>\n",
       "      <td>pic</td>\n",
       "      <td>perfsonar-bandwidth.grid.surfsara.nl</td>\n",
       "      <td>picperfsonar-bandwidth.pic.es</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         src                    dest  \\\n",
       "16915  2001:610:108:203a::32  2001:67c:1148:204::250   \n",
       "\n",
       "                    asns_updated  cnt_asn  cnt_total_measures  \\\n",
       "16915  [43115, 20965, 766, 1103]        4                  67   \n",
       "\n",
       "                                           hash  all_dests_reached   first  \\\n",
       "16915  d189b5f69fa1c4c41e9ad51ff06bbc36c58213b8              False  [1103]   \n",
       "\n",
       "               last                                          pair  hash_freq  \\\n",
       "16915  [43115, 766]  2001:610:108:203a::32-2001:67c:1148:204::250   0.492537   \n",
       "\n",
       "          src_site dest_site                              src_host  \\\n",
       "16915  SARA-MATRIX       pic  perfsonar-bandwidth.grid.surfsara.nl   \n",
       "\n",
       "                           dest_host  \n",
       "16915  picperfsonar-bandwidth.pic.es  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dest</th>\n",
       "      <th>asns_updated</th>\n",
       "      <th>cnt_asn</th>\n",
       "      <th>cnt_total_measures</th>\n",
       "      <th>hash</th>\n",
       "      <th>all_dests_reached</th>\n",
       "      <th>first</th>\n",
       "      <th>last</th>\n",
       "      <th>pair</th>\n",
       "      <th>hash_freq</th>\n",
       "      <th>src_site</th>\n",
       "      <th>dest_site</th>\n",
       "      <th>src_host</th>\n",
       "      <th>dest_host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16916</th>\n",
       "      <td>2001:610:108:203a::32</td>\n",
       "      <td>2001:67c:1148:204::250</td>\n",
       "      <td>[513, 20965, 43115, 1103]</td>\n",
       "      <td>4</td>\n",
       "      <td>67</td>\n",
       "      <td>8be6fd9e4c75b0cddbfc139196602acc09dbba4e</td>\n",
       "      <td>False</td>\n",
       "      <td>[1103]</td>\n",
       "      <td>[43115, 766]</td>\n",
       "      <td>2001:610:108:203a::32-2001:67c:1148:204::250</td>\n",
       "      <td>0.492537</td>\n",
       "      <td>SARA-MATRIX</td>\n",
       "      <td>pic</td>\n",
       "      <td>perfsonar-bandwidth.grid.surfsara.nl</td>\n",
       "      <td>picperfsonar-bandwidth.pic.es</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16917</th>\n",
       "      <td>2001:610:108:203a::32</td>\n",
       "      <td>2001:67c:1148:204::250</td>\n",
       "      <td>[20965, 766, 1103]</td>\n",
       "      <td>3</td>\n",
       "      <td>67</td>\n",
       "      <td>2906cedbd8966dab46eb04670a6f283e9bbff8b7</td>\n",
       "      <td>False</td>\n",
       "      <td>[1103]</td>\n",
       "      <td>[43115, 766]</td>\n",
       "      <td>2001:610:108:203a::32-2001:67c:1148:204::250</td>\n",
       "      <td>0.014925</td>\n",
       "      <td>SARA-MATRIX</td>\n",
       "      <td>pic</td>\n",
       "      <td>perfsonar-bandwidth.grid.surfsara.nl</td>\n",
       "      <td>picperfsonar-bandwidth.pic.es</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         src                    dest  \\\n",
       "16916  2001:610:108:203a::32  2001:67c:1148:204::250   \n",
       "16917  2001:610:108:203a::32  2001:67c:1148:204::250   \n",
       "\n",
       "                    asns_updated  cnt_asn  cnt_total_measures  \\\n",
       "16916  [513, 20965, 43115, 1103]        4                  67   \n",
       "16917         [20965, 766, 1103]        3                  67   \n",
       "\n",
       "                                           hash  all_dests_reached   first  \\\n",
       "16916  8be6fd9e4c75b0cddbfc139196602acc09dbba4e              False  [1103]   \n",
       "16917  2906cedbd8966dab46eb04670a6f283e9bbff8b7              False  [1103]   \n",
       "\n",
       "               last                                          pair  hash_freq  \\\n",
       "16916  [43115, 766]  2001:610:108:203a::32-2001:67c:1148:204::250   0.492537   \n",
       "16917  [43115, 766]  2001:610:108:203a::32-2001:67c:1148:204::250   0.014925   \n",
       "\n",
       "          src_site dest_site                              src_host  \\\n",
       "16916  SARA-MATRIX       pic  perfsonar-bandwidth.grid.surfsara.nl   \n",
       "16917  SARA-MATRIX       pic  perfsonar-bandwidth.grid.surfsara.nl   \n",
       "\n",
       "                           dest_host  \n",
       "16916  picperfsonar-bandwidth.pic.es  \n",
       "16917  picperfsonar-bandwidth.pic.es  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = list(diffFr)[86]\n",
    "\n",
    "display(baseLine[baseLine['pair']==p])\n",
    "display(compare2[compare2['pair']==p])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20080"
      ]
     },
     "execution_count": 366,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "diff: [789]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dest</th>\n",
       "      <th>asns_updated</th>\n",
       "      <th>cnt_asn</th>\n",
       "      <th>cnt_total_measures</th>\n",
       "      <th>hash</th>\n",
       "      <th>all_dests_reached</th>\n",
       "      <th>first</th>\n",
       "      <th>last</th>\n",
       "      <th>pair</th>\n",
       "      <th>hash_freq</th>\n",
       "      <th>src_site</th>\n",
       "      <th>dest_site</th>\n",
       "      <th>src_host</th>\n",
       "      <th>dest_host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19405</th>\n",
       "      <td>2001:660:5009:9:193:48:99:76</td>\n",
       "      <td>2001:67c:1148:203::242</td>\n",
       "      <td>[2200, 513, 43115, 20965]</td>\n",
       "      <td>4</td>\n",
       "      <td>67</td>\n",
       "      <td>1ecfaf1d6aad85102708cdac02717007af7c310a</td>\n",
       "      <td>False</td>\n",
       "      <td>[2200, 789]</td>\n",
       "      <td>[43115]</td>\n",
       "      <td>2001:660:5009:9:193:48:99:76-2001:67c:1148:203...</td>\n",
       "      <td>0.343284</td>\n",
       "      <td>IN2P3-CC</td>\n",
       "      <td>pic</td>\n",
       "      <td>ccperfsonar2.in2p3.fr</td>\n",
       "      <td>picperfsonar-latency.pic.es</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                src                    dest  \\\n",
       "19405  2001:660:5009:9:193:48:99:76  2001:67c:1148:203::242   \n",
       "\n",
       "                    asns_updated  cnt_asn  cnt_total_measures  \\\n",
       "19405  [2200, 513, 43115, 20965]        4                  67   \n",
       "\n",
       "                                           hash  all_dests_reached  \\\n",
       "19405  1ecfaf1d6aad85102708cdac02717007af7c310a              False   \n",
       "\n",
       "             first     last  \\\n",
       "19405  [2200, 789]  [43115]   \n",
       "\n",
       "                                                    pair  hash_freq  src_site  \\\n",
       "19405  2001:660:5009:9:193:48:99:76-2001:67c:1148:203...   0.343284  IN2P3-CC   \n",
       "\n",
       "      dest_site               src_host                    dest_host  \n",
       "19405       pic  ccperfsonar2.in2p3.fr  picperfsonar-latency.pic.es  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src</th>\n",
       "      <th>dest</th>\n",
       "      <th>asns_updated</th>\n",
       "      <th>cnt_asn</th>\n",
       "      <th>cnt_total_measures</th>\n",
       "      <th>hash</th>\n",
       "      <th>all_dests_reached</th>\n",
       "      <th>first</th>\n",
       "      <th>last</th>\n",
       "      <th>pair</th>\n",
       "      <th>hash_freq</th>\n",
       "      <th>src_site</th>\n",
       "      <th>dest_site</th>\n",
       "      <th>src_host</th>\n",
       "      <th>dest_host</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19404</th>\n",
       "      <td>2001:660:5009:9:193:48:99:76</td>\n",
       "      <td>2001:67c:1148:203::242</td>\n",
       "      <td>[513, 43115, 789]</td>\n",
       "      <td>3</td>\n",
       "      <td>67</td>\n",
       "      <td>c47d1d97a0489116ed77001e6d6455f4ce8bc53f</td>\n",
       "      <td>False</td>\n",
       "      <td>[2200, 789]</td>\n",
       "      <td>[43115]</td>\n",
       "      <td>2001:660:5009:9:193:48:99:76-2001:67c:1148:203...</td>\n",
       "      <td>0.164179</td>\n",
       "      <td>IN2P3-CC</td>\n",
       "      <td>pic</td>\n",
       "      <td>ccperfsonar2.in2p3.fr</td>\n",
       "      <td>picperfsonar-latency.pic.es</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19406</th>\n",
       "      <td>2001:660:5009:9:193:48:99:76</td>\n",
       "      <td>2001:67c:1148:203::242</td>\n",
       "      <td>[513, 20965, 43115, 789]</td>\n",
       "      <td>4</td>\n",
       "      <td>67</td>\n",
       "      <td>5a702fdd6c197a8173790c854c922dc75e2a499f</td>\n",
       "      <td>False</td>\n",
       "      <td>[2200, 789]</td>\n",
       "      <td>[43115]</td>\n",
       "      <td>2001:660:5009:9:193:48:99:76-2001:67c:1148:203...</td>\n",
       "      <td>0.328358</td>\n",
       "      <td>IN2P3-CC</td>\n",
       "      <td>pic</td>\n",
       "      <td>ccperfsonar2.in2p3.fr</td>\n",
       "      <td>picperfsonar-latency.pic.es</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19407</th>\n",
       "      <td>2001:660:5009:9:193:48:99:76</td>\n",
       "      <td>2001:67c:1148:203::242</td>\n",
       "      <td>[2200, 513, 43115]</td>\n",
       "      <td>3</td>\n",
       "      <td>67</td>\n",
       "      <td>d84a22d14a15a76ad749c3673e3de20d7e5ba923</td>\n",
       "      <td>False</td>\n",
       "      <td>[2200, 789]</td>\n",
       "      <td>[43115]</td>\n",
       "      <td>2001:660:5009:9:193:48:99:76-2001:67c:1148:203...</td>\n",
       "      <td>0.164179</td>\n",
       "      <td>IN2P3-CC</td>\n",
       "      <td>pic</td>\n",
       "      <td>ccperfsonar2.in2p3.fr</td>\n",
       "      <td>picperfsonar-latency.pic.es</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                src                    dest  \\\n",
       "19404  2001:660:5009:9:193:48:99:76  2001:67c:1148:203::242   \n",
       "19406  2001:660:5009:9:193:48:99:76  2001:67c:1148:203::242   \n",
       "19407  2001:660:5009:9:193:48:99:76  2001:67c:1148:203::242   \n",
       "\n",
       "                   asns_updated  cnt_asn  cnt_total_measures  \\\n",
       "19404         [513, 43115, 789]        3                  67   \n",
       "19406  [513, 20965, 43115, 789]        4                  67   \n",
       "19407        [2200, 513, 43115]        3                  67   \n",
       "\n",
       "                                           hash  all_dests_reached  \\\n",
       "19404  c47d1d97a0489116ed77001e6d6455f4ce8bc53f              False   \n",
       "19406  5a702fdd6c197a8173790c854c922dc75e2a499f              False   \n",
       "19407  d84a22d14a15a76ad749c3673e3de20d7e5ba923              False   \n",
       "\n",
       "             first     last  \\\n",
       "19404  [2200, 789]  [43115]   \n",
       "19406  [2200, 789]  [43115]   \n",
       "19407  [2200, 789]  [43115]   \n",
       "\n",
       "                                                    pair  hash_freq  src_site  \\\n",
       "19404  2001:660:5009:9:193:48:99:76-2001:67c:1148:203...   0.164179  IN2P3-CC   \n",
       "19406  2001:660:5009:9:193:48:99:76-2001:67c:1148:203...   0.328358  IN2P3-CC   \n",
       "19407  2001:660:5009:9:193:48:99:76-2001:67c:1148:203...   0.164179  IN2P3-CC   \n",
       "\n",
       "      dest_site               src_host                    dest_host  \n",
       "19404       pic  ccperfsonar2.in2p3.fr  picperfsonar-latency.pic.es  \n",
       "19406       pic  ccperfsonar2.in2p3.fr  picperfsonar-latency.pic.es  \n",
       "19407       pic  ccperfsonar2.in2p3.fr  picperfsonar-latency.pic.es  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2001:660:5009:9:193:48:99:76-2001:67c:1148:203::242\n",
      "\n",
      "\n",
      "[789, 513, 513, 513, 43115]\n",
      "['193.48.99.100', '192.16.166.193', '192.16.166.213', '192.16.166.58', '193.109.172.242']\n",
      "\n",
      "[2200, 513, 513, 20965, 43115]\n",
      "['2001:660:5009:9:193:48:99:100', '2001:1458:302:31::1', '2001:1458:302:35::1', '2001:798:111:1::11', '2001:67c:1148:203::242']\n"
     ]
    }
   ],
   "source": [
    "p = list(diffFr)[174]\n",
    "print('diff:',diffFr[p])\n",
    "\n",
    "\n",
    "display(baseLine[baseLine['pair']==p])\n",
    "display(compare2[compare2['pair']==p])\n",
    "\n",
    "\n",
    "print(p)\n",
    "print()\n",
    "\n",
    "\n",
    "# print(df[df['pair']==p][['hops']].values[2])\n",
    "print()\n",
    "\n",
    "print(relDf[relDf['pair']==p]['asns_updated'].values[0])\n",
    "print(relDf[relDf['pair']==p]['hops'].values[0])\n",
    "\n",
    "print()\n",
    "print(relDf[relDf['pair']==p]['asns_updated'].values[1])\n",
    "print(relDf[relDf['pair']==p]['hops'].values[1])\n",
    "\n",
    "\n",
    "# print(relDf[relDf['pair']==p]['asns'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['perfsonar2.ultralight.org'],\n",
       "       ['hcc-ps01.unl.edu'],\n",
       "       ['hcc-ps02.unl.edu'],\n",
       "       ['perfsonar01.hep.wisc.edu'],\n",
       "       ['perfsonar-cms2.itns.purdue.edu'],\n",
       "       ['perfsonar-cms1.itns.purdue.edu'],\n",
       "       ['psmsu02.aglt2.org'],\n",
       "       ['lhcmon3.bnl.gov'],\n",
       "       ['perfsonar.ultralight.org'],\n",
       "       ['perfsonar02.hep.wisc.edu']], dtype=object)"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df[(df['dest']=='')|(df['src']=='')][['src','src_host','dest','dest_host']].drop_duplicates()\n",
    "\n",
    "\n",
    "s = df[(df['src']=='')][['src_host']].drop_duplicates()\n",
    "\n",
    "s.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['perfson1.ppgrid1.rhul.ac.uk'],\n",
       " ['ps-02.lnl.infn.it'],\n",
       " ['ps02.iihe.ac.be'],\n",
       " ['ps-bandwidth.sfu.westgrid.ca'],\n",
       " ['psonar2.lal.in2p3.fr'],\n",
       " ['psfrascati02.lnf.infn.it'],\n",
       " ['ps-gsdc02.sdfarm.kr'],\n",
       " ['perfsonar-de-kit.gridka.de'],\n",
       " ['lcgperf.shef.ac.uk'],\n",
       " ['marperf01.in2p3.fr'],\n",
       " ['llrpsonar2.in2p3.fr'],\n",
       " ['gridpp-ps-band.ecdf.ed.ac.uk'],\n",
       " ['picperfsonar-latency.pic.es'],\n",
       " ['perfsonar02.ft.uam.es'],\n",
       " ['psific01.ific.uv.es'],\n",
       " ['lpnhe-psl.in2p3.fr'],\n",
       " ['perfsonar-bw.hepgrid.uerj.br'],\n",
       " ['ps-gsdc01.sdfarm.kr'],\n",
       " ['perfsonar2-grid.uaic.ro'],\n",
       " ['lhcb-perf.nipne.ro'],\n",
       " ['atrogr007.nipne.ro'],\n",
       " ['perfsonar03-iep-grid.saske.sk'],\n",
       " ['perfsonar2.nipne.ro'],\n",
       " ['t1-pfsn2.jinr-t1.ru'],\n",
       " ['psific02.ific.uv.es'],\n",
       " ['perfsonar1.nipne.ro'],\n",
       " ['lapp-ps01.in2p3.fr'],\n",
       " ['perfsonar-lt.hepgrid.uerj.br'],\n",
       " ['ccperfsonar2.in2p3.fr'],\n",
       " ['perfsonar01.ft.uam.es']]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = df[(df['dest']=='')][['dest_host']].drop_duplicates()\n",
    "d.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "383"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import collections\n",
    "\n",
    "toCheckFr, diffsFr = [],[]\n",
    "diffFr = {}\n",
    "\n",
    "\n",
    "def getChanged(group):\n",
    "    try:\n",
    "        # print(group.name)\n",
    "        base = baseLine[baseLine['pair']==group.name]['asns_updated'].values.tolist()[0]\n",
    "        \n",
    "        casns = compare2[compare2['pair']==group.name]['asns_updated'].tolist()\n",
    "        asns_expanded = list([j for i in casns for j in i])\n",
    "        counter=collections.Counter(asns_expanded)\n",
    "\n",
    "        flag = False\n",
    "        diff_temp = []\n",
    "        \n",
    "        \n",
    "\n",
    "        for i, asns in enumerate(group.asns_updated):\n",
    "            first = group['first'].values[i]\n",
    "            last = group['last'].values[i]\n",
    "            freq = group['hash_freq'].values[i]\n",
    "\n",
    "            diff = list(set(asns)-set(base))\n",
    "    \n",
    "\n",
    "            # for d in diff:\n",
    "            #     if d in first:\n",
    "            #         diff.remove(d)\n",
    "                    \n",
    "                # if len(diff)>0 and group.all_dests_reached.values[0]==True:\n",
    "                #      if d in last:\n",
    "                #         diff.remove(d)\n",
    "            \n",
    "                \n",
    "            if len(diff)>0:\n",
    "                for d in diff:                \n",
    "                    if d in altsOfAlts.keys():\n",
    "                        flag = all([False if alt in base else True for alt in altsOfAlts[d]])\n",
    "                    else: flag = True\n",
    "\n",
    "\n",
    "            if len(diff)>0:\n",
    "                diff_temp.extend(diff)\n",
    "\n",
    "        # if flag and len(base)>2 and (freq<0.48 or freq>0.51):\n",
    "        if flag and len(base)>=2:\n",
    "        # if flag:\n",
    "            toCheckFr.append(group.name)\n",
    "            diffsFr.append(list(set(diff_temp)))\n",
    "            diffFr[group.name] = list(set(diff_temp))\n",
    "    except Exception as e:\n",
    "        print('Issue wtih:',group.name,e)\n",
    "        print(base)\n",
    "\n",
    "\n",
    "sites2remove = ['ATLAS-CBPF', 'NCP-LCG2', 'UTA_SWT2', 'RRC_KI']\n",
    "cut = compare2[(~compare2['src_site'].isin(sites2remove))&(~compare2['dest_site'].isin(['NCP-LCG2', 'UTA_SWT2', 'RRC_KI']))]\n",
    "\n",
    "cut[(cut['hash_freq']<1)&~(cut['dest']=='')&~(cut['src']=='')].groupby('pair').apply(lambda x: getChanged(x))\n",
    "\n",
    "# cut[(cut['pair']==p)].groupby('pair').apply(lambda x: getChanged(x))\n",
    "\n",
    "len(toCheckFr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "616"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import collections\n",
    "\n",
    "toCheckFr, diffsFr = [],[]\n",
    "diffFr = {}\n",
    "\n",
    "\n",
    "def getChanged(group):\n",
    "    try:\n",
    "        # print(group.name)\n",
    "        base = baseLine[baseLine['pair']==group.name]['asns_updated'].values.tolist()[0]\n",
    "        \n",
    "        casns = compare2[compare2['pair']==group.name]['asns_updated'].tolist()\n",
    "        asns_expanded = list([j for i in casns for j in i])\n",
    "        counter=collections.Counter(asns_expanded)\n",
    "\n",
    "        flag = False\n",
    "        diff_temp = []\n",
    "        \n",
    "        \n",
    "\n",
    "        for i, asns in enumerate(group.asns_updated):\n",
    "            first = group['first'].values[i]\n",
    "            last = group['last'].values[i]\n",
    "            freq = group['hash_freq'].values[i]\n",
    "\n",
    "            diff = list(set(asns)-set(base))\n",
    "    \n",
    "\n",
    "            for d in diff:\n",
    "                if d in first:\n",
    "                    diff.remove(d)\n",
    "                    \n",
    "                if len(diff)>0 and group.all_dests_reached.values[0]==True:\n",
    "                     if d in last:\n",
    "                        diff.remove(d)\n",
    "            \n",
    "            if len(diff)>0:\n",
    "                for d in diff:                \n",
    "                    if d in altsOfAlts.keys():\n",
    "                        if not altsOfAlts[d] in base:\n",
    "                            flag = True\n",
    "                    else: flag = True\n",
    "\n",
    "\n",
    "            if len(diff)>0:\n",
    "                diff_temp.extend(diff)\n",
    "\n",
    "        # if flag and len(base)>2 and (freq<0.48 or freq>0.51):\n",
    "        if flag and len(base)>2:\n",
    "            toCheckFr.append(group.name)\n",
    "            diffsFr.append(list(set(diff_temp)))\n",
    "            diffFr[group.name] = list(set(diff_temp))\n",
    "    except Exception as e:\n",
    "        print('Issue wtih:',group.name,e)\n",
    "        print(base)\n",
    "        print(group.first[i])\n",
    "\n",
    "p = '10.10.0.2-131.169.168.30'\n",
    "\n",
    "cut = compare2[(~compare2['src_site'].isin(['NCP-LCG2', 'UTA_SWT2', 'RRC_KI']))&(~compare2['dest_site'].isin(['NCP-LCG2', 'UTA_SWT2', 'RRC_KI']))]\n",
    "cut[(cut['hash_freq']<1)&~(cut['dest']=='')&~(cut['src']=='')].groupby('pair').apply(lambda x: getChanged(x))\n",
    "# cut[(cut['pair']==p)].groupby('pair').apply(lambda x: getChanged(x))\n",
    "\n",
    "\n",
    "\n",
    "# cut[cut['pair']==list(diffFr)[3]].groupby('pair').apply(lambda x: getChanged(x))\n",
    "len(toCheckFr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 482 - remove first\n",
    "# down to\n",
    "# 363 - remove last if all reached\n",
    "# 312 - remove all baseline < 3\n",
    "###### 304 - freq<0.48 or freq>0.51\n",
    "# 302 - ~(cut['dest']=='')&~(cut['src']=='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pair</th>\n",
       "      <th>diff</th>\n",
       "      <th>dest</th>\n",
       "      <th>src</th>\n",
       "      <th>src_host</th>\n",
       "      <th>dest_host</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>src_site</th>\n",
       "      <th>dest_site</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AGLT2</th>\n",
       "      <th>UAM-LCG2</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">BEIJING-LCG2</th>\n",
       "      <th>JINR-LCG2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JINR-T1</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">BNL-ATLAS</th>\n",
       "      <th>CBPF</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IN2P3-CC</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UKI-SCOTGRID-ECDF</th>\n",
       "      <th>JINR-LCG2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UKI-SOUTHGRID-RALPP</th>\n",
       "      <th>CBPF</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>USCMS-FNAL-WC1</th>\n",
       "      <th>UCSDT2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WT2</th>\n",
       "      <th>UAM-LCG2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>praguelcg2</th>\n",
       "      <th>CBPF</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>137 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                               pair  diff  dest  src  src_host  dest_host\n",
       "src_site            dest_site                                            \n",
       "AGLT2               UAM-LCG2     10    10    10   10        10         10\n",
       "BEIJING-LCG2        JINR-LCG2     3     3     3    3         3          3\n",
       "                    JINR-T1       3     3     3    3         3          3\n",
       "BNL-ATLAS           CBPF          4     4     4    4         4          4\n",
       "                    IN2P3-CC      2     2     2    2         2          2\n",
       "...                             ...   ...   ...  ...       ...        ...\n",
       "UKI-SCOTGRID-ECDF   JINR-LCG2     1     1     1    1         1          1\n",
       "UKI-SOUTHGRID-RALPP CBPF          4     4     4    4         4          4\n",
       "USCMS-FNAL-WC1      UCSDT2        1     1     1    1         1          1\n",
       "WT2                 UAM-LCG2      2     2     2    2         2          2\n",
       "praguelcg2          CBPF          5     5     5    5         5          5\n",
       "\n",
       "[137 rows x 6 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frl = []\n",
    "for pair, diff in diffFr.items():\n",
    "    for d in diff:\n",
    "        frl.append([pair,d])\n",
    "        \n",
    "        \n",
    "freqdf = pd.DataFrame(frl, columns=['pair', 'diff'])\n",
    "\n",
    "sub = df[['dest', 'src_site', 'src', 'dest_site', 'src_host', 'dest_host', 'pair']].drop_duplicates()\n",
    "freqdf = pd.merge(freqdf, sub, on='pair', how='left')\n",
    "freqdf.groupby(['src_site','dest_site']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'temp-plot.html'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "import plotly.figure_factory as ff\n",
    "import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "import plotly as py\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objects as go\n",
    "import collections\n",
    "\n",
    "\n",
    "\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "d3 = {k:v for k,v in diffFr.items()}\n",
    "\n",
    "\n",
    "pairs = d3\n",
    "start,stop = 0,len(d3)\n",
    "# start,stop = 150,len(d3)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def plotVals(pair, diff):\n",
    "    vectors = relDf[(relDf['pair']==pair)]['asns_updated'].tolist()\n",
    "\n",
    "    asnsExpanded = list([j for i in vectors for j in i])\n",
    "    asnDisplayVals = {i:pos+1 for pos, i in enumerate(set(asnsExpanded))}\n",
    "    numMeasures = len(vectors)\n",
    "    positions = max([len(l) for l in vectors])\n",
    "\n",
    "    # asns_expanded = list([j for i in clean[(clean['pair']==pair)]['asns_updated'].tolist() for j in i])\n",
    "    # counter=collections.Counter(asns_expanded)\n",
    "    \n",
    "    asnDisplaycolors = {i:10 if i in diff else 1 for i in list(set(asnsExpanded))}\n",
    "    \n",
    "    def giveColorEach(l):\n",
    "        return [asnDisplaycolors[i] for i in l]\n",
    "\n",
    "    colored = [giveColorEach(j) for j in vectors]\n",
    "    \n",
    "    # print(colored)\n",
    "    \n",
    "    return [vectors, colored]\n",
    "\n",
    "\n",
    "def getData(diffs):\n",
    "    lv, tl = [],[]\n",
    "    # for i in range(start, stop):\n",
    "    i = 0\n",
    "    for pair, diff in diffs.items():\n",
    "        # print(pair, diff)\n",
    "        lv.append(plotVals(pair, diff))\n",
    "\n",
    "        sites = clean[(clean[\"pair\"]==pair)][[\"src_site\", \"dest_site\"]].drop_duplicates().values[0].tolist()\n",
    "        # print(pair)\n",
    "        # print(f'{i} Baseline: {baseLineDfFr[baseLineDfFr[\"pair\"]==pair][\"asn\"].values[0]}  Diffs: {diff}')\n",
    "        tl.append(f'{i} {pair} {sites[0]}-{sites[1]} Baseline: {baseLine[baseLine[\"pair\"]==pair][\"asns_updated\"].values[0]}  Diffs: {diff}')\n",
    "        # tl.append(f'{i}  {pair}  Diffs: {diff}')\n",
    "        i+=1\n",
    "\n",
    "\n",
    "    return [lv, tl]\n",
    "\n",
    "\n",
    "num_rows = round((stop-start)/2)\n",
    "# num_rows = 1\n",
    "dataFr = getData(d3)\n",
    "# height = \n",
    "\n",
    "fig = make_subplots(rows=num_rows, cols=2, \n",
    "                    subplot_titles=dataFr[1],\n",
    "                   horizontal_spacing = 0.03, vertical_spacing = 0.001  \n",
    "                   )\n",
    "\n",
    "i=0\n",
    "for row in range(1,num_rows+1):\n",
    "    for col in [1,2]:\n",
    "        if i<len(pairs)-start:\n",
    "            fig.add_trace(\n",
    "                go.Heatmap(\n",
    "                    z=dataFr[0][i][1],\n",
    "                    text=dataFr[0][i][0],\n",
    "                    texttemplate=\"%{text}\",\n",
    "                ),\n",
    "                row=row, col=col\n",
    "            )\n",
    "            i+=1\n",
    "        \n",
    "\n",
    "fig.update_layout(template='plotly_white',\n",
    "                  height=90000\n",
    "                 )\n",
    "fig.update_annotations(font_size=12)\n",
    "fig.update_traces(showscale=False)\n",
    "py.offline.plot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'temp-plot.html'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Compare!!\n",
    "import plotly.figure_factory as ff\n",
    "import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "import plotly as py\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objects as go\n",
    "import collections\n",
    "\n",
    "\n",
    "\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "\n",
    "# onesideonly = list(set(diffFr.keys())-set(diffUp.keys()))\n",
    "\n",
    "\n",
    "# len(onesideonly)\n",
    "\n",
    "# d1 = {k:v for k,v in diffUp.items() if k in intersection[:200]}\n",
    "# d3 = {k:v for k,v in diffFr.items() if k in onesideonly[:200]}\n",
    "d3 = {k:v for k,v in diffFr.items()}\n",
    "\n",
    "\n",
    "pairs = d3\n",
    "start,stop = 0,200\n",
    "\n",
    "\n",
    "\n",
    "def plotVals(pair, diff):\n",
    "    vectors = df[(df['pair']==pair)]['asns'].tolist()\n",
    "\n",
    "    asnsExpanded = list([j for i in vectors for j in i])\n",
    "    asnDisplayVals = {i:pos+1 for pos, i in enumerate(set(asnsExpanded))}\n",
    "    numMeasures = len(vectors)\n",
    "    positions = max([len(l) for l in vectors])\n",
    "\n",
    "    asns_expanded = list([j for i in vectors for j in i])\n",
    "    counter=collections.Counter(asns_expanded)\n",
    "    \n",
    "    asnDisplaycolors = {i:1 if i not in diff else 10 for i in list(set(asnsExpanded))}\n",
    "    \n",
    "    def giveColorEach(l):\n",
    "        return [asnDisplaycolors[i] for i in l]\n",
    "\n",
    "    colored = [giveColorEach(j) for j in vectors]\n",
    "    \n",
    "    # print(colored)\n",
    "    \n",
    "    return [vectors, colored]\n",
    "\n",
    "\n",
    "def getData(start,stop,diffs):\n",
    "    lv, tl = [],[]\n",
    "    # for i in range(start, stop):\n",
    "    i = 0\n",
    "    for pair, diff in diffs.items():\n",
    "        \n",
    "        lv.append(plotVals(pair, diff))\n",
    "\n",
    "        sites = df[(df[\"pair\"]==pair)][[\"src_site\", \"dest_site\"]].drop_duplicates().values[0].tolist()\n",
    "        # print(pair)\n",
    "        # print(f'{i} Baseline: {baseLineDfFr[baseLineDfFr[\"pair\"]==pair][\"asn\"].values[0]}  Diffs: {diff}')\n",
    "        tl.append(f'{i} {sites[0]}-{sites[1]} Baseline: {baseLineDfFr[baseLineDfFr[\"pair\"]==pair][\"asn\"].values[0]}  Diffs: {diff}')\n",
    "        # tl.append(f'{i}  {pair}  Diffs: {diff}')\n",
    "        i+=1\n",
    "\n",
    "\n",
    "    return [lv, tl]\n",
    "\n",
    "\n",
    "num_rows = round((stop-start)/2)\n",
    "dataFr = getData(start,stop, d3)\n",
    "\n",
    "fig = make_subplots(rows=num_rows, cols=2, \n",
    "                    subplot_titles=dataFr[1],\n",
    "                   horizontal_spacing = 0.03, vertical_spacing = 0.002)\n",
    "\n",
    "i=0\n",
    "for row in range(1,num_rows+1):\n",
    "    for col in [1,2]:\n",
    "        if i<len(pairs)-start:\n",
    "            fig.add_trace(\n",
    "                go.Heatmap(\n",
    "                    z=dataFr[0][i][1],\n",
    "                    text=dataFr[0][i][0],\n",
    "                    texttemplate=\"%{text}\",\n",
    "                ),\n",
    "                row=row, col=col\n",
    "            )\n",
    "            i+=1\n",
    "        \n",
    "\n",
    "fig.update_layout(template='plotly_white',\n",
    "                  height=40000\n",
    "                 )\n",
    "fig.update_annotations(font_size=12)\n",
    "fig.update_traces(showscale=False)\n",
    "py.offline.plot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}