url_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b1eae66a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cde8f98b",
   "metadata": {},
   "outputs": [],
   "source": [
    "url_df = pd.read_csv(\"all_urls.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4c9e0103",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>br-icloud.com.br</td>\n",
       "      <td>phishing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mp3raid.com/music/krizz_kaliko.html</td>\n",
       "      <td>benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bopsecrets.org/rexroth/cr/1.htm</td>\n",
       "      <td>benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://www.garage-pirenne.be/index.php?option=...</td>\n",
       "      <td>defacement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://adventure-nicaragua.net/index.php?optio...</td>\n",
       "      <td>defacement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651186</th>\n",
       "      <td>xbox360.ign.com/objects/850/850402.html</td>\n",
       "      <td>phishing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651187</th>\n",
       "      <td>games.teamxbox.com/xbox-360/1860/Dead-Space/</td>\n",
       "      <td>phishing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651188</th>\n",
       "      <td>www.gamespot.com/xbox360/action/deadspace/</td>\n",
       "      <td>phishing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651189</th>\n",
       "      <td>en.wikipedia.org/wiki/Dead_Space_(video_game)</td>\n",
       "      <td>phishing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651190</th>\n",
       "      <td>www.angelfire.com/goth/devilmaycrytonite/</td>\n",
       "      <td>phishing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>651191 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      url        type\n",
       "0                                        br-icloud.com.br    phishing\n",
       "1                     mp3raid.com/music/krizz_kaliko.html      benign\n",
       "2                         bopsecrets.org/rexroth/cr/1.htm      benign\n",
       "3       http://www.garage-pirenne.be/index.php?option=...  defacement\n",
       "4       http://adventure-nicaragua.net/index.php?optio...  defacement\n",
       "...                                                   ...         ...\n",
       "651186            xbox360.ign.com/objects/850/850402.html    phishing\n",
       "651187       games.teamxbox.com/xbox-360/1860/Dead-Space/    phishing\n",
       "651188         www.gamespot.com/xbox360/action/deadspace/    phishing\n",
       "651189      en.wikipedia.org/wiki/Dead_Space_(video_game)    phishing\n",
       "651190          www.angelfire.com/goth/devilmaycrytonite/    phishing\n",
       "\n",
       "[651191 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c5108245",
   "metadata": {},
   "outputs": [],
   "source": [
    "only_benign = url_df.loc[url_df['type'] == 'benign']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "768ef6e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# isolating all the bad urls\n",
    "only_malicious = url_df.loc[(url_df['type'] == 'phishing') | (url_df['type'] == 'malware') | (url_df['type'] == 'defacement')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b59e73ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "v = TfidfVectorizer(analyzer='char')\n",
    "x = v.fit_transform(only_malicious['url'])\n",
    "y = v.fit_transform(only_benign['url'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5c58316b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_scores(vectorizer, tfidf_result_malicious, tfidf_result_benign):\n",
    "    # http://stackoverflow.com/questions/16078015/\n",
    "    scores_malicious = set(zip(vectorizer.get_feature_names(),\n",
    "                 np.asarray(tfidf_result_malicious.sum(axis=0)).ravel()))\n",
    "    scores_benign = set(zip(vectorizer.get_feature_names(),\n",
    "                 np.asarray(tfidf_result_benign.sum(axis=0)).ravel()))\n",
    "    sorted_strictly_malicious = sorted(scores_malicious.difference(scores_benign), key = lambda x: x[1], reverse=True)\n",
    "    for item in sorted_strictly_malicious:\n",
    "        print(\"{0:50} Score: {1}\".format(item[0], item[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6bc5ac1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                  Score: 52138.096944768746\n",
      "r                                                  Score: 49592.12282723011\n",
      "<                                                  Score: 43611.67457081842\n",
      "|                                                  Score: 41787.367735695625\n",
      "v                                                  Score: 39413.517236784035\n",
      "n                                                  Score: 36614.993137798556\n",
      ";                                                  Score: 35685.594248607595\n",
      "                                                  Score: 35407.99891787387\n",
      "{                                                  Score: 34415.03676393541\n",
      "}                                                  Score: 32125.44500635107\n",
      "p                                                  Score: 31633.79738831132\n",
      "                                                  Score: 30655.897049646606\n",
      "                                                  Score: 30322.87397156014\n",
      "z                                                  Score: 29917.71497167206\n",
      "y                                                  Score: 27682.91874181926\n",
      "q                                                  Score: 25266.242606745258\n",
      "u                                                  Score: 24514.753940642255\n",
      "",
      "                                                  Score: 19330.22382407169\n",
      "d                                                  Score: 17922.113566282926\n",
      ":                                                  Score: 17605.0163213493\n",
      "t                                                  Score: 17530.850329019457\n",
      "o                                                  Score: 16073.931240167638\n",
      ">                                                  Score: 14864.55547288643\n",
      "s                                                  Score: 14329.545372445813\n",
      "3                                                  Score: 13505.612567169572\n",
      "?                                                  Score: 12747.416962229541\n",
      "x                                                  Score: 12082.399799142608\n",
      "=                                                  Score: 11881.1748717274\n",
      "a                                                  Score: 11696.758399482544\n",
      "                                                  Score: 11528.397532564702\n",
      "                                                  Score: 11298.514402582245\n",
      "                                                  Score: 10574.374553679274\n",
      "@                                                  Score: 9637.338972844038\n",
      "[                                                  Score: 9115.22493711665\n",
      "\\                                                  Score: 8796.030400546997\n",
      "_                                                  Score: 8288.43052654816\n",
      "`                                                  Score: 8253.594028694537\n",
      "]                                                  Score: 8034.887923372116\n",
      "w                                                  Score: 7940.5413694913095\n",
      "                                                  Score: 7700.206647056202\n",
      "^                                                  Score: 7546.503149202838\n",
      "l                                                  Score: 6311.598810411728\n",
      "f                                                  Score: 5569.144136942781\n",
      "2                                                  Score: 4846.373444098721\n",
      "~                                                  Score: 3363.7003393955633\n",
      "                                                  Score: 1736.9840436955265\n",
      "b                                                  Score: 717.5317462216744\n",
      "9                                                  Score: 362.6522256377617\n",
      "i                                                  Score: 291.7954904606723\n",
      "8                                                  Score: 290.9201583727623\n",
      "4                                                  Score: 190.27572565677863\n",
      "g                                                  Score: 179.02748976903277\n",
      "0                                                  Score: 125.11430289791531\n",
      ".                                                  Score: 74.45072590036015\n",
      "6                                                  Score: 62.70479805309341\n",
      "5                                                  Score: 62.26809395922193\n",
      "                                                  Score: 48.522179154337934\n",
      "-                                                  Score: 25.946593812713093\n",
      "1                                                  Score: 21.185334828816075\n",
      "j                                                  Score: 19.22290010736628\n",
      "h                                                  Score: 19.074256208469215\n",
      "                                                  Score: 17.543986967708015\n",
      "                                                  Score: 16.87300062091451\n",
      "7                                                  Score: 12.45263459226127\n",
      "š                                                  Score: 11.436316432388383\n",
      "ž                                                  Score: 9.826639590400715\n",
      "•                                                  Score: 6.618900154263505\n",
      "¼                                                  Score: 6.421289272032842\n",
      "                                                  Score: 6.238490539855807\n",
      "                                                  Score: 5.990385676845447\n",
      "…                                                  Score: 5.983018423855041\n",
      "—                                                  Score: 5.861830875824645\n",
      "“                                                  Score: 5.839933156143898\n",
      "™                                                  Score: 5.627091579971058\n",
      "”                                                  Score: 5.608601830542936\n",
      "‰                                                  Score: 5.481508790165133\n",
      "ƒ                                                  Score: 5.448153444703181\n",
      "€                                                  Score: 5.407177066238751\n",
      "m                                                  Score: 5.120179237955677\n",
      "¥                                                  Score: 4.898294420364909\n",
      "’                                                  Score: 4.614693119065934\n",
      "–                                                  Score: 4.596548322922764\n",
      "ý                                                  Score: 4.576961411281502\n",
      "                                                  Score: 4.516642463423799\n",
      "‚                                                  Score: 4.412598600256772\n",
      "k                                                  Score: 4.407662208467637\n",
      "ø                                                  Score: 4.272134571009285\n",
      "/                                                  Score: 4.032309776102797\n",
      "                                                   Score: 4.029612195063534\n",
      "ÿ                                                  Score: 4.012135503139995\n",
      "õ                                                  Score: 3.9896064419817563\n",
      "ú                                                  Score: 3.9277112822133766\n",
      "ô                                                  Score: 3.9056721726453802\n",
      "\b                                                  Score: 3.8941545439464815\n",
      "\u001a                                                  Score: 3.7526824736692457\n",
      "c                                                  Score: 3.7202014364055276\n",
      "ò                                                  Score: 3.71153077573114\n",
      "î                                                  Score: 3.704459138440476\n",
      "(                                                  Score: 3.692213187543483\n",
      ")                                                  Score: 3.679274696063664\n",
      "\u001f                                                  Score: 3.658279990003686\n",
      "û                                                  Score: 3.611209894478881\n",
      "e                                                  Score: 3.6102989734910884\n",
      "ê                                                  Score: 3.5162245450503793\n",
      "¶                                                  Score: 3.5065039143637757\n",
      "°                                                  Score: 3.445097463813571\n",
      "ö                                                  Score: 3.4039545087903567\n",
      "                                                  Score: 3.378235307545922\n",
      "\u001c",
      "                                                  Score: 3.36540608021293\n",
      "¸                                                  Score: 3.333273932481869\n",
      "ó                                                  Score: 3.3041680347453117\n",
      "é                                                  Score: 3.2998204141270344\n",
      "ù                                                  Score: 3.260307630056323\n",
      "\u0019                                                  Score: 3.2429500721597146\n",
      "ñ                                                  Score: 3.2325151917801773\n",
      "±                                                  Score: 3.2229767194201546\n",
      "\u0016                                                  Score: 3.220067124008916\n",
      "\u001d",
      "                                                  Score: 3.208814096041728\n",
      "'                                                  Score: 3.2081851591226984\n",
      "´                                                  Score: 3.1595818370536537\n",
      "\u001e",
      "                                                  Score: 3.150116486501201\n",
      "\u0005                                                  Score: 2.9714905216730214\n",
      "ã                                                  Score: 2.963852613089088\n",
      "!                                                  Score: 2.9615248798289744\n",
      "æ                                                  Score: 2.9457222439471984\n",
      "®                                                  Score: 2.9407929814764935\n",
      "­                                                  Score: 2.932470784698399\n",
      "\u0014                                                  Score: 2.8933984030063096\n",
      "$                                                  Score: 2.866119332804519\n",
      "ë                                                  Score: 2.8380515127710275\n",
      "²                                                  Score: 2.82181716198604\n",
      "è                                                  Score: 2.8040748241244042\n",
      "½                                                  Score: 2.760597101145586\n",
      "\u0001                                                  Score: 2.7603131187109256\n",
      "á                                                  Score: 2.748255373636439\n",
      "¨                                                  Score: 2.709176532811889\n",
      "ç                                                  Score: 2.7063191581516834\n",
      "                                                   Score: 2.7051762785051987\n",
      "ß                                                  Score: 2.692125344207226\n",
      "º                                                  Score: 2.680000572647185\n",
      "×                                                  Score: 2.6726821945440666\n",
      "\u000f                                                  Score: 2.6651050437971686\n",
      "³                                                  Score: 2.638033097925466\n",
      "\u0002                                                  Score: 2.632617116240167\n",
      "ì                                                  Score: 2.629018098391824\n",
      "\t                                                  Score: 2.6242428332908236\n",
      "                                                  Score: 2.6221300314985037\n",
      "à                                                  Score: 2.603777102380183\n",
      "ª                                                  Score: 2.6022938307636028\n",
      "ä                                                  Score: 2.5893547284686753\n",
      ",                                                  Score: 2.552208908311689\n",
      "¿                                                  Score: 2.549359723409032\n",
      "â                                                  Score: 2.5419448707657555\n",
      "%                                                  Score: 2.494747069316238\n",
      "                                                  Score: 2.485387928864873\n",
      "ï                                                  Score: 2.4794313486586743\n",
      "#                                                  Score: 2.4070805197653824\n",
      "&                                                  Score: 2.3958846791659965\n",
      "*                                                  Score: 2.386972460070355\n",
      "ü                                                  Score: 2.3841607930864686\n",
      "£                                                  Score: 2.38221372755896\n",
      "©                                                  Score: 2.370163162966792\n",
      "ð                                                  Score: 2.3691818592754177\n",
      "·                                                  Score: 2.331941500846907\n",
      "¬                                                  Score: 2.3255722275766244\n",
      "                                                  Score: 2.324144516805439\n",
      "\u0004                                                  Score: 2.243921557045221\n",
      "å                                                  Score: 2.2314912670247407\n",
      "                                                  Score: 2.2152409283729178\n",
      "                                                  Score: 2.1979365429815005\n",
      "¦                                                  Score: 2.168159839361218\n",
      "¢                                                  Score: 2.081429059040334\n",
      "í                                                  Score: 2.078999826891884\n",
      "+                                                  Score: 1.9348352245152434\n",
      "                                                  Score: 1.9057944288033473\n",
      "¡                                                  Score: 1.879623195761884\n",
      "\u0011                                                  Score: 1.8610567438615666\n",
      "\u0006                                                  Score: 1.7984461427782308\n",
      "»                                                  Score: 1.772177666765184\n",
      "÷                                                  Score: 1.6922066785809013\n",
      "§                                                  Score: 1.3390995431483836\n",
      "\u0018                                                  Score: 0.38959723500355353\n",
      "\u0013                                                  Score: 0.3388664839610687\n"
     ]
    }
   ],
   "source": [
    "# displaying top features of malicious urls\n",
    "display_scores(v, x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dc023aa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_suspicious_chars(row):\n",
    "    unicode_count, inequality_count, pipe_count, semicolon_count, curlybrace_count, caret_count, hyphen_count, underscore_count = 0, 0, 0, 0, 0, 0, 0, 0\n",
    "    for char in row['url']:\n",
    "        if str(type(char)) == \"<type 'unicode'>\":\n",
    "            unicode_count += 1\n",
    "        elif char == '<' or char == '>':\n",
    "            inequality_count += 1\n",
    "        elif char == '|':\n",
    "            pipe_count += 1\n",
    "        elif char == ';':\n",
    "            semicolon_count += 1\n",
    "        elif char == '{' or char == '}':\n",
    "            curlybrace_count += 1\n",
    "        elif char == '^':\n",
    "            caret_count += 1\n",
    "        elif char == '-':\n",
    "            hyphen_count += 1\n",
    "        elif char == '_':\n",
    "            underscore_count += 1\n",
    "    \n",
    "    consecutive_num_count = str(max(re.findall(r\"\\d+\", row['url']), key=len, default=0))\n",
    "    if consecutive_num_count == '0':\n",
    "        consecutive_num_count = 0\n",
    "    else:\n",
    "        consecutive_num_count = len(consecutive_num_count)\n",
    "\n",
    "    return unicode_count, inequality_count, pipe_count, semicolon_count, curlybrace_count, caret_count, \\\n",
    "consecutive_num_count, hyphen_count, underscore_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "427b1994",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_counts = url_df.apply(lambda row: count_suspicious_chars(row), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f1ff86ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0         (0, 0, 0, 0, 0, 0, 0, 1, 0)\\n1         (0, 0, 0, 0, 0, 0, 1, 0, 1)\\n2         (0, 0, 0, 0, 0, 0, 1, 0, 0)\\n3         (0, 0, 0, 0, 0, 0, 2, 1, 2)\\n4         (0, 0, 0, 0, 0, 0, 3, 1, 1)\\n                     ...             \\n651186    (0, 0, 0, 0, 0, 0, 6, 0, 0)\\n651187    (0, 0, 0, 0, 0, 0, 4, 2, 0)\\n651188    (0, 0, 0, 0, 0, 0, 3, 0, 0)\\n651189    (0, 0, 0, 0, 0, 0, 0, 0, 3)\\n651190    (0, 0, 0, 0, 0, 0, 0, 0, 0)\\nLength: 651191, dtype: object'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repr(all_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "01fa6a7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "url_df['unicode'] = [tup[0] for tup in all_counts]\n",
    "url_df['inequality'] = [tup[1] for tup in all_counts]\n",
    "url_df['pipe'] = [tup[2] for tup in all_counts]\n",
    "url_df['semicolon'] = [tup[3] for tup in all_counts]\n",
    "url_df['curlybrace'] = [tup[4] for tup in all_counts]\n",
    "url_df['caret'] = [tup[5] for tup in all_counts]\n",
    "url_df['consecutive_num'] = [tup[6] for tup in all_counts]\n",
    "url_df['hyphen'] = [tup[7] for tup in all_counts]\n",
    "url_df['underscore'] = [tup[8] for tup in all_counts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "02a8f322",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>type</th>\n",
       "      <th>unicode</th>\n",
       "      <th>inequality</th>\n",
       "      <th>pipe</th>\n",
       "      <th>semicolon</th>\n",
       "      <th>curlybrace</th>\n",
       "      <th>caret</th>\n",
       "      <th>consecutive_num</th>\n",
       "      <th>hyphen</th>\n",
       "      <th>underscore</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>br-icloud.com.br</td>\n",
       "      <td>phishing</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mp3raid.com/music/krizz_kaliko.html</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bopsecrets.org/rexroth/cr/1.htm</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://www.garage-pirenne.be/index.php?option=...</td>\n",
       "      <td>defacement</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://adventure-nicaragua.net/index.php?optio...</td>\n",
       "      <td>defacement</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651186</th>\n",
       "      <td>xbox360.ign.com/objects/850/850402.html</td>\n",
       "      <td>phishing</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651187</th>\n",
       "      <td>games.teamxbox.com/xbox-360/1860/Dead-Space/</td>\n",
       "      <td>phishing</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651188</th>\n",
       "      <td>www.gamespot.com/xbox360/action/deadspace/</td>\n",
       "      <td>phishing</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651189</th>\n",
       "      <td>en.wikipedia.org/wiki/Dead_Space_(video_game)</td>\n",
       "      <td>phishing</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651190</th>\n",
       "      <td>www.angelfire.com/goth/devilmaycrytonite/</td>\n",
       "      <td>phishing</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>651191 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      url        type  \\\n",
       "0                                        br-icloud.com.br    phishing   \n",
       "1                     mp3raid.com/music/krizz_kaliko.html      benign   \n",
       "2                         bopsecrets.org/rexroth/cr/1.htm      benign   \n",
       "3       http://www.garage-pirenne.be/index.php?option=...  defacement   \n",
       "4       http://adventure-nicaragua.net/index.php?optio...  defacement   \n",
       "...                                                   ...         ...   \n",
       "651186            xbox360.ign.com/objects/850/850402.html    phishing   \n",
       "651187       games.teamxbox.com/xbox-360/1860/Dead-Space/    phishing   \n",
       "651188         www.gamespot.com/xbox360/action/deadspace/    phishing   \n",
       "651189      en.wikipedia.org/wiki/Dead_Space_(video_game)    phishing   \n",
       "651190          www.angelfire.com/goth/devilmaycrytonite/    phishing   \n",
       "\n",
       "        unicode  inequality  pipe  semicolon  curlybrace  caret  \\\n",
       "0             0           0     0          0           0      0   \n",
       "1             0           0     0          0           0      0   \n",
       "2             0           0     0          0           0      0   \n",
       "3             0           0     0          0           0      0   \n",
       "4             0           0     0          0           0      0   \n",
       "...         ...         ...   ...        ...         ...    ...   \n",
       "651186        0           0     0          0           0      0   \n",
       "651187        0           0     0          0           0      0   \n",
       "651188        0           0     0          0           0      0   \n",
       "651189        0           0     0          0           0      0   \n",
       "651190        0           0     0          0           0      0   \n",
       "\n",
       "        consecutive_num  hyphen  underscore  \n",
       "0                     0       1           0  \n",
       "1                     1       0           1  \n",
       "2                     1       0           0  \n",
       "3                     2       1           2  \n",
       "4                     3       1           1  \n",
       "...                 ...     ...         ...  \n",
       "651186                6       0           0  \n",
       "651187                4       2           0  \n",
       "651188                3       0           0  \n",
       "651189                0       0           3  \n",
       "651190                0       0           0  \n",
       "\n",
       "[651191 rows x 11 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3de8aacc",
   "metadata": {},
   "outputs": [],
   "source": [
    "url_df.to_csv(\"processed_urls.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d176d87f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}