autophrase_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AutoPhrase vs. ECON Extracted Concept Results Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "\n",
    "from common.constants import file_paths\n",
    "from common.constants import file_suffixes\n",
    "from common.constants import regex_patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file_path(file_suffix):\n",
    "    extensionless_input_file_name = file_paths.INPUT_FILE.split('.')[0]\n",
    "    return f'{file_paths.DATA_DIR}/{extensionless_input_file_name}{file_suffix}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "econ_embedding_path = get_file_path(file_suffixes.ECON_EMBEDDING)\n",
    "autophrase_results_path = get_file_path(file_suffixes.AUTOPHRASE_RESULTS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 4413 AutoPhrase concepts.\n"
     ]
    }
   ],
   "source": [
    "with open(autophrase_results_path, 'r') as f:\n",
    "    autophrase_concepts = f.read().split('\\n')[:-1]\n",
    "    f.close()\n",
    "print(f'Loaded {len(autophrase_concepts)} AutoPhrase concepts.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_raw_concept(tagged_econ_concept):\n",
    "    return tagged_econ_concept.replace('_', ' ')[3:-4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 4473 ECON concepts.\n"
     ]
    }
   ],
   "source": [
    "model = Word2Vec.load(econ_embedding_path)\n",
    "econ_concepts = [get_raw_concept(w) for w in model.wv.index2word if regex_patterns.CONCEPT_TAGGED_PHRASE.match(w)]\n",
    "print(f'Loaded {len(econ_concepts)} ECON concepts.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "autophrase_concept_set = set(autophrase_concepts)\n",
    "econ_concept_set = set(econ_concepts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Difference Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96 ECON concepts not found in AutoPhrase concepts.\n"
     ]
    }
   ],
   "source": [
    "print(f'{len(econ_concept_set - autophrase_concept_set)} ECON concepts not found in AutoPhrase concepts.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['computational performance',\n",
       " 'ability to capture',\n",
       " 'data efficient',\n",
       " 'supervised approaches',\n",
       " 'bayes classifier',\n",
       " 'structured learning',\n",
       " 'ensemble approach',\n",
       " 'convex problems',\n",
       " 'complex models',\n",
       " 'challenging problem',\n",
       " 'proposed algorithm',\n",
       " 'based method',\n",
       " 'task performance',\n",
       " 'connectionist temporal classification',\n",
       " 'non convex optimization',\n",
       " 'a long standing',\n",
       " 'learning objective',\n",
       " 'classification loss',\n",
       " 'connected layers',\n",
       " 'generative framework',\n",
       " 'deterministic policy',\n",
       " 'difficult to train',\n",
       " 'problems in machine learning',\n",
       " 'specific knowledge',\n",
       " 'learning task',\n",
       " 'and vice versa',\n",
       " 'pca based',\n",
       " 'score based',\n",
       " 'temporal data',\n",
       " 'machine learning models',\n",
       " 'number of model parameters',\n",
       " 'curriculum learning',\n",
       " 'accuracy and computational',\n",
       " 'dimensional representation',\n",
       " 'learning strategy',\n",
       " 'simple and effective',\n",
       " 'based regularization',\n",
       " 'prediction results',\n",
       " 'regression and classification',\n",
       " 'graphical models',\n",
       " 'deep model',\n",
       " 'number of training samples',\n",
       " 'deep recurrent neural network',\n",
       " 'vector representations',\n",
       " 'the paper presents',\n",
       " 'general classification',\n",
       " 'number of iterations',\n",
       " 'real life data',\n",
       " 'trained models',\n",
       " 'computer aided',\n",
       " 'clustering approach',\n",
       " 'boosting algorithm',\n",
       " 'learning representations',\n",
       " 'non trivial',\n",
       " 'learning process',\n",
       " 'classification models',\n",
       " 'ability to learn',\n",
       " 'similarity learning',\n",
       " 'selection methods',\n",
       " 'low dimensional representations',\n",
       " 'standard classification',\n",
       " 'classification regression',\n",
       " 'low noise',\n",
       " 'classification algorithm',\n",
       " 'data clustering',\n",
       " 'deep linear',\n",
       " 'linear activation',\n",
       " 'learning systems',\n",
       " 'non invasive',\n",
       " 'based optimization',\n",
       " 'patient data',\n",
       " 'image data',\n",
       " 'representations of words',\n",
       " 'publicly available datasets',\n",
       " 'specific task',\n",
       " 'selection algorithm',\n",
       " 'jointly learning',\n",
       " 'training distribution',\n",
       " 'synthetic and real world datasets',\n",
       " 'learning dynamics',\n",
       " 'classification methods',\n",
       " 'efficient online',\n",
       " 'trained model',\n",
       " 'cnn models',\n",
       " 'prediction methods',\n",
       " 'learning parameters',\n",
       " 'k svd',\n",
       " 'classification techniques',\n",
       " 'simulated and real world data',\n",
       " 'linear classification',\n",
       " 'each data point',\n",
       " 'recently achieved',\n",
       " 'models including',\n",
       " 'search algorithms',\n",
       " 'computer vision and machine learning',\n",
       " 'large data']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(econ_concept_set - autophrase_concept_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36 AutoPhrase concepts not found in ECON concepts.\n"
     ]
    }
   ],
   "source": [
    "print(f'{len(autophrase_concept_set - econ_concept_set)} AutoPhrase concepts not found in ECON concepts.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['optimal performance',\n",
       " 'next frame',\n",
       " 'recommender system',\n",
       " 'traditional machine learning',\n",
       " 'efficient solutions',\n",
       " 'high computational',\n",
       " 'norm based',\n",
       " 'o log',\n",
       " 'mathcal f',\n",
       " 'provide insights',\n",
       " 'dynamic bayesian',\n",
       " 'ell 2 norm',\n",
       " 'function estimation',\n",
       " 'approach combines',\n",
       " 'cnns trained',\n",
       " 'structured sparse',\n",
       " 'input distribution',\n",
       " 'pre specified',\n",
       " 'annotated training',\n",
       " 'training images',\n",
       " 'noise regime',\n",
       " 'top down',\n",
       " 'singular value',\n",
       " 'handle large',\n",
       " 'proposed method achieves',\n",
       " 'inductive learning',\n",
       " 'second order statistics',\n",
       " 'global information',\n",
       " 'renewed interest',\n",
       " 'action classification',\n",
       " 'sparse gaussian',\n",
       " 'framework outperforms',\n",
       " '100 000',\n",
       " 'bf x',\n",
       " 'improve accuracy',\n",
       " 'publicly available']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(autophrase_concept_set - econ_concept_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sub/Super-Sequence Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concept_contains_words_in_order(concept, words):\n",
    "    concept_words = set(concept.split())\n",
    "    for word in words:\n",
    "        if word not in concept_words:\n",
    "            return False\n",
    "    return ' '.join(words) in concept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concept_contains_concept(c1, c2):\n",
    "    return concept_contains_words_in_order(c1, c2.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_sub_super_sequence_comparison(results, for_econ):\n",
    "    first_method = 'AutoPhrase'\n",
    "    second_method = 'ECON'\n",
    "    if for_econ:\n",
    "        first_method = 'ECON'\n",
    "        second_method = 'AutoPhrase'\n",
    "    for result in results:\n",
    "        is_good_concept, c1, c2, is_subsequence = result\n",
    "        good_concept_text = 'BAD'\n",
    "        if is_good_concept:\n",
    "            good_concept_text = 'GOOD'\n",
    "        sequence_text = 'SUPERSEQUENCE'\n",
    "        if is_subsequence:\n",
    "            sequence_text = 'SUBSEQUENCE'\n",
    "        print(f'{good_concept_text : <5} | {first_method} Concept: {c1 : <40} | {second_method} concept: {c2 : <40} | {sequence_text}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECON\n",
    "\n",
    "Identify each ECON concept, `ec`, such that `ec` is a non-matching subsequence/supersequence of an AutoPhrase concept, `ac`, given that AutoPhrase did not extract `ec`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "temporal data 1\n",
      "computer vision and machine learning 0\n",
      "problems in machine learning 0\n",
      "machine learning models 1\n",
      "synthetic and real world datasets 1\n",
      "simulated and real world data 1\n",
      "non convex optimization 1\n",
      "based optimization 0\n",
      "deep recurrent neural network 1\n",
      "real life data 1\n",
      "publicly available datasets 1\n",
      "simulated and real world data 1\n",
      "graphical models 1\n",
      "synthetic and real world datasets 1\n",
      "computer vision and machine learning 0\n",
      "deep recurrent neural network 1\n",
      "and vice versa 0\n",
      "connected layers 1\n",
      "each data point 0\n",
      "simulated and real world data 1\n",
      "computer aided 0\n",
      "low dimensional representations 1\n",
      "graphical models 1\n",
      "simulated and real world data 1\n",
      "number of training samples 0\n",
      "bayes classifier 1\n",
      "synthetic and real world datasets 1\n",
      "graphical models 1\n",
      "deep recurrent neural network 1\n",
      "large data 0\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for ac in autophrase_concepts:\n",
    "    for ec in econ_concepts:\n",
    "        if ec != ac and ec not in autophrase_concept_set:\n",
    "            if concept_contains_concept(ac, ec):\n",
    "                is_good_concept = input(ec)\n",
    "                results.append((is_good_concept == '1', ec, ac, True))\n",
    "            elif concept_contains_concept(ec, ac):\n",
    "                is_good_concept = input(ec)\n",
    "                results.append((is_good_concept == '1', ec, ac, False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GOOD  | ECON Concept: temporal data                            | AutoPhrase concept: spatio temporal data                     | SUBSEQUENCE\n",
      "BAD   | ECON Concept: computer vision and machine learning     | AutoPhrase concept: machine learning                         | SUPERSEQUENCE\n",
      "BAD   | ECON Concept: problems in machine learning             | AutoPhrase concept: machine learning                         | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: machine learning models                  | AutoPhrase concept: machine learning                         | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: synthetic and real world datasets        | AutoPhrase concept: real world                               | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: simulated and real world data            | AutoPhrase concept: real world                               | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: non convex optimization                  | AutoPhrase concept: convex optimization                      | SUPERSEQUENCE\n",
      "BAD   | ECON Concept: based optimization                       | AutoPhrase concept: gradient based optimization              | SUBSEQUENCE\n",
      "GOOD  | ECON Concept: deep recurrent neural network            | AutoPhrase concept: neural network                           | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: real life data                           | AutoPhrase concept: real life                                | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: publicly available datasets              | AutoPhrase concept: publicly available                       | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: simulated and real world data            | AutoPhrase concept: simulated and real                       | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: graphical models                         | AutoPhrase concept: undirected graphical models              | SUBSEQUENCE\n",
      "GOOD  | ECON Concept: synthetic and real world datasets        | AutoPhrase concept: synthetic and real                       | SUPERSEQUENCE\n",
      "BAD   | ECON Concept: computer vision and machine learning     | AutoPhrase concept: computer vision                          | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: deep recurrent neural network            | AutoPhrase concept: recurrent neural                         | SUPERSEQUENCE\n",
      "BAD   | ECON Concept: and vice versa                           | AutoPhrase concept: vice versa                               | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: connected layers                         | AutoPhrase concept: fully connected layers                   | SUBSEQUENCE\n",
      "BAD   | ECON Concept: each data point                          | AutoPhrase concept: data point                               | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: simulated and real world data            | AutoPhrase concept: simulated and real world                 | SUPERSEQUENCE\n",
      "BAD   | ECON Concept: computer aided                           | AutoPhrase concept: computer aided diagnosis                 | SUBSEQUENCE\n",
      "GOOD  | ECON Concept: low dimensional representations          | AutoPhrase concept: low dimensional                          | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: graphical models                         | AutoPhrase concept: probabilistic graphical models           | SUBSEQUENCE\n",
      "GOOD  | ECON Concept: simulated and real world data            | AutoPhrase concept: real world data                          | SUPERSEQUENCE\n",
      "BAD   | ECON Concept: number of training samples               | AutoPhrase concept: training samples                         | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: bayes classifier                         | AutoPhrase concept: naive bayes classifier                   | SUBSEQUENCE\n",
      "GOOD  | ECON Concept: synthetic and real world datasets        | AutoPhrase concept: synthetic and real world                 | SUPERSEQUENCE\n",
      "GOOD  | ECON Concept: graphical models                         | AutoPhrase concept: gaussian graphical models                | SUBSEQUENCE\n",
      "GOOD  | ECON Concept: deep recurrent neural network            | AutoPhrase concept: recurrent neural network                 | SUPERSEQUENCE\n",
      "BAD   | ECON Concept: large data                               | AutoPhrase concept: large data sets                          | SUBSEQUENCE\n"
     ]
    }
   ],
   "source": [
    "print_sub_super_sequence_comparison(results, for_econ=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AutoPhrase\n",
    "\n",
    "Identify each AutoPhrase concept, `ac`, such that `ac` is a non-matching subsequence/supersequence of an ECON concept, `ec`, given that ECON did not extract `ac`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "traditional machine learning 1\n",
      "proposed method achieves 0\n",
      "proposed method achieves 0\n",
      "ell 2 norm 1\n",
      "publicly available 0\n",
      "high computational 0\n",
      "dynamic bayesian 0\n",
      "singular value 1\n",
      "singular value 1\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for ec in econ_concepts:\n",
    "    for ac in autophrase_concepts:\n",
    "        if ac != ec and ac not in econ_concept_set:\n",
    "            if concept_contains_concept(ec, ac):\n",
    "                is_good_concept = input(ac)\n",
    "                results.append((is_good_concept == '1', ac, ec, True))\n",
    "            elif concept_contains_concept(ac, ec):\n",
    "                is_good_concept = input(ac)\n",
    "                results.append((is_good_concept == '1', ac, ec, False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GOOD  | AutoPhrase Concept: traditional machine learning             | ECON concept: machine learning                         | SUPERSEQUENCE\n",
      "BAD   | AutoPhrase Concept: proposed method achieves                 | ECON concept: proposed method                          | SUPERSEQUENCE\n",
      "BAD   | AutoPhrase Concept: proposed method achieves                 | ECON concept: method achieves                          | SUPERSEQUENCE\n",
      "GOOD  | AutoPhrase Concept: ell 2 norm                               | ECON concept: ell 2                                    | SUPERSEQUENCE\n",
      "BAD   | AutoPhrase Concept: publicly available                       | ECON concept: publicly available datasets              | SUBSEQUENCE\n",
      "BAD   | AutoPhrase Concept: high computational                       | ECON concept: high computational cost                  | SUBSEQUENCE\n",
      "BAD   | AutoPhrase Concept: dynamic bayesian                         | ECON concept: dynamic bayesian networks                | SUBSEQUENCE\n",
      "GOOD  | AutoPhrase Concept: singular value                           | ECON concept: singular value decomposition svd         | SUBSEQUENCE\n",
      "GOOD  | AutoPhrase Concept: singular value                           | ECON concept: singular value decomposition             | SUBSEQUENCE\n"
     ]
    }
   ],
   "source": [
    "print_sub_super_sequence_comparison(results, for_econ=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "se",
   "language": "python",
   "name": "se"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}