weak_recsys_goodreads_children.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weakly Supervised Recommendation Systems"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiments steps:\n",
    " 1. **User's Preferences Model**: Leverage the most *explicit* ratings to build a *rate/rank prediction model*. This is a simple *Explicit Matrix Factorization* model. \n",
    " 2. **Generate Weak DataSet**: Use the above model to *predict* for all user/item pairs $(u,i)$ in *implicit feedback dataset* to build a new *weak explicit dataset* $(u, i, r^*)$.\n",
    " 3. **Evaluate**: Use the intact test split in the most explicit feedback, in order to evaluate the performance of any model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explicit Model Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section contains all the experiments based on the explicit matrix factorization model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explicit Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 550,599 interactions of 144,609 users and 122,321 items\n",
      "Explicit dataset (VALID) contains 550,600 interactions of 144,609 users and 122,321 items\n",
      "Explicit dataset (TRAIN) contains 4,404,793 interactions of 144,609 users and 122,321 items\n",
      "Implicit dataset (READ) contains 5,663,348 interactions of 144,609 users and 122,321 items\n",
      "Implicit dataset (SHELVE) contains 7,926,668 interactions of 144,609 users and 122,321 items\n",
      "--------------------\n",
      "RMSE: 0.4063\n",
      "MRR: 0.0271\n",
      "nDCG: 0.0346\n",
      "nDCG@10: 0.0239\n",
      "nDCG@5: 0.0139\n",
      "MAP: 0.0141\n",
      "success@10: 0.0952\n",
      "success@5: 0.0442\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from spotlight.evaluation import rmse_score\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_shelve = utils.parse_goodreads(\n",
    "    path='/local/terrier/Collections/Recommendations/Goodreads/goodreads_interactions_children.json.gz')\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (SHELVE) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_shelve.ratings), ','),\n",
    "          format(dataset_shelve.num_users, ','),\n",
    "          format(dataset_shelve.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_explicit(train_interactions=dataset_recommend_train, \n",
    "                             valid_interactions=dataset_recommend_dev,\n",
    "                             run_name='model_goodreads_children_explicit_rate')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove all valid/test ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_interact = set()\n",
    "for (uid, iid) in zip(dataset_recommend_test.user_ids, dataset_recommend_test.item_ids):\n",
    "    test_interact.add((uid, iid))\n",
    "\n",
    "for (uid, iid) in zip(dataset_recommend_dev.user_ids, dataset_recommend_dev.item_ids):\n",
    "    test_interact.add((uid, iid))\n",
    "\n",
    "# clean implicit dataset from test/dev rating\n",
    "for idx, (uid, iid, r) in enumerate(zip(dataset_read.user_ids, dataset_read.item_ids, dataset_read.ratings)):\n",
    "    if (uid, iid) in test_interact:\n",
    "        dataset_read.ratings[idx] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explicit Read Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Leverage the **explicit rate model** trained at the previous section to annotate **missing values** in the **read** dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 start at:  Tue Apr 23 12:51:05 2019\n",
      "epoch 1 end at:  Tue Apr 23 12:55:38 2019\n",
      "RMSE: 0.4099\n",
      "epoch 2 start at:  Tue Apr 23 12:55:46 2019\n",
      "epoch 2 end at:  Tue Apr 23 13:00:22 2019\n",
      "RMSE: 0.4157\n",
      "--------------------\n",
      "RMSE: 0.4160\n",
      "MRR: 0.0833\n",
      "nDCG: 0.0726\n",
      "nDCG@10: 0.0641\n",
      "nDCG@5: 0.0548\n",
      "MAP: 0.0428\n",
      "success@10: 0.1862\n",
      "success@5: 0.1366\n"
     ]
    }
   ],
   "source": [
    "# annotate the missing values in the play dataset based on the explicit recommend model\n",
    "dataset_read = utils.annotate(interactions=dataset_read, \n",
    "                              model=model, \n",
    "                              run_name='dataset_goodreads_children_read_explicit_annotated')\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_explicit(train_interactions=dataset_read, \n",
    "                             valid_interactions=dataset_recommend_dev,\n",
    "                             run_name='model_goodreads_children_explicit_read')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implicit Model Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section contains all the experiments based on the implicit matrix factorization model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implicit Model using Negative Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 550,599 interactions of 144,609 users and 122,321 items\n",
      "Explicit dataset (VALID) contains 550,600 interactions of 144,609 users and 122,321 items\n",
      "Explicit dataset (TRAIN) contains 4,404,793 interactions of 144,609 users and 122,321 items\n",
      "Implicit dataset (READ) contains 5,663,348 interactions of 144,609 users and 122,321 items\n",
      "Implicit dataset (SHELVE) contains 7,926,668 interactions of 144,609 users and 122,321 items\n",
      "epoch 1 start at:  Sat Apr 20 13:30:52 2019\n",
      "epoch 1 end at:  Sat Apr 20 13:36:11 2019\n",
      "MRR: 0.1126\n",
      "epoch 2 start at:  Sat Apr 20 14:28:33 2019\n",
      "epoch 2 end at:  Sat Apr 20 14:32:54 2019\n",
      "MRR: 0.1129\n",
      "epoch 3 start at:  Sat Apr 20 15:19:30 2019\n",
      "epoch 3 end at:  Sat Apr 20 15:23:50 2019\n",
      "MRR: 0.1131\n",
      "epoch 4 start at:  Sat Apr 20 16:10:24 2019\n",
      "epoch 4 end at:  Sat Apr 20 16:14:44 2019\n",
      "MRR: 0.1129\n",
      "--------------------\n",
      "RMSE: 4.0915\n",
      "MRR: 0.1127\n",
      "nDCG: 0.1072\n",
      "nDCG@10: 0.0875\n",
      "nDCG@5: 0.0694\n",
      "MAP: 0.0606\n",
      "success@10: 0.2578\n",
      "success@5: 0.1682\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from spotlight.evaluation import rmse_score\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_shelve = utils.parse_goodreads(\n",
    "    path='/local/terrier/Collections/Recommendations/Goodreads/goodreads_interactions_children.json.gz')\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (SHELVE) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_shelve.ratings), ','),\n",
    "          format(dataset_shelve.num_users, ','),\n",
    "          format(dataset_shelve.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_implicit_negative_sampling(train_interactions=dataset_read, \n",
    "                                               valid_interactions=dataset_recommend_dev,\n",
    "                                               run_name='model_goodreads_children_implicit')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Popularity Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 550,599 interactions of 144,609 users and 122,321 items\n",
      "Explicit dataset (VALID) contains 550,600 interactions of 144,609 users and 122,321 items\n",
      "Explicit dataset (TRAIN) contains 4,404,793 interactions of 144,609 users and 122,321 items\n",
      "Implicit dataset (READ) contains 5,663,348 interactions of 144,609 users and 122,321 items\n",
      "Implicit dataset (SHELVE) contains 7,926,668 interactions of 144,609 users and 122,321 items\n",
      "fit the model\n",
      "evaluate the model\n",
      "MRR: 0.1122\n",
      "nDCG: 0.1067\n",
      "nDCG@10: 0.0872\n",
      "nDCG@5: 0.0691\n",
      "MAP: 0.0605\n",
      "success@10: 0.2560\n",
      "success@5: 0.1662\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from popularity import PopularityModel\n",
    "from spotlight.evaluation import rmse_score\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_shelve = utils.parse_goodreads(\n",
    "    path='/local/terrier/Collections/Recommendations/Goodreads/goodreads_interactions_children.json.gz')\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (SHELVE) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_shelve.ratings), ','),\n",
    "          format(dataset_shelve.num_users, ','),\n",
    "          format(dataset_shelve.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = PopularityModel()\n",
    "print('fit the model')\n",
    "model.fit(interactions=dataset_recommend_train)\n",
    "\n",
    "# evaluate the new model\n",
    "print('evaluate the model')\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "# rmse = rmse_score(model=model, test=dataset_recommend_test, batch_size=512)\n",
    "# print('-'*20)\n",
    "# print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}