CHIMP_DataLoader.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "import os\n",
    "import torch\n",
    "\n",
    "import random\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from PIL import Image\n",
    "from skimage import io\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "verbose = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Chimp_Dataset(Dataset):\n",
    "    def __init__(self, dat_folder,transform,percent,train = True):\n",
    "        super(Chimp_Dataset, self).__init__()\n",
    "        self.transform = transform\n",
    "        self.percent = percent\n",
    "        self.root_dir = os.path.dirname(dat_folder) \n",
    "        \n",
    "        self.csv_file_train = os.path.join(self.root_dir, \"line_data_Train.csv\")\n",
    "        self.csv_file_test = os.path.join(self.root_dir, \"line_data_Test.csv\") \n",
    "        self.num_ind_file = os.path.join(self.root_dir,\"num_ind_file.csv\")\n",
    "        \n",
    "        num_ind = []\n",
    "        \n",
    "        if not os.path.basename(self.csv_file_train) in os.listdir(self.root_dir) or not os.path.basename(self.csv_file_test)  in os.listdir(self.root_dir): \n",
    "        #list of folders \n",
    "            with open(os.path.join(self.root_dir,'annotations_czoo.txt'),\"r\") as f:          \n",
    "                x_split= [x.split(' ') for x in f.readlines()]\n",
    "\n",
    "            filenames = [x[1] for x in x_split]\n",
    "            ind = [x[3] for x in x_split]\n",
    "            known_ind = []           \n",
    "            \n",
    "            for n_i,i in enumerate(ind):                \n",
    "                if not i in known_ind:\n",
    "                    position_i = [nn for nn,n in enumerate(ind) if n == i]\n",
    "                    N = len(position_i)*0.8\n",
    "                    if N > 3:\n",
    "                        random.shuffle(position_i)\n",
    "                \n",
    "                        line_list = []\n",
    "                        for j in range(int(N)):                            \n",
    "                            if os.path.isfile(os.path.join(self.root_dir,filenames[position_i[j]])):\n",
    "                                image_filename = filenames[position_i[j]]\n",
    "                                if verbose: print(\"image {}{}\".format(j,image_filename))\n",
    "                                line = [str(j),image_filename,str(n_i)]\n",
    "                                line_list.append([line])\n",
    "                        \n",
    "                        with open(self.csv_file_train,\"a\") as f:\n",
    "                            writer = csv.writer(f,delimiter=',')    \n",
    "                            for lines in line_list:\n",
    "                                writer.writerows(lines)\n",
    "                                \n",
    "                        line_list = []\n",
    "                        for j in range(int(N),len(position_i)):\n",
    "                            if os.path.isfile(os.path.join(self.root_dir,filenames[position_i[j]])):\n",
    "                                image_filename = filenames[position_i[j]]\n",
    "                                if verbose: print(\"image {}{}\".format(j,image_filename))\n",
    "                                line = [str(j),image_filename,str(n_i)]\n",
    "                                line_list.append([line])\n",
    "                        \n",
    "                        with open(self.csv_file_test,\"a\") as f:\n",
    "                            writer = csv.writer(f,delimiter=',')     \n",
    "                            for lines in line_list:\n",
    "                                writer.writerows(lines)\n",
    "                        \n",
    "                    known_ind.append(i)\n",
    "                    num_ind.append(1)\n",
    "                else:\n",
    "                    n = known_ind.index(i)\n",
    "                    num_ind[n] += 1\n",
    "                        \n",
    "            print('individuals {}'.format(len(known_ind)))\n",
    "            print('num_ind {}'.format(len(num_ind)))\n",
    "            self.num_ind = num_ind\n",
    "            lines = [[str(l),str(self.num_ind[l])] for l in range(len(self.num_ind))]\n",
    "            with open(self.num_ind_file,\"w\") as f:\n",
    "                writer = csv.writer(f,delimiter =',')\n",
    "                for line in lines:\n",
    "                    writer.writerows([line])\n",
    "        else:\n",
    "            num_ind_file = pd.read_csv(self.num_ind_file)\n",
    "            for l in range(len(num_ind_file)):\n",
    "                num_ind.append(int(num_ind_file.iloc[l,1]))\n",
    "            self.num_ind = num_ind\n",
    "        if train: self.CHIM_datafile = pd.read_csv(self.csv_file_train)\n",
    "        else: self.CHIM_datafile = pd.read_csv(self.csv_file_test) \n",
    "            \n",
    "    def __len__(self):\n",
    "        return len(self.CHIM_datafile)\n",
    "    \n",
    "    def __getitem__(self,idx):\n",
    "        img_name = self.CHIM_datafile.iloc[idx,1]\n",
    "        image = self.__loadfile(os.path.join(self.root_dir,img_name))\n",
    "        target = self.CHIM_datafile.iloc[idx,2]\n",
    "        if self.transform:\n",
    "            image = Image.fromarray(image)\n",
    "            sample = self.transform(image)\n",
    "        else:\n",
    "            sample = image\n",
    "        return (sample,target)\n",
    "    \n",
    "    def __loadfile(self, data_file):\n",
    "        image = io.imread(data_file)\n",
    "        if len(image.shape)<3:\n",
    "            image = np.stack((image,)*3, axis=-1)\n",
    "        return image\n",
    "    \n",
    "    def prepare_batch(self,percent = 0.2):\n",
    "\n",
    "        train_line_list = []        \n",
    "        labels_train = []        \n",
    "        known_labels = []\n",
    "        \n",
    "\n",
    "        for i in range(len(self.CHIM_datafile)):\n",
    "            label = self.CHIM_datafile.iloc[i,2]\n",
    "\n",
    "            min_numind = int(np.min(self.num_ind)*percent)\n",
    "            if not label in known_labels: \n",
    "                position_i = [nn for nn in range(len(self.CHIM_datafile)) if self.CHIM_datafile.iloc[nn,2] == label]\n",
    "\n",
    "                N = len(position_i)      \n",
    "                if N > min_numind:\n",
    "                    random.shuffle(position_i)\n",
    "\n",
    "                    for j in range(min_numind):\n",
    "                        if os.path.isfile(os.path.join(self.root_dir,self.CHIM_datafile.iloc[j,1])):\n",
    "                            image_filename = self.CHIM_datafile.iloc[j,1]\n",
    "                            if verbose: print(\"image {}{}\".format(j,image_filename))\n",
    "                            train_line_list.append(image_filename)\n",
    "                            labels_train.append(int(label))\n",
    "\n",
    "                    known_labels.append(label)\n",
    "                #print('j {}'.format(j))\n",
    "    \n",
    "        image_train = [self.transform(Image.fromarray(self.__loadfile(os.path.join(self.root_dir,train_line_list[i])))) for i in range(len(train_line_list))]  \n",
    "        return image_train, labels_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#filename = '/home/rita/JupyterProjects/EYE-SEA/DataSets/Verification/chimpanzee_faces-master/datasets_cropped_chimpanzee_faces/data_CZoo/annotations_czoo.txt'\n",
    "#transform = transforms.Compose([\n",
    "#    transforms.Resize((224,224)),\n",
    "#    transforms.ToTensor(),        \n",
    "#    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261))\n",
    "#])\n",
    "#c = Chimp_Dataset(filename,transform,0.8)\n",
    "#c_t = Chimp_Dataset(filename,transform,0.8,train=False)\n",
    "#image_train, labels_train = c.prepare_batch(0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import import_ipynb\n",
    "#import ResNetCaps_E\n",
    "#import losses\n",
    "\n",
    "#import torch.nn as nn\n",
    "#import torch.optim as optim\n",
    "#from torch.optim import lr_scheduler\n",
    "#from torch.optim import Adam\n",
    "#import torch.nn.functional as F\n",
    "\n",
    "#device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "#model = ResNetCaps_E.ResNetCaps_E(DigitEnd=True)\n",
    "\n",
    "#if torch.cuda.device_count() > 1:\n",
    "#    print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\n",
    "  # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs\n",
    "#    model = nn.DataParallel(model)\n",
    "#model = model.to(device)\n",
    "#optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),lr = 0.001)\n",
    "#criterion = losses.HAP2STripletLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#image_train = torch.stack(image_train)\n",
    "#image_train = image_train.to(device)\n",
    "#labels_train = torch.Tensor(labels_train).to(device)\n",
    "\n",
    "#emb_a = model(image_train)\n",
    "#emb_a = emb_a.view(image_train.size(0),-1)\n",
    "#optimizer.zero_grad()\n",
    "#loss = criterion(emb_a.squeeze(),labels_train)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}