filter_out_steam_banners.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "filter_out_steam_banners.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "nBui4AZhiI0N",
        "JYzoK9yAWejc",
        "WFmgWTgAcv1v",
        "OhNp7ZgMctA_",
        "I1JKvvgALadW",
        "pjUYtEyNXOAp",
        "s31qd7is5MMn",
        "TKBMk0qfj1Jw"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T_HzBn1HWOIx"
      },
      "source": [
        "# Filter out Steam banners"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nBui4AZhiI0N"
      },
      "source": [
        "## Utility functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N7_E2p5Cen8R"
      },
      "source": [
        "def get_input_file_name():\r\n",
        "  return 'original_vertical_steam_banners.tar'\r\n",
        "\r\n",
        "def get_output_resized_folder():\r\n",
        "  return 'resized_vertical_steam_banners'\r\n",
        "\r\n",
        "def get_gdrive_data_folder():\r\n",
        "  return 'data/'\r\n",
        "\r\n",
        "def get_local_data_folder():\r\n",
        "  return 'data/original_vertical_steam_banners/'\r\n",
        "\r\n",
        "def get_image_root_name():\r\n",
        "  return 'library_600x900'\r\n",
        "\r\n",
        "def get_image_extension():\r\n",
        "  return '.jpg'"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JYzoK9yAWejc"
      },
      "source": [
        "## Import data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WEY2ey3riCta",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4a55ab1e-923e-4ceb-ddb2-7d60beb811e7"
      },
      "source": [
        "%pip install google-colab-transfer"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting google-colab-transfer\n",
            "  Downloading https://files.pythonhosted.org/packages/a0/90/76fc38bcad442018977ed0e4e663473ef56a4d15395b2aa09055e8c49185/Google_Colab_Transfer-0.1.6-py3-none-any.whl\n",
            "Installing collected packages: google-colab-transfer\n",
            "Successfully installed google-colab-transfer-0.1.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "llpF55-bWHLA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fbe8dbef-484d-4017-fe06-e953028cd4e4"
      },
      "source": [
        "import colab_transfer as ct\n",
        "\n",
        "ct.mount_google_drive()\n",
        "\n",
        "file_name = get_input_file_name()\n",
        "\n",
        "gd = ct.get_path_to_home_of_google_drive()\n",
        "lm = ct.get_path_to_home_of_local_machine()\n",
        "\n",
        "ct.copy_file(file_name,\n",
        "             source = gd + get_gdrive_data_folder(),\n",
        "             destination = lm)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive/\n",
            "Copying /content/drive/My Drive/data/original_vertical_steam_banners.tar to /content/original_vertical_steam_banners.tar\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FGH0hK0fWvaH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bb731f2f-b8a3-4eb5-99f1-b5d0d6b9c476"
      },
      "source": [
        "from pathlib import Path\n",
        "\n",
        "if not Path(get_local_data_folder()).exists():\n",
        "  %cd /content/\n",
        "  !echo tar -xf {file_name}\n",
        "  !tar -xf {file_name}"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "tar -xf original_vertical_steam_banners.tar\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YwIllLgSavWn"
      },
      "source": [
        "## Boilerplate functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WFmgWTgAcv1v"
      },
      "source": [
        "### Disk utils"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5DycpLO3axh-"
      },
      "source": [
        "import glob\r\n",
        "\r\n",
        "def list_file_names(image_dir = None):\r\n",
        "  if image_dir is None:\r\n",
        "    image_dir = get_local_data_folder()\r\n",
        "\r\n",
        "  if not image_dir.endswith('/'):\r\n",
        "    image_dir += '/'\r\n",
        "\r\n",
        "  file_names = glob.glob(image_dir + '*' + get_image_extension())\r\n",
        "\r\n",
        "  return file_names"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qJqVtazYdpDU"
      },
      "source": [
        "def save_app_ids_to_file(output_file_name, \r\n",
        "                         image_dir = None):\r\n",
        "  file_names = list_file_names(image_dir)\r\n",
        "\r\n",
        "  app_ids = [\r\n",
        "            str(get_app_id(f)) for f in file_names\r\n",
        "  ]\r\n",
        "\r\n",
        "  app_ids = sorted(app_ids, key=int)\r\n",
        "\r\n",
        "  print('#appIDs = {}'.format(len(app_ids)))\r\n",
        "\r\n",
        "  with open(output_file_name, 'w') as f:\r\n",
        "    f.write('\\n'.join(app_ids))\r\n",
        "\r\n",
        "  return"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OhNp7ZgMctA_"
      },
      "source": [
        "### Parsing utils"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3hlV0AVeiVaC"
      },
      "source": [
        "from pathlib import Path\r\n",
        "\r\n",
        "def get_app_id(file_name):\r\n",
        "  return Path(file_name).name.replace(get_image_extension(), '')\r\n",
        "\r\n",
        "def get_fname(app_id):\r\n",
        "  return get_local_data_folder() + str(app_id) + get_image_extension()"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "woLefcApcQG9"
      },
      "source": [
        "def get_steamdb_url(file_name=None,\r\n",
        "                    app_id=None):\r\n",
        "\r\n",
        "  if app_id is None:\r\n",
        "    app_id = get_app_id(file_name)\r\n",
        "\r\n",
        "  return 'https://steamdb.info/app/{}'.format(app_id)\r\n",
        "\r\n",
        "def get_url(file_name=None,\r\n",
        "            app_id=None):\r\n",
        "\r\n",
        "  if app_id is None:\r\n",
        "    app_id = get_app_id(file_name)\r\n",
        "\r\n",
        "  base_url = 'https://cdn.cloudflare.steamstatic.com/steam/apps/{}/{}{}'\r\n",
        "\r\n",
        "  url = base_url.format(app_id,\r\n",
        "                        get_image_root_name(),\r\n",
        "                        get_image_extension())\r\n",
        "\r\n",
        "  return url  "
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zmtORGy_IUjj"
      },
      "source": [
        "### Detection utils"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FoSGN-k3IVNt"
      },
      "source": [
        "from PIL import Image\r\n",
        "\r\n",
        "def detect_issues_with_images(file_names,\r\n",
        "                              check_blank = False,\r\n",
        "                              check_image_size = True,\r\n",
        "                              check_channels = True,\r\n",
        "                              verbose=True):\r\n",
        "  # Caveat: if (check_blank == True), then code is very slow! Plus, in practice\r\n",
        "  #         here, it is useless as it overlaps with (check_channels == True).\r\n",
        "\r\n",
        "  app_ids_with_blank_image = [] # list of appIDs\r\n",
        "  d = dict() # bands -> list of appIDs\r\n",
        "  dd = dict() # image size -> list of appIDs\r\n",
        "\r\n",
        "  for i, f in enumerate(file_names, start=1):\r\n",
        "    app_id = get_app_id(f)\r\n",
        "    \r\n",
        "    img = Image.open(f)\r\n",
        "\r\n",
        "    ## Remove blank banners\r\n",
        "\r\n",
        "    if check_blank:\r\n",
        "      # Reference: https://stackoverflow.com/a/38749333    \r\n",
        "      extrema = img.convert(\"L\").getextrema()\r\n",
        "\r\n",
        "      if extrema[0] == extrema[1]:\r\n",
        "        app_ids_with_blank_image.append(app_id)\r\n",
        "\r\n",
        "    ## Remove banners with uncommon resolution (anything but 300x450)\r\n",
        "\r\n",
        "    if check_image_size:\r\n",
        "      sz = img.size\r\n",
        "\r\n",
        "      try:\r\n",
        "        dd[sz].append(app_id)\r\n",
        "      except KeyError:\r\n",
        "        dd[sz] = [app_id]\r\n",
        "\r\n",
        "    ## Remove banners with uncommon bands (anything but RGB)    \r\n",
        "    if check_channels:\r\n",
        "      channels = img.getbands()\r\n",
        "      channels_as_str = ''.join(channels)\r\n",
        "      \r\n",
        "      try:\r\n",
        "        d[channels_as_str].append(app_id)\r\n",
        "      except KeyError:\r\n",
        "        d[channels_as_str] = [app_id]\r\n",
        "\r\n",
        "    if verbose and i % 1000 == 0:\r\n",
        "      print('{}/{}'.format(i, len(file_names)))\r\n",
        "\r\n",
        "  return app_ids_with_blank_image, d, dd"
      ],
      "execution_count": 181,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I1JKvvgALadW"
      },
      "source": [
        "### Removal utils"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3JN7XMDYLe8x"
      },
      "source": [
        "from pathlib import Path\r\n",
        "import shutil\r\n",
        "\r\n",
        "def remove_app_ids(file_names,\r\n",
        "                   folder_other,\r\n",
        "                   app_ids_to_remove):\r\n",
        "  \r\n",
        "  for f in file_names:\r\n",
        "    app_id = get_app_id(f)\r\n",
        "\r\n",
        "    if app_id in app_ids_to_remove:\r\n",
        "\r\n",
        "      input_file_name = f\r\n",
        "      output_file_name = folder_other + '/' + str(app_id) + get_image_extension()\r\n",
        "\r\n",
        "      if Path(input_file_name).exists():\r\n",
        "        shutil.move(input_file_name, output_file_name)\r\n",
        "\r\n",
        "  fname = folder_other + '.txt'\r\n",
        "\r\n",
        "  save_app_ids_to_file(fname,\r\n",
        "                      image_dir=folder_other)\r\n",
        "\r\n",
        "  return"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pjUYtEyNXOAp"
      },
      "source": [
        "## Detect"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zEv8K0cyZ2m0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a0fc9e3e-6678-4b80-fa8f-96ba5cfd7a71"
      },
      "source": [
        "image_dir = 'data/original_vertical_steam_banners/'\r\n",
        "\r\n",
        "file_names = list_file_names(image_dir=image_dir)\r\n",
        "\r\n",
        "# Recommendation: use check_blank == False to go fast.\r\n",
        "#                 Plus, here, this check overlaps with 'check_channels' anyway.\r\n",
        "app_ids_with_blank_image, d, dd = detect_issues_with_images(file_names,\r\n",
        "                                                            check_blank=False)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000/29982\n",
            "2000/29982\n",
            "3000/29982\n",
            "4000/29982\n",
            "5000/29982\n",
            "6000/29982\n",
            "7000/29982\n",
            "8000/29982\n",
            "9000/29982\n",
            "10000/29982\n",
            "11000/29982\n",
            "12000/29982\n",
            "13000/29982\n",
            "14000/29982\n",
            "15000/29982\n",
            "16000/29982\n",
            "17000/29982\n",
            "18000/29982\n",
            "19000/29982\n",
            "20000/29982\n",
            "21000/29982\n",
            "22000/29982\n",
            "23000/29982\n",
            "24000/29982\n",
            "25000/29982\n",
            "26000/29982\n",
            "27000/29982\n",
            "28000/29982\n",
            "29000/29982\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXwZo20vKBmK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "902aeb93-698b-4691-a472-df1bdd772939"
      },
      "source": [
        "print(app_ids_with_blank_image)\r\n",
        "\r\n",
        "for k, v in d.items():\r\n",
        "  print(k, len(v))\r\n",
        "\r\n",
        "for k, v in dd.items():\r\n",
        "  print(k, len(v))  "
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[]\n",
            "RGB 29642\n",
            "L 306\n",
            "CMYK 34\n",
            "(300, 450) 29981\n",
            "(600, 900) 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s31qd7is5MMn"
      },
      "source": [
        "## Count"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5VJLVQZh0AdI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "57999b7a-14f5-40f3-f890-6522b0149f41"
      },
      "source": [
        "print(app_ids_with_blank_image)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SeICEUGTaB03",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1a79995f-5485-4b95-8ba3-e66cdc09f481"
      },
      "source": [
        "dd_count = dict() # image size -> number of appIDs\r\n",
        "\r\n",
        "for sz in dd:\r\n",
        "  dd_count[sz] = len(dd[sz])\r\n",
        "\r\n",
        "print(dd_count)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{(300, 450): 29981, (600, 900): 1}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WoQJBvWvkbDV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a27c8d88-2cb0-47b6-d67b-c42d6b05d836"
      },
      "source": [
        "d_count = dict() # bands -> number of appIDs\r\n",
        "\r\n",
        "for channels_as_str in d:\r\n",
        "  d_count[channels_as_str] = len(d[channels_as_str])\r\n",
        "\r\n",
        "print(d_count)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'RGB': 29642, 'L': 306, 'CMYK': 34}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TKBMk0qfj1Jw"
      },
      "source": [
        "## Visualize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gu79pmPb7opy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c5fe8fce-852f-4588-d50f-55204a82f59c"
      },
      "source": [
        "len(app_ids_with_blank_image)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uJluA9GBZ4YB"
      },
      "source": [
        "import cv2\r\n",
        "from google.colab.patches import cv2_imshow\r\n",
        "\r\n",
        "for app_id in app_ids_with_blank_image:\r\n",
        "  print('Uniform image for appID = {}'.format(app_id))\r\n",
        "\r\n",
        "  print(get_steamdb_url(app_id=app_id))\r\n",
        "\r\n",
        "  url = get_url(app_id=app_id)\r\n",
        "  print(url)\r\n",
        "\r\n",
        "  cv2_imshow(cv2.imread(get_fname(app_id)))"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hWKv3tv937IW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cc90b952-dc17-40e1-ba33-0b10478b5026"
      },
      "source": [
        "l_size = sorted(dd_count.keys(), key=lambda x: dd_count[x], reverse=True)\r\n",
        "\r\n",
        "app_ids_with_wrong_size = [\r\n",
        "                           app_id\r\n",
        "                           for app_id in dd[sz]\r\n",
        "                           for sz in l_size[1:]\r\n",
        "                           ]\r\n",
        "\r\n",
        "len(app_ids_with_wrong_size)                           "
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NlPdv1PHaEp4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c38d606b-d587-4299-d386-348ed1ca02cf"
      },
      "source": [
        "import cv2\r\n",
        "from google.colab.patches import cv2_imshow\r\n",
        "\r\n",
        "for i, sz in enumerate(l_size, start=1):\r\n",
        "  ratio = sz[0]/sz[1]\r\n",
        "  print('{}) {} (ratio {:.2f}): #apps = {}'.format(i, sz, ratio, dd_count[sz]))\r\n",
        "  \r\n",
        "  examplar_index = -1\r\n",
        "  examplar_app_id = dd[sz][examplar_index]\r\n",
        "\r\n",
        "  print(get_steamdb_url(app_id=examplar_app_id))\r\n",
        "\r\n",
        "  url = get_url(app_id=examplar_app_id)\r\n",
        "  print(url)\r\n",
        "\r\n",
        "  fname = get_fname(examplar_app_id)\r\n",
        "  # cv2_imshow(cv2.imread(fname))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1) (300, 450) (ratio 0.67): #apps = 29981\n",
            "https://steamdb.info/app/1217440\n",
            "https://cdn.cloudflare.steamstatic.com/steam/apps/1217440/library_600x900.jpg\n",
            "2) (600, 900) (ratio 0.67): #apps = 1\n",
            "https://steamdb.info/app/346110\n",
            "https://cdn.cloudflare.steamstatic.com/steam/apps/346110/library_600x900.jpg\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wwI_oRxL4exf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7a862599-1d94-44df-81aa-d3d0abe48d4f"
      },
      "source": [
        "l_channels = sorted(d_count.keys(), key=lambda x: d_count[x], reverse=True)\r\n",
        "\r\n",
        "app_ids_with_wrong_channels = [\r\n",
        "                               app_id\r\n",
        "                               for bands in l_channels[1:]\r\n",
        "                               for app_id in d[bands]\r\n",
        "                               ]\r\n",
        "\r\n",
        "len(app_ids_with_wrong_channels)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "340"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9XkyCYbTaRXG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "13302007-2659-4c44-ed99-b9f216b50f3b"
      },
      "source": [
        "import PIL\r\n",
        "import cv2\r\n",
        "from google.colab.patches import cv2_imshow\r\n",
        "\r\n",
        "for i, bands in enumerate(l_channels, start=1):\r\n",
        "  print('{}) {}: #apps = {}'.format(i, bands, d_count[bands]))\r\n",
        "  \r\n",
        "  examplar_index = -1\r\n",
        "  examplar_app_id = d[bands][examplar_index]\r\n",
        "\r\n",
        "  print(get_steamdb_url(app_id=examplar_app_id))\r\n",
        "\r\n",
        "  url = get_url(app_id=examplar_app_id)\r\n",
        "  print(url)\r\n",
        "\r\n",
        "  fname = get_fname(examplar_app_id)\r\n",
        "  # cv2_imshow(cv2.imread(fname))"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1) RGB: #apps = 29642\n",
            "https://steamdb.info/app/1217440\n",
            "https://cdn.cloudflare.steamstatic.com/steam/apps/1217440/library_600x900.jpg\n",
            "2) L: #apps = 306\n",
            "https://steamdb.info/app/1409110\n",
            "https://cdn.cloudflare.steamstatic.com/steam/apps/1409110/library_600x900.jpg\n",
            "3) CMYK: #apps = 34\n",
            "https://steamdb.info/app/17470\n",
            "https://cdn.cloudflare.steamstatic.com/steam/apps/17470/library_600x900.jpg\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8gZURDZoWFEI"
      },
      "source": [
        "## Filter out original images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OW2rXEPYWS86"
      },
      "source": [
        "%cd /content/\n",
        "\n",
        "for folder_other in ['blank_image', 'wrong_size', 'wrong_channels']:\n",
        "  %mkdir -p {folder_other}\n",
        "\n",
        "  if folder_other == 'blank_image':\n",
        "    app_ids_to_remove = app_ids_with_blank_image\n",
        "  elif folder_other == 'wrong_size':\n",
        "    app_ids_to_remove = app_ids_with_wrong_size\n",
        "  else:\n",
        "    app_ids_to_remove = app_ids_with_wrong_channels\n",
        "\n",
        "  remove_app_ids(file_names = list_file_names(),\n",
        "                 folder_other = folder_other,\n",
        "                 app_ids_to_remove = app_ids_to_remove)                 "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "apWs61kE9DaR"
      },
      "source": [
        "# This is why we have 338 images instead of 340 for 'wrong_channels':\r\n",
        "set(app_ids_with_wrong_channels).intersection(app_ids_with_blank_image)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VNtepnAyXoey"
      },
      "source": [
        "## Resize images to 224x224"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I4MYjmPOXsVo"
      },
      "source": [
        "# !apt-get update > /dev/null\r\n",
        "# !apt-get install imagemagick > /dev/null"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7HmVDeld6Hnf"
      },
      "source": [
        "input_resolution = None\r\n",
        "\r\n",
        "file_extension = get_image_extension()"
      ],
      "execution_count": 131,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CZXGlsVB3XK_",
        "outputId": "36cfd92d-4063-499b-b2fa-62b7cc8039f2"
      },
      "source": [
        "if input_resolution is None:\r\n",
        "  output_resolution = 224\r\n",
        "else:\r\n",
        "  output_resolution = int(input_resolution/2)\r\n",
        "\r\n",
        "print('Output resolution: {}'.format(output_resolution))\r\n",
        "\r\n",
        "assert(output_resolution > 0)"
      ],
      "execution_count": 135,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Output resolution: 224\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UoPswOos5__P",
        "outputId": "71bc55e9-7a65-4e80-bacc-6a9e9148cf69"
      },
      "source": [
        "if input_resolution is None or input_resolution > 256:\r\n",
        "  local_data_folder = get_local_data_folder()\r\n",
        "else:\r\n",
        "  local_data_folder = 'data/' + get_output_resized_folder() + '_{}/'.format(input_resolution)\r\n",
        "\r\n",
        "print('From: {}'.format(local_data_folder))"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "From: data/original_vertical_steam_banners/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6O8ph6TW6BXn",
        "outputId": "3a07eff6-3a61-4cfe-d9fc-56c72aab20e1"
      },
      "source": [
        "output_resized_folder = get_output_resized_folder() + '_{}'.format(output_resolution)\r\n",
        "\r\n",
        "%mkdir -p /content/data/{output_resized_folder}\r\n",
        "\r\n",
        "print('To: {}'.format(output_resized_folder))"
      ],
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "To: resized_vertical_steam_banners_224\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mdBWb3CR6D8I",
        "outputId": "2a79425f-cf81-4ab3-cd54-c01dc880aef8"
      },
      "source": [
        "mogrifty_str = f'{output_resolution}x{output_resolution}!'\r\n",
        "\r\n",
        "print('ImageMagick argument: {}'.format(mogrifty_str))"
      ],
      "execution_count": 138,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ImageMagick argument: 224x224!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GnARf_Oh6ZLp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8aa68dc7-fa45-44f9-d685-d08d041cd7f3"
      },
      "source": [
        "file_names = list_file_names(local_data_folder)\r\n",
        "\r\n",
        "len(file_names)"
      ],
      "execution_count": 139,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "29982"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cYi-jrrZYL9l"
      },
      "source": [
        "!mogrify \\\n",
        " -resize {mogrifty_str} \\\n",
        " -path /content/data/{output_resized_folder} \\\n",
        " /content/{local_data_folder}*{file_extension}"
      ],
      "execution_count": 141,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "trRTUxVz3M2A",
        "outputId": "fba7359e-0b0b-418d-b8d8-a9099aa01d67"
      },
      "source": [
        "file_names = list_file_names('data/' + output_resized_folder)\r\n",
        "\r\n",
        "len(file_names)"
      ],
      "execution_count": 142,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "29982"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4ENMQr4BiOX"
      },
      "source": [
        "resolutions = [224]\r\n",
        "resolutions += [int(pow(2, i)) for i in range(9)]\r\n",
        "\r\n",
        "resolutions.sort()\r\n",
        "\r\n",
        "print(len(resolutions))"
      ],
      "execution_count": 173,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J5nt02PfJG5D"
      },
      "source": [
        "## Filter out resized images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ygqwxahLkZX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "29f3f502-fc69-4045-91cd-ea56bb74f886"
      },
      "source": [
        "%cd /content/\r\n",
        "\r\n",
        "for resolution in resolutions:\r\n",
        "  print('\\nResolution: {}'.format(resolution))\r\n",
        "\r\n",
        "  output_resized_folder = get_output_resized_folder() + '_{}'.format(resolution)\r\n",
        "  image_dir = 'data/' + output_resized_folder\r\n",
        "\r\n",
        "  folder_other = 'wrong_channels_after_resizing_to_{}'.format(resolution)\r\n",
        "  %mkdir -p {folder_other}\r\n",
        "\r\n",
        "  # Recommendation: use check_blank == False to go fast.\r\n",
        "  #                 Plus, here, this check overlaps with 'check_channels' anyway.\r\n",
        "  app_ids_with_blank_image, d, dd = detect_issues_with_images(list_file_names(image_dir = image_dir),\r\n",
        "                                                              check_blank=False,\r\n",
        "                                                              check_image_size=True,\r\n",
        "                                                              check_channels=True,\r\n",
        "                                                              verbose=False)\r\n",
        "\r\n",
        "  print(app_ids_with_blank_image)\r\n",
        "  for k, v in d.items():\r\n",
        "    print(k, len(v))\r\n",
        "  for k, v in dd.items():\r\n",
        "    print(k, len(v))\r\n",
        "\r\n",
        "  most_common = max(d, key=lambda x: len(d[x]))\r\n",
        "  print('Most common: {}'.format(most_common))\r\n",
        "\r\n",
        "  app_ids_to_remove = []\r\n",
        "  for k in d:\r\n",
        "    if k != most_common:\r\n",
        "      app_ids_to_remove += d[k]\r\n",
        "  print('#apps to remove = {}'.format(len(app_ids_to_remove)))  \r\n",
        "\r\n",
        "  remove_app_ids(file_names = list_file_names(image_dir = image_dir),\r\n",
        "                 folder_other = folder_other,\r\n",
        "                 app_ids_to_remove = app_ids_to_remove)"
      ],
      "execution_count": 182,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "\n",
            "Resolution: 1\n",
            "[]\n",
            "RGB 29196\n",
            "L 752\n",
            "CMYK 34\n",
            "(1, 1) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 786\n",
            "#appIDs = 786\n",
            "\n",
            "Resolution: 2\n",
            "[]\n",
            "RGB 29308\n",
            "L 640\n",
            "CMYK 34\n",
            "(2, 2) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 674\n",
            "#appIDs = 674\n",
            "\n",
            "Resolution: 4\n",
            "[]\n",
            "RGB 29362\n",
            "L 586\n",
            "CMYK 34\n",
            "(4, 4) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 620\n",
            "#appIDs = 620\n",
            "\n",
            "Resolution: 8\n",
            "[]\n",
            "RGB 29448\n",
            "L 500\n",
            "CMYK 34\n",
            "(8, 8) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 534\n",
            "#appIDs = 534\n",
            "\n",
            "Resolution: 16\n",
            "[]\n",
            "RGB 29486\n",
            "L 462\n",
            "CMYK 34\n",
            "(16, 16) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 496\n",
            "#appIDs = 496\n",
            "\n",
            "Resolution: 32\n",
            "[]\n",
            "RGB 29523\n",
            "L 425\n",
            "CMYK 34\n",
            "(32, 32) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 459\n",
            "#appIDs = 459\n",
            "\n",
            "Resolution: 64\n",
            "[]\n",
            "RGB 29555\n",
            "L 393\n",
            "CMYK 34\n",
            "(64, 64) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 427\n",
            "#appIDs = 427\n",
            "\n",
            "Resolution: 128\n",
            "[]\n",
            "RGB 29576\n",
            "L 372\n",
            "CMYK 34\n",
            "(128, 128) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 406\n",
            "#appIDs = 406\n",
            "\n",
            "Resolution: 224\n",
            "[]\n",
            "RGB 29592\n",
            "L 356\n",
            "CMYK 34\n",
            "(224, 224) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 390\n",
            "#appIDs = 390\n",
            "\n",
            "Resolution: 256\n",
            "[]\n",
            "RGB 29592\n",
            "L 356\n",
            "CMYK 34\n",
            "(256, 256) 29982\n",
            "Most common: RGB\n",
            "#apps to remove = 390\n",
            "#appIDs = 390\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GTo1GaAyKsKv"
      },
      "source": [
        "log_folder = '/content/drive/MyDrive/data/filter_after_resizing/'\r\n",
        "\r\n",
        "%mkdir -p {log_folder}\r\n",
        "%cp wrong_channels_after_resizing_to_*.txt {log_folder}"
      ],
      "execution_count": 184,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vzHp6PX_JU7e"
      },
      "source": [
        "## Export a `.tar` archive of filtered and resized images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R0HQVzxoZjul",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c956c3b6-4102-4459-d3cd-d5cc1889d180"
      },
      "source": [
        "from pathlib import Path\r\n",
        "\r\n",
        "for resolution in resolutions:\r\n",
        "  output_resized_folder = get_output_resized_folder() + f'_{resolution}'\r\n",
        "  output_archive_name = output_resized_folder + '.tar'\r\n",
        "  output_data_folder = 'data/' + output_resized_folder\r\n",
        "\r\n",
        "  file_names = list_file_names(output_data_folder)\r\n",
        "  print('#files = {}'.format(len(file_names)))\r\n",
        "\r\n",
        "  if Path(output_data_folder).exists():\r\n",
        "    if not Path(output_archive_name).exists():\r\n",
        "      !echo tar cf {output_archive_name} {output_data_folder}\r\n",
        "      !tar cf {output_archive_name} {output_data_folder}\r\n",
        "    \r\n",
        "    !du -sh {output_archive_name}  "
      ],
      "execution_count": 179,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#files = 29982\n",
            "30M\tresized_vertical_steam_banners_1.tar\n",
            "#files = 29982\n",
            "30M\tresized_vertical_steam_banners_2.tar\n",
            "#files = 29982\n",
            "30M\tresized_vertical_steam_banners_4.tar\n",
            "#files = 29982\n",
            "30M\tresized_vertical_steam_banners_8.tar\n",
            "#files = 29982\n",
            "39M\tresized_vertical_steam_banners_16.tar\n",
            "#files = 29982\n",
            "54M\tresized_vertical_steam_banners_32.tar\n",
            "#files = 29982\n",
            "109M\tresized_vertical_steam_banners_64.tar\n",
            "#files = 29982\n",
            "286M\tresized_vertical_steam_banners_128.tar\n",
            "#files = 29982\n",
            "677M\tresized_vertical_steam_banners_224.tar\n",
            "#files = 29982\n",
            "835M\tresized_vertical_steam_banners_256.tar\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zwCL1Yj-Hfkw"
      },
      "source": [
        "for resolution in resolutions:\n",
        "  output_resized_folder = get_output_resized_folder() + f'_{resolution}'\n",
        "  output_archive_name = output_resized_folder + '.tar'\n",
        "\n",
        "  ct.copy_file(output_archive_name,\n",
        "               source = lm,\n",
        "               destination = gd + get_gdrive_data_folder())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sqziIe97AGF4"
      },
      "source": [
        "Done."
      ]
    }
  ]
}