clustering-phash.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "import imagehash\n",
    "from PIL import Image\n",
    "import os\n",
    "import os\n",
    "import sys\n",
    "sys.path.append('../../sharechat-scraper') # to access env variables\n",
    "import pymongo\n",
    "from pymongo import MongoClient\n",
    "from s3_mongo_helper import initialize_mongo\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import date, datetime, timedelta\n",
    "import wget\n",
    "import logging\n",
    "from time import perf_counter\n",
    "import numpy as np\n",
    "import sklearn\n",
    "from sklearn.cluster import KMeans\n",
    "import random\n",
    "import scipy.spatial.distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = initialize_mongo()\n",
    "start = datetime(2020, 10, 5)\n",
    "end = datetime(2020, 10, 12)\n",
    "cursor = coll.find({\"scraped_date\": {'$gte':start,'$lt':end},\n",
    "                    \"media_type\": \"image\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18418"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll.count_documents({\"scraped_date\": {'$gte':start,'$lt':end},\n",
    "                    \"media_type\": \"image\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = perf_counter()\n",
    "c=0\n",
    "for i in cursor:\n",
    "    try:\n",
    "        wget.download(i[\"s3_url\"], out=\"images/\")\n",
    "        c+=1\n",
    "    except Exception as e: \n",
    "        print(e)\n",
    "        continue\n",
    "delta = perf_counter() - start\n",
    "print(delta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10800\n"
     ]
    }
   ],
   "source": [
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_hashes(path):\n",
    "    hashfunc = imagehash.phash\n",
    "    failed = []\n",
    "    total = 0\n",
    "    success = 0\n",
    "    image_filenames = []\n",
    "    image_filenames += [file for file in os.listdir(path)]\n",
    "#     image_filenames += [os.path.join(userpath, path) for path in os.listdir(userpath)]\n",
    "    images = {}\n",
    "    for img in image_filenames:\n",
    "        if img.split(\".\")[-1] in [\"jpg\", \"jpeg\", \"png\"]:\n",
    "            try:\n",
    "                phash = hashfunc(Image.open(os.path.join(path, img)))\n",
    "                images[img] = str(phash)\n",
    "                success +=1\n",
    "            except Exception:\n",
    "                failed.append(img)\n",
    "                continue\n",
    "            total +=1\n",
    "    print(\"phash generated for {} of {} images\".format(success, total))\n",
    "    return images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "phash generated for 10788 of 10788 images\n",
      "127.05611668900383\n"
     ]
    }
   ],
   "source": [
    "start = perf_counter()\n",
    "hashes = get_hashes(os.getcwd()+\"/images\")\n",
    "delta = perf_counter() - start\n",
    "print(delta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>phash</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1ee8da5f-326d-427d-85db-d54bea136ab0.jpg</td>\n",
       "      <td>989d1cded6d69610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>75d01f9b-31c9-4c06-9ab0-c7968f3617da.jpg</td>\n",
       "      <td>f0a59e5e254b42bc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>89c5d57c-c3c5-4359-bf68-ae830fceedcf.jpg</td>\n",
       "      <td>94971ef8694161b7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      index             phash\n",
       "0  1ee8da5f-326d-427d-85db-d54bea136ab0.jpg  989d1cded6d69610\n",
       "1  75d01f9b-31c9-4c06-9ab0-c7968f3617da.jpg  f0a59e5e254b42bc\n",
       "2  89c5d57c-c3c5-4359-bf68-ae830fceedcf.jpg  94971ef8694161b7"
      ]
     },
     "execution_count": 464,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame.from_dict(hashes, orient=\"index\", columns = [\"phash\"])\n",
    "df.reset_index(inplace=True)\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 511,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_url(filename):\n",
    "    url = \"https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/\" + filename\n",
    "    return url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 525,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"img_urls\"] = df[\"index\"].map(add_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 526,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped = df.groupby(by=\"phash\").agg({\"index\":\"size\", \"img_urls\":list})\n",
    "grouped.rename(columns={'index':'count'}, inplace=True)\n",
    "sorted = grouped.sort_values(\"count\", ascending=False)\n",
    "sorted.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 527,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>phash</th>\n",
       "      <th>count</th>\n",
       "      <th>img_urls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>81d07fffe0611a85</td>\n",
       "      <td>34</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e3b24096fc0d3cad</td>\n",
       "      <td>20</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f8076bbc53817cc1</td>\n",
       "      <td>13</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>85d3ba7ca5359a12</td>\n",
       "      <td>13</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aad2d00ed3392cb7</td>\n",
       "      <td>12</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ccbbe064401f7d59</td>\n",
       "      <td>11</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>d225baeb205aeb25</td>\n",
       "      <td>9</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>d7e7c00219199fd5</td>\n",
       "      <td>9</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>f43f4f99609060de</td>\n",
       "      <td>9</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>f788d82790fa2574</td>\n",
       "      <td>9</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              phash  count                                           img_urls\n",
       "0  81d07fffe0611a85     34  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "1  e3b24096fc0d3cad     20  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "2  f8076bbc53817cc1     13  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "3  85d3ba7ca5359a12     13  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "4  aad2d00ed3392cb7     12  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "5  ccbbe064401f7d59     11  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "6  d225baeb205aeb25      9  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "7  d7e7c00219199fd5      9  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "8  f43f4f99609060de      9  [https://s3.ap-south-1.amazonaws.com/sharechat...\n",
       "9  f788d82790fa2574      9  [https://s3.ap-south-1.amazonaws.com/sharechat..."
      ]
     },
     "execution_count": 527,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "655"
      ]
     },
     "execution_count": 543,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Narrow down to images that occur at least twice\n",
    "len(sorted[sorted[\"count\"] >= 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 585,
   "metadata": {},
   "outputs": [],
   "source": [
    "identical_clusters = sorted[sorted[\"count\"] >= 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 586,
   "metadata": {},
   "outputs": [],
   "source": [
    "identical_clusters.reset_index(inplace=True,drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 588,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>phash</th>\n",
       "      <th>count</th>\n",
       "      <th>img_urls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>81d07fffe0611a85</td>\n",
       "      <td>34</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e3b24096fc0d3cad</td>\n",
       "      <td>20</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f8076bbc53817cc1</td>\n",
       "      <td>13</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>85d3ba7ca5359a12</td>\n",
       "      <td>13</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aad2d00ed3392cb7</td>\n",
       "      <td>12</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>a7b3586db72243b0</td>\n",
       "      <td>2</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>cccc43ae34371e3c</td>\n",
       "      <td>2</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>652</th>\n",
       "      <td>b038cec7e1c33738</td>\n",
       "      <td>2</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>ea94944e5b998c3e</td>\n",
       "      <td>2</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654</th>\n",
       "      <td>db8aa535d4d4f418</td>\n",
       "      <td>2</td>\n",
       "      <td>[https://s3.ap-south-1.amazonaws.com/sharechat...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>655 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                phash  count  \\\n",
       "0    81d07fffe0611a85     34   \n",
       "1    e3b24096fc0d3cad     20   \n",
       "2    f8076bbc53817cc1     13   \n",
       "3    85d3ba7ca5359a12     13   \n",
       "4    aad2d00ed3392cb7     12   \n",
       "..                ...    ...   \n",
       "650  a7b3586db72243b0      2   \n",
       "651  cccc43ae34371e3c      2   \n",
       "652  b038cec7e1c33738      2   \n",
       "653  ea94944e5b998c3e      2   \n",
       "654  db8aa535d4d4f418      2   \n",
       "\n",
       "                                              img_urls  \n",
       "0    [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "1    [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "2    [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "3    [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "4    [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "..                                                 ...  \n",
       "650  [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "651  [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "652  [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "653  [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "654  [https://s3.ap-south-1.amazonaws.com/sharechat...  \n",
       "\n",
       "[655 rows x 3 columns]"
      ]
     },
     "execution_count": 588,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "identical_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 555,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81d07fffe0611a85\n",
      "0\n",
      "['https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/fc0ae0b0-5f1e-4e02-b438-2e2e58603eb8.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/b70f8fdd-3b47-4cc5-9194-e63a1836bb3e.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/03163a95-2392-4088-ba7d-81cf9908edae.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/b31c3138-c7b9-472a-899c-2f32c829eb5a.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/ff25d508-30e9-49ac-8a40-eee2f7298b54.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/2a29c11c-d4aa-49c5-953f-02510d548146.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/a511dfd3-0035-4361-b250-39296ac54173.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/b5b3c7b5-cdd6-43a4-b620-657f9b9bf925.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/de07c84e-25e2-43fa-bc55-5c31af1c6e43.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/9853aaeb-226c-43bd-bd77-d3af21234b12.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/4fa8565b-d715-4da9-be8c-5af7db363c06.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/b16a7811-9eaf-4d14-808e-edd1f7644cbd.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/be1abf26-dbed-435a-be03-986462e26ec8.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/bcf7b8b8-7040-4629-91df-0c8a597d8a98.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/9504292c-363c-4981-86c8-5f8d2dbde232.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/02492fe6-5ec5-49ee-856c-db20b179fbee.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/39a25832-b389-41c2-b620-3e150de163e8.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/a0d84316-a65d-479a-950a-9ef859d079c9.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/e4e2cc64-8d7b-412a-9da1-74bdc8b2361e.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/f8ac87b2-f259-4b5c-8a9c-6bf65eae2ace.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/e4bb1861-dddb-4486-a277-0adc9d5636e3.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/82a714b0-e4c3-4426-8078-ca4898f2a045.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/15b5fa80-d855-4759-9197-48857b9180bb.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/58a6541c-109f-4061-bfa7-f3fae5d58da7.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/3eca3461-6110-479c-933e-1639a7978400.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/ea0677d6-95e0-4bd8-a9f8-a5fff7b76d52.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/58cc1e9a-d2a3-4e6b-bef2-e1b242078396.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/adccecc4-b768-437b-a29f-3ddb926bc092.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/ac828a75-7b78-42d5-bc02-1b22a6d3051a.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/050e1f6a-4466-45c3-91c1-8d64e1bae916.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/02ae3e94-6259-49a6-8b00-bf42d982a39a.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/96516784-98c8-4a96-96fa-fe28ee180664.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/d258018a-75a4-4cb0-8686-f550f42bc68c.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/5c089c57-2698-4125-b77c-8e8a4957377d.jpg']\n",
      "87d03e7ee0259e85\n",
      "10\n",
      "['https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/dd1b4a8d-0c1b-4232-a731-8e17e8ca266b.jpg', 'https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/439b298f-2457-4b00-a72c-c235073d482d.jpg']\n",
      "c1967e7ee0211e8d\n",
      "10\n",
      "['https://s3.ap-south-1.amazonaws.com/sharechat-scraper.tattle.co.in/b1819760-874a-42cb-a946-fe8020b2bb2c.jpg']\n"
     ]
    }
   ],
   "source": [
    "# Find similar images\n",
    "query = \"81d07fffe0611a85\"\n",
    "for i, row in sorted.iterrows():\n",
    "    # Convert hash hex string to original hash (binary array) \n",
    "    diff = imagehash.hex_to_hash(query) - imagehash.hex_to_hash(row[\"phash\"])\n",
    "    # phashes with Hamming distance less than 10 are likely to be similar images\n",
    "    if diff <= 10:\n",
    "        print(row[\"phash\"])\n",
    "        print(diff)\n",
    "        print(row[\"img_urls\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 589,
   "metadata": {},
   "outputs": [],
   "source": [
    "identical_clusters.to_csv(\"phash_clusters.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 556,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"original_hash\"] = df[\"phash\"].map(imagehash.hex_to_hash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "arrays = np.array(df[\"original_hash\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 28 32 ... 30 34 26]\n",
      " [28 0 22 ... 34 32 28]\n",
      " [32 22 0 ... 30 30 30]\n",
      " ...\n",
      " [30 34 30 ... 0 34 26]\n",
      " [34 32 30 ... 34 0 30]\n",
      " [26 28 30 ... 26 30 0]]\n"
     ]
    }
   ],
   "source": [
    "# Create pairwise (Hamming) difference matrix using numpy broadcasting\n",
    "x = arrays[:, np.newaxis] - arrays[np.newaxis, :]\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10788"
      ]
     },
     "execution_count": 394,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modify Imagehash library func to return array instead of ImageHash obj\n",
    "\n",
    "def hex_to_hash(hexstr):\n",
    "\t\"\"\"\n",
    "\tConvert a stored hash (hex, as retrieved from str(Imagehash))\n",
    "\tback to a Imagehash object.\n",
    "\tNotes:\n",
    "\t1. This algorithm assumes all hashes are either\n",
    "\t   bidimensional arrays with dimensions hash_size * hash_size,\n",
    "\t   or onedimensional arrays with dimensions binbits * 14.\n",
    "\t2. This algorithm does not work for hash_size < 2.\n",
    "\t\"\"\"\n",
    "\thash_size = int(np.sqrt(len(hexstr)*4))\n",
    "\t#assert hash_size == numpy.sqrt(len(hexstr)*4)\n",
    "\tbinary_array = '{:0>{width}b}'.format(int(hexstr, 16), width = hash_size * hash_size)\n",
    "\tbit_rows = [binary_array[i:i+hash_size] for i in range(0, len(binary_array), hash_size)]\n",
    "\thash_array = np.array([[bool(int(d)) for d in row] for row in bit_rows])\n",
    "\treturn hash_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 575,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False, False, False, False,  True,  True, False],\n",
       "       [False, False, False, False, False, False, False, False],\n",
       "       [False,  True, False, False, False, False, False,  True],\n",
       "       [ True, False, False, False, False, False, False,  True],\n",
       "       [False, False, False, False, False, False, False, False],\n",
       "       [False,  True, False, False, False,  True, False, False],\n",
       "       [ True, False, False, False, False,  True, False, False],\n",
       "       [False, False, False, False, False, False, False, False]])"
      ]
     },
     "execution_count": 575,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate Hamming distance \n",
    "xor = hex_to_hash(\"87d03e7ee0259e85\") ^ hex_to_hash(\"81d07fffe0611a85\")\n",
    "xor "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 576,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 576,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Subract matching bits from total number of bits to get difference\n",
    "np.count_nonzero(xor != 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 577,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 577,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imagehash.hex_to_hash(\"87d03e7ee0259e85\") - imagehash.hex_to_hash(\"81d07fffe0611a85\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 579,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare hashes for kmeans clustering\n",
    "arr = []\n",
    "for i in df[\"phash\"][:50]:\n",
    "    arr.append(hex_to_hash(i).flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 580,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64,)"
      ]
     },
     "execution_count": 580,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 581,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How do you decide the number of clusters?\n",
    "kmeans = KMeans(n_clusters=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 582,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KMeans()"
      ]
     },
     "execution_count": 582,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kmeans.fit(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 583,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 4, 4, 2, 7, 6, 4, 1, 0, 1, 5, 5, 3, 0, 3, 0, 5, 7, 1, 3, 3, 0,\n",
       "       3, 2, 2, 2, 2, 2, 2, 1, 6, 1, 5, 6, 1, 6, 7, 7, 4, 2, 3, 0, 3, 2,\n",
       "       0, 5, 7, 4, 1, 3], dtype=int32)"
      ]
     },
     "execution_count": 583,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Are the clusters meaningful? \n",
    "kmeans.labels_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}