data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aff60d8b",
   "metadata": {},
   "source": [
    "# Data Preparation\n",
    "In this notebook we are going to save the images as vectors in order to be able to give them as input to the neural network for training it.\n",
    "We are going to assing label 0 for images with a mask and label 1 for images that do not wear a mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0d6f337a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.preprocessing.image import img_to_array\n",
    "from tensorflow.keras.preprocessing.image import load_img\n",
    "import random\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6da63219",
   "metadata": {},
   "source": [
    "## Read Names of the images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d8afb1e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"dataset\"\n",
    "with_mask = \"with_mask\" #0\n",
    "without_mask = \"without_mask\" #+1\n",
    "img_size = 224\n",
    "\n",
    "names_images_with_mask = np.array(os.listdir(os.path.join(path,with_mask)))\n",
    "names_images_without_mask = np.array(os.listdir(os.path.join(path,without_mask)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc86050a",
   "metadata": {},
   "source": [
    "## Read Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "212b799e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_images(general_path,names_images,index_class):\n",
    "    images_classes = []\n",
    "    for name in names_images:\n",
    "        try:            \n",
    "            img = load_img(general_path + \"\\\\\" + name, target_size=(img_size,img_size))\n",
    "            img = img_to_array(img)\n",
    "            img = img/255\n",
    "            images_classes.append([img,index_class])\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "    return images_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9828a9e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Francesco\\anaconda3\\lib\\site-packages\\PIL\\Image.py:962: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "images_with_mask = read_images(os.path.join(path,with_mask),names_images_with_mask,0)\n",
    "images_without_mask = read_images(os.path.join(path,without_mask),names_images_without_mask,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c211622",
   "metadata": {},
   "source": [
    "## Create Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d3c5266b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataset(images_with_mask,images_without_mask):\n",
    "    return images_with_mask + images_without_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5f62cda5",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = create_dataset(images_with_mask,images_without_mask)\n",
    "random.shuffle(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e62291d",
   "metadata": {},
   "source": [
    "## Design Matrix and Vector of Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "73a3838b",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []\n",
    "for features, label in dataset:\n",
    "    X.append(features)\n",
    "    y.append(label)\n",
    "\n",
    "X = np.array(X,dtype=\"float32\")\n",
    "y = np.array(y)\n",
    "y = LabelBinarizer().fit_transform(y)\n",
    "y = to_categorical(y,num_classes=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41b6a578",
   "metadata": {},
   "source": [
    "## Split into Training, Validation and Test Set\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "38f1b157",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_size = round(0.6 * X.shape[0])\n",
    "validation_size = round(0.2 * X.shape[0])\n",
    "test_size = X.shape[0] - training_size - validation_size\n",
    "\n",
    "X_train = X[0:training_size,:,:,:]\n",
    "X_validation = X[training_size : training_size + validation_size,:,:,:]\n",
    "X_test = X[training_size + validation_size: X.shape[0],:,:,:]\n",
    "\n",
    "y_train = y[0:training_size]\n",
    "y_validation = y[training_size : training_size + validation_size]\n",
    "y_test = y[training_size + validation_size: len(y)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01743ffc",
   "metadata": {},
   "source": [
    "## Save Training, Validation and Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ab807f39",
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_out = open(\"data\\X_train.data\",\"wb\")\n",
    "pickle.dump(X_train, pickle_out, protocol = 4) #more than 4 gb\n",
    "pickle_out.close()\n",
    "\n",
    "pickle_out = open(\"data\\X_validation.data\",\"wb\")\n",
    "pickle.dump(X_validation,pickle_out,protocol = 4) #more than 4 gb\n",
    "pickle_out.close()\n",
    "\n",
    "pickle_out = open(\"data\\X_test.data\",\"wb\")\n",
    "pickle.dump(X_test,pickle_out,protocol = 4) # more than 4 gb\n",
    "pickle_out.close()\n",
    "\n",
    "pickle_out = open(\"data\\y_train.data\",\"wb\")\n",
    "pickle.dump(y_train,pickle_out)\n",
    "pickle_out.close()\n",
    "\n",
    "pickle_out = open(\"data\\y_validation.data\",\"wb\")\n",
    "pickle.dump(y_validation,pickle_out)\n",
    "pickle_out.close()\n",
    "\n",
    "pickle_out = open(\"data\\y_test.data\",\"wb\")\n",
    "pickle.dump(y_test,pickle_out)\n",
    "pickle_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "722c470b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}