preprocess_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import  numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split by day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>checkin_id</th>\n",
       "      <th>venue_id</th>\n",
       "      <th>new_tid</th>\n",
       "      <th>tid</th>\n",
       "      <th>lat_lon</th>\n",
       "      <th>split_day</th>\n",
       "      <th>time</th>\n",
       "      <th>rating</th>\n",
       "      <th>price</th>\n",
       "      <th>day</th>\n",
       "      <th>weather</th>\n",
       "      <th>poi</th>\n",
       "      <th>type</th>\n",
       "      <th>root_type</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>338007</td>\n",
       "      <td>4abc1f51f964a520798620e3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7815580000000 -73.9757920000000</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>1171</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Ocean Grill</td>\n",
       "      <td>Seafood Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>339213</td>\n",
       "      <td>4d4ac10da0ef54814b6ffff6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7840184369222 -73.9745239913464</td>\n",
       "      <td>Saturday</td>\n",
       "      <td>822</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Saturday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Ditch Plains</td>\n",
       "      <td>New American Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>341155</td>\n",
       "      <td>4db44994cda1c57c82583709</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7393979409296 -73.9932097342661</td>\n",
       "      <td>Sunday</td>\n",
       "      <td>860</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>Sunday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Happy Easterpocalypse 2012</td>\n",
       "      <td>General Entertainment</td>\n",
       "      <td>Arts &amp; Entertainment</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>341334</td>\n",
       "      <td>4a541923f964a52008b31fe3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7856770000000 -73.9764980000000</td>\n",
       "      <td>Sunday</td>\n",
       "      <td>962</td>\n",
       "      <td>7.1</td>\n",
       "      <td>2</td>\n",
       "      <td>Sunday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Hi-Life</td>\n",
       "      <td>American Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>342247</td>\n",
       "      <td>40f1d480f964a5205b0a1fe3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7199290238534 -74.0085318264274</td>\n",
       "      <td>Monday</td>\n",
       "      <td>740</td>\n",
       "      <td>9.1</td>\n",
       "      <td>2</td>\n",
       "      <td>Monday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Bubby's</td>\n",
       "      <td>American Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  checkin_id                  venue_id  new_tid  tid  \\\n",
       "0     338007  4abc1f51f964a520798620e3        0    1   \n",
       "1     339213  4d4ac10da0ef54814b6ffff6        1    1   \n",
       "2     341155  4db44994cda1c57c82583709        2    1   \n",
       "3     341334  4a541923f964a52008b31fe3        2    1   \n",
       "4     342247  40f1d480f964a5205b0a1fe3        3    1   \n",
       "\n",
       "                              lat_lon  split_day  time  rating  price  \\\n",
       "0  40.7815580000000 -73.9757920000000  Wednesday  1171    -1.0      3   \n",
       "1  40.7840184369222 -73.9745239913464   Saturday   822    -1.0      3   \n",
       "2  40.7393979409296 -73.9932097342661     Sunday   860    -1.0     -1   \n",
       "3  40.7856770000000 -73.9764980000000     Sunday   962     7.1      2   \n",
       "4  40.7199290238534 -74.0085318264274     Monday   740     9.1      2   \n",
       "\n",
       "         day weather                         poi                     type  \\\n",
       "0  Wednesday   Clear                 Ocean Grill       Seafood Restaurant   \n",
       "1   Saturday   Clear                Ditch Plains  New American Restaurant   \n",
       "2     Sunday   Clear  Happy Easterpocalypse 2012    General Entertainment   \n",
       "3     Sunday   Clear                     Hi-Life      American Restaurant   \n",
       "4     Monday   Clear                     Bubby's      American Restaurant   \n",
       "\n",
       "              root_type  label  \n",
       "0                  Food      1  \n",
       "1                  Food      1  \n",
       "2  Arts & Entertainment      1  \n",
       "3                  Food      1  \n",
       "4                  Food      1  "
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_foursquare_day = pd.read_csv('data/real_application/foursquare_NY/NYByDay_d.csv', sep=\";\")\n",
    "df_foursquare_day.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of trajectories:  91788\n",
      "Number of unique POIs:  30091\n",
      "Number of unique POIs type:  570\n",
      "Number of unique POIs root_type:  10\n",
      "\n",
      "Basics:\n",
      "Number of users:  1083\n",
      "Number of trajectories:  91788\n",
      "AVG number of trajectories per user is 84.95 with STD of 33.66\n",
      "AVG trajectory length per user is 2.25 with STD of 1.39\n",
      "\n",
      "More stats:\n",
      "Number of users:  1083\n",
      "Number of trajectories:  91788\n",
      "\n",
      "Check-ins:\n",
      "Min number of check-ins in an user:  96\n",
      "Max number of check-ins in an user:  2681\n",
      "Number of average check-ins per user 209.233 with std of 187.271.\n",
      "Number of unique POIs in the dataset:  570\n",
      "\n",
      "Trajectories:\n",
      "Min number of trajectories in an user:  2\n",
      "Max number of trajectories in an user:  235\n",
      "Number of average trajectories per user 84.950 with std of 33.661.\n"
     ]
    }
   ],
   "source": [
    "print('Number of trajectories: ',df_foursquare_day['new_tid'].nunique())\n",
    "print('Number of unique POIs: ',df_foursquare_day['poi'].nunique())\n",
    "print('Number of unique POIs type: ',df_foursquare_day['type'].nunique())\n",
    "print('Number of unique POIs root_type: ',df_foursquare_day['root_type'].nunique())\n",
    "basic_stats(df_foursquare_day)\n",
    "df_main_info(df_foursquare_day)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split by week"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>checkin_id</th>\n",
       "      <th>venue_id</th>\n",
       "      <th>new_tid</th>\n",
       "      <th>tid</th>\n",
       "      <th>lat_lon</th>\n",
       "      <th>split_year_week</th>\n",
       "      <th>time</th>\n",
       "      <th>rating</th>\n",
       "      <th>price</th>\n",
       "      <th>day</th>\n",
       "      <th>weather</th>\n",
       "      <th>poi</th>\n",
       "      <th>type</th>\n",
       "      <th>root_type</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>338007</td>\n",
       "      <td>4abc1f51f964a520798620e3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7815580000000 -73.9757920000000</td>\n",
       "      <td>2012 14</td>\n",
       "      <td>1171</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Ocean Grill</td>\n",
       "      <td>Seafood Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>339213</td>\n",
       "      <td>4d4ac10da0ef54814b6ffff6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7840184369222 -73.9745239913464</td>\n",
       "      <td>2012 14</td>\n",
       "      <td>822</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Saturday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Ditch Plains</td>\n",
       "      <td>New American Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>341155</td>\n",
       "      <td>4db44994cda1c57c82583709</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7393979409296 -73.9932097342661</td>\n",
       "      <td>2012 14</td>\n",
       "      <td>860</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>Sunday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Happy Easterpocalypse 2012</td>\n",
       "      <td>General Entertainment</td>\n",
       "      <td>Arts &amp; Entertainment</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>341334</td>\n",
       "      <td>4a541923f964a52008b31fe3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7856770000000 -73.9764980000000</td>\n",
       "      <td>2012 14</td>\n",
       "      <td>962</td>\n",
       "      <td>7.1</td>\n",
       "      <td>2</td>\n",
       "      <td>Sunday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Hi-Life</td>\n",
       "      <td>American Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>342247</td>\n",
       "      <td>40f1d480f964a5205b0a1fe3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7199290238534 -74.0085318264274</td>\n",
       "      <td>2012 15</td>\n",
       "      <td>740</td>\n",
       "      <td>9.1</td>\n",
       "      <td>2</td>\n",
       "      <td>Monday</td>\n",
       "      <td>Clear</td>\n",
       "      <td>Bubby's</td>\n",
       "      <td>American Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  checkin_id                  venue_id  new_tid  tid  \\\n",
       "0     338007  4abc1f51f964a520798620e3        0    1   \n",
       "1     339213  4d4ac10da0ef54814b6ffff6        0    1   \n",
       "2     341155  4db44994cda1c57c82583709        0    1   \n",
       "3     341334  4a541923f964a52008b31fe3        0    1   \n",
       "4     342247  40f1d480f964a5205b0a1fe3        1    1   \n",
       "\n",
       "                              lat_lon split_year_week  time  rating  price  \\\n",
       "0  40.7815580000000 -73.9757920000000         2012 14  1171    -1.0      3   \n",
       "1  40.7840184369222 -73.9745239913464         2012 14   822    -1.0      3   \n",
       "2  40.7393979409296 -73.9932097342661         2012 14   860    -1.0     -1   \n",
       "3  40.7856770000000 -73.9764980000000         2012 14   962     7.1      2   \n",
       "4  40.7199290238534 -74.0085318264274         2012 15   740     9.1      2   \n",
       "\n",
       "         day weather                         poi                     type  \\\n",
       "0  Wednesday   Clear                 Ocean Grill       Seafood Restaurant   \n",
       "1   Saturday   Clear                Ditch Plains  New American Restaurant   \n",
       "2     Sunday   Clear  Happy Easterpocalypse 2012    General Entertainment   \n",
       "3     Sunday   Clear                     Hi-Life      American Restaurant   \n",
       "4     Monday   Clear                     Bubby's      American Restaurant   \n",
       "\n",
       "              root_type  label  \n",
       "0                  Food      1  \n",
       "1                  Food      1  \n",
       "2  Arts & Entertainment      1  \n",
       "3                  Food      1  \n",
       "4                  Food      1  "
      ]
     },
     "execution_count": 355,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_foursquare_yw = pd.read_csv('data/real_application/foursquare_NY/NYByYearWeek_yw.csv', sep=\";\")\n",
    "df_foursquare_yw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of trajectories:  30509\n",
      "Number of unique POIs:  30091\n",
      "Number of unique POIs type:  570\n",
      "Number of unique POIs root_type:  10\n",
      "\n",
      "Basics:\n",
      "Number of users:  1083\n",
      "Number of trajectories:  30509\n",
      "AVG number of trajectories per user is 28.17 with STD of 9.34\n",
      "AVG trajectory length per user is 5.34 with STD of 3.41\n",
      "\n",
      "More stats:\n",
      "Number of users:  1083\n",
      "Number of trajectories:  30509\n",
      "\n",
      "Check-ins:\n",
      "Min number of check-ins in an user:  96\n",
      "Max number of check-ins in an user:  2681\n",
      "Number of average check-ins per user 209.237 with std of 187.273.\n",
      "Number of unique POIs in the dataset:  570\n",
      "\n",
      "Trajectories:\n",
      "Min number of trajectories in an user:  1\n",
      "Max number of trajectories in an user:  44\n",
      "Number of average trajectories per user 28.173 with std of 9.342.\n"
     ]
    }
   ],
   "source": [
    "print('Number of trajectories: ',df_foursquare_yw['new_tid'].nunique())\n",
    "print('Number of unique POIs: ',df_foursquare_yw['poi'].nunique())\n",
    "print('Number of unique POIs type: ',df_foursquare_yw['type'].nunique())\n",
    "print('Number of unique POIs root_type: ',df_foursquare_yw['root_type'].nunique())\n",
    "basic_stats(df_foursquare_yw)\n",
    "df_main_info(df_foursquare_yw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create sequence format file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = \"data/real_application/foursquare_NY/fs_ny_week_sequences.dat\"\n",
    "create_sequence_file_data(df_foursquare_yw,file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FS top 10 users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>new_tid</th>\n",
       "      <th>tid</th>\n",
       "      <th>lat_lon</th>\n",
       "      <th>time</th>\n",
       "      <th>day</th>\n",
       "      <th>poi</th>\n",
       "      <th>type</th>\n",
       "      <th>root_type</th>\n",
       "      <th>rating</th>\n",
       "      <th>weather</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5184</td>\n",
       "      <td>40.9654169422336 -74.0628837642180</td>\n",
       "      <td>1056</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>my bed</td>\n",
       "      <td>Strip Club</td>\n",
       "      <td>Nightlife Spot</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clear</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5184</td>\n",
       "      <td>40.9653287788182 -74.0629239214116</td>\n",
       "      <td>1099</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>nanas :)</td>\n",
       "      <td>American Restaurant</td>\n",
       "      <td>Food</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clear</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>5184</td>\n",
       "      <td>40.9653550190346 -74.0628321335281</td>\n",
       "      <td>1191</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>The Bush Hot Tub :)</td>\n",
       "      <td>Pool</td>\n",
       "      <td>Outdoors &amp; Recreation</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clear</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5184</td>\n",
       "      <td>40.9653379500000 -74.0628079000000</td>\n",
       "      <td>1261</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>The Hang Out</td>\n",
       "      <td>Other Nightlife</td>\n",
       "      <td>Nightlife Spot</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clouds</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>5184</td>\n",
       "      <td>40.9653558300157 -74.0628550805032</td>\n",
       "      <td>1320</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>The Bush Hideout</td>\n",
       "      <td>Speakeasy</td>\n",
       "      <td>Nightlife Spot</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clouds</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   new_tid   tid                             lat_lon  time      day  \\\n",
       "0        0  5184  40.9654169422336 -74.0628837642180  1056  Tuesday   \n",
       "1        0  5184  40.9653287788182 -74.0629239214116  1099  Tuesday   \n",
       "2        0  5184  40.9653550190346 -74.0628321335281  1191  Tuesday   \n",
       "3        0  5184  40.9653379500000 -74.0628079000000  1261  Tuesday   \n",
       "4        0  5184  40.9653558300157 -74.0628550805032  1320  Tuesday   \n",
       "\n",
       "                   poi                 type              root_type  rating  \\\n",
       "0               my bed           Strip Club         Nightlife Spot    -1.0   \n",
       "1             nanas :)  American Restaurant                   Food    -1.0   \n",
       "2  The Bush Hot Tub :)                 Pool  Outdoors & Recreation    -1.0   \n",
       "3         The Hang Out      Other Nightlife         Nightlife Spot    -1.0   \n",
       "4     The Bush Hideout            Speakeasy         Nightlife Spot    -1.0   \n",
       "\n",
       "  weather  label  \n",
       "0   Clear    185  \n",
       "1   Clear    185  \n",
       "2   Clear    185  \n",
       "3  Clouds    185  \n",
       "4  Clouds    185  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fs_top = pd.read_csv('data/real_application/foursquare_NY/fs_ny_top_users_10.csv', sep=\";\")\n",
    "df_fs_top.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       new_tid  tid  time  day  poi  type  root_type  rating  weather  label\n",
      "label                                                                       \n",
      "185         42   42   959    7  139    76          8      31        6      1\n",
      "293         41   41   686    7  128    55          8      34        6      1\n",
      "315         40   40   797    7  115    72          8      27        5      1\n",
      "349         31   31   600    7  588   187         10      52        5      1\n",
      "354         36   36   799    7  384   102          9      51        5      1\n",
      "372         31   31   401    7  327   128         10      50        5      1\n",
      "689         35   35    22    7    9     8          5       8        5      1\n",
      "730         30   30    39    7   19    15          8      15        6      1\n",
      "885         32   32   363    7   50    36          9      22        6      1\n",
      "943         34   34   463    7  176   102          8      49        6      1\n"
     ]
    }
   ],
   "source": [
    "print(df_fs_top.groupby(['label']).nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of trajectories:  352\n",
      "Number of unique POIs:  1741\n",
      "Number of unique POIs type:  310\n",
      "Number of unique POIs root_type:  10\n",
      "\n",
      "Basics:\n",
      "Number of users:  10\n",
      "Number of trajectories:  352\n",
      "AVG number of trajectories per user is 35.20 with STD of 4.44\n",
      "AVG trajectory length per user is 32.23 with STD of 15.46\n",
      "\n",
      "More stats:\n",
      "Number of users:  10\n",
      "Number of trajectories:  352\n",
      "\n",
      "Check-ins:\n",
      "Min number of check-ins in an user:  568\n",
      "Max number of check-ins in an user:  1952\n",
      "Number of average check-ins per user 1171.500 with std of 598.203.\n",
      "Number of unique POIs in the dataset:  310\n",
      "\n",
      "Trajectories:\n",
      "Min number of trajectories in an user:  30\n",
      "Max number of trajectories in an user:  42\n",
      "Number of average trajectories per user 35.200 with std of 4.442.\n"
     ]
    }
   ],
   "source": [
    "print('Number of trajectories: ',df_fs_top['new_tid'].nunique())\n",
    "print('Number of unique POIs: ',df_fs_top['poi'].nunique())\n",
    "print('Number of unique POIs type: ',df_fs_top['type'].nunique())\n",
    "print('Number of unique POIs root_type: ',df_fs_top['root_type'].nunique())\n",
    "basic_stats(df_fs_top)\n",
    "df_main_info(df_fs_top)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create sequence file format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processando DataFrame: 100%|\u001b[34m██████████████████████████████████████████████████\u001b[0m| 11715/11715 [00:00<00:00, 40713.27it/s]\u001b[0m"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences:  352\n",
      "Process completed!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "file = \"data/real_application/foursquare_NY/preprocessed/fs_ny_top_10_week_sequences.dat\"\n",
    "create_sequence_file_data(df_fs_top,file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.5 2.247220505424423 5.05\n",
      "119.5 5.678908345800274 32.25\n"
     ]
    }
   ],
   "source": [
    "e = [10,14,18,15,17,15,13,15,12,16]\n",
    "f = [110,120,125,115,130,115,120,115,125,120]\n",
    "print(np.mean(e),np.std(e),np.var(e))\n",
    "print(np.mean(f),np.std(f),np.var(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FS to 81 users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>new_tid</th>\n",
       "      <th>tid</th>\n",
       "      <th>lat_lon</th>\n",
       "      <th>time</th>\n",
       "      <th>day</th>\n",
       "      <th>poi</th>\n",
       "      <th>type</th>\n",
       "      <th>root_type</th>\n",
       "      <th>rating</th>\n",
       "      <th>weather</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>329</td>\n",
       "      <td>40.7421883084845 -73.9879241890915</td>\n",
       "      <td>1135</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>Madison Square Park</td>\n",
       "      <td>Park</td>\n",
       "      <td>Outdoors &amp; Recreation</td>\n",
       "      <td>9.6</td>\n",
       "      <td>Clear</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>329</td>\n",
       "      <td>40.7421883084845 -73.9879241890915</td>\n",
       "      <td>421</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>Madison Square Park</td>\n",
       "      <td>Park</td>\n",
       "      <td>Outdoors &amp; Recreation</td>\n",
       "      <td>9.6</td>\n",
       "      <td>Clouds</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>329</td>\n",
       "      <td>40.7416730481413 -73.9742878438115</td>\n",
       "      <td>588</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>NYU Langone Medical Center</td>\n",
       "      <td>Hospital</td>\n",
       "      <td>Professional &amp; Other Places</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clear</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>329</td>\n",
       "      <td>40.7421883084845 -73.9879241890915</td>\n",
       "      <td>949</td>\n",
       "      <td>Wednesday</td>\n",
       "      <td>Madison Square Park</td>\n",
       "      <td>Park</td>\n",
       "      <td>Outdoors &amp; Recreation</td>\n",
       "      <td>9.6</td>\n",
       "      <td>Clear</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>329</td>\n",
       "      <td>40.7421883084845 -73.9879241890915</td>\n",
       "      <td>428</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>Madison Square Park</td>\n",
       "      <td>Park</td>\n",
       "      <td>Outdoors &amp; Recreation</td>\n",
       "      <td>9.6</td>\n",
       "      <td>Clear</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   new_tid  tid                             lat_lon  time        day  \\\n",
       "0        0  329  40.7421883084845 -73.9879241890915  1135    Tuesday   \n",
       "1        0  329  40.7421883084845 -73.9879241890915   421  Wednesday   \n",
       "2        0  329  40.7416730481413 -73.9742878438115   588  Wednesday   \n",
       "3        0  329  40.7421883084845 -73.9879241890915   949  Wednesday   \n",
       "4        0  329  40.7421883084845 -73.9879241890915   428   Thursday   \n",
       "\n",
       "                          poi      type                    root_type  rating  \\\n",
       "0         Madison Square Park      Park        Outdoors & Recreation     9.6   \n",
       "1         Madison Square Park      Park        Outdoors & Recreation     9.6   \n",
       "2  NYU Langone Medical Center  Hospital  Professional & Other Places    -1.0   \n",
       "3         Madison Square Park      Park        Outdoors & Recreation     9.6   \n",
       "4         Madison Square Park      Park        Outdoors & Recreation     9.6   \n",
       "\n",
       "  weather  label  \n",
       "0   Clear     14  \n",
       "1  Clouds     14  \n",
       "2   Clear     14  \n",
       "3   Clear     14  \n",
       "4   Clear     14  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fs_81 = pd.read_csv('data/real_application/foursquare_NY/fs_ny_top_users_81.csv', sep=\";\")\n",
    "print(df_fs_81['label'].nunique())\n",
    "df_fs_81.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_summary = create_df_map_traj_user(df_fs_81)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(df_summary.head())\n",
    "# print(df_summary['User'].nunique())\n",
    "# tmp = df_summary['User'].value_counts().to_dict()\n",
    "# x = list(tmp.keys())\n",
    "# y = list(tmp.values())\n",
    "# top_81 = x[:81]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_fs_81 = df_fs_81[df_fs_81['label'].isin(top_81)]\n",
    "# df_fs_81.reset_index(inplace=True,drop=True)\n",
    "# # print(df_fs_81['label'].nunique())\n",
    "# print(df_fs_81.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of trajectories:  1749\n",
      "Number of unique POIs:  6324\n",
      "Number of unique POIs type:  437\n",
      "Number of unique POIs root_type:  10\n",
      "\n",
      "Basics:\n",
      "Number of users:  81\n",
      "Number of trajectories:  1749\n",
      "AVG number of trajectories per user is 21.59 with STD of 6.09\n",
      "AVG trajectory length per user is 22.81 with STD of 9.66\n",
      "\n",
      "More stats:\n",
      "Number of users:  81\n",
      "Number of trajectories:  1749\n",
      "\n",
      "Check-ins:\n",
      "Min number of check-ins in an user:  202\n",
      "Max number of check-ins in an user:  1952\n",
      "Number of average check-ins per user 515.901 with std of 365.313.\n",
      "Number of unique POIs in the dataset:  437\n",
      "\n",
      "Trajectories:\n",
      "Min number of trajectories in an user:  16\n",
      "Max number of trajectories in an user:  42\n",
      "Number of average trajectories per user 21.593 with std of 6.088.\n"
     ]
    }
   ],
   "source": [
    "print('Number of trajectories: ',df_fs_81['new_tid'].nunique())\n",
    "print('Number of unique POIs: ',df_fs_81['poi'].nunique())\n",
    "print('Number of unique POIs type: ',df_fs_81['type'].nunique())\n",
    "print('Number of unique POIs root_type: ',df_fs_81['root_type'].nunique())\n",
    "basic_stats(df_fs_81)\n",
    "df_main_info(df_fs_81)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processando DataFrame: 100%|\u001b[34m██████████████████████████████████████████████████\u001b[0m| 41788/41788 [00:00<00:00, 51099.55it/s]\u001b[0m"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences:  1750\n",
      "Process completed!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "file = \"data/real_application/foursquare_NY/preprocessed/fs_ny_top_81_week_sequences.dat\"\n",
    "create_sequence_file_data(df_fs_81,file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FS NY 193"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "193\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>new_tid</th>\n",
       "      <th>tid</th>\n",
       "      <th>lat_lon</th>\n",
       "      <th>time</th>\n",
       "      <th>day</th>\n",
       "      <th>poi</th>\n",
       "      <th>type</th>\n",
       "      <th>root_type</th>\n",
       "      <th>rating</th>\n",
       "      <th>weather</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>126</td>\n",
       "      <td>40.8331652006224 -73.9418603427692</td>\n",
       "      <td>317</td>\n",
       "      <td>Monday</td>\n",
       "      <td>The Lair Of Modern Strange Cowboy</td>\n",
       "      <td>Home (private)</td>\n",
       "      <td>Residence</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clear</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>126</td>\n",
       "      <td>40.8340978041072 -73.9452672225881</td>\n",
       "      <td>1404</td>\n",
       "      <td>Monday</td>\n",
       "      <td>Galaxy Gourmet Deli</td>\n",
       "      <td>Deli / Bodega</td>\n",
       "      <td>Food</td>\n",
       "      <td>8.2</td>\n",
       "      <td>Clouds</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>126</td>\n",
       "      <td>40.8331652006224 -73.9418603427692</td>\n",
       "      <td>0</td>\n",
       "      <td>Tuesday</td>\n",
       "      <td>The Lair Of Modern Strange Cowboy</td>\n",
       "      <td>Home (private)</td>\n",
       "      <td>Residence</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clouds</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>126</td>\n",
       "      <td>40.7646959283254 -73.8851974964414</td>\n",
       "      <td>1069</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>Popeyes Louisiana Kitchen</td>\n",
       "      <td>Fried Chicken Joint</td>\n",
       "      <td>Food</td>\n",
       "      <td>6.6</td>\n",
       "      <td>Clear</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>126</td>\n",
       "      <td>40.7660790376824 -73.8835287094116</td>\n",
       "      <td>1120</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>MTA Bus Operations Depot - LaGuardia</td>\n",
       "      <td>Bus Station</td>\n",
       "      <td>Travel &amp; Transport</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Clear</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   new_tid  tid                             lat_lon  time       day  \\\n",
       "0        0  126  40.8331652006224 -73.9418603427692   317    Monday   \n",
       "1        0  126  40.8340978041072 -73.9452672225881  1404    Monday   \n",
       "2        0  126  40.8331652006224 -73.9418603427692     0   Tuesday   \n",
       "3        0  126  40.7646959283254 -73.8851974964414  1069  Thursday   \n",
       "4        0  126  40.7660790376824 -73.8835287094116  1120  Thursday   \n",
       "\n",
       "                                    poi                 type  \\\n",
       "0     The Lair Of Modern Strange Cowboy       Home (private)   \n",
       "1                   Galaxy Gourmet Deli        Deli / Bodega   \n",
       "2     The Lair Of Modern Strange Cowboy       Home (private)   \n",
       "3             Popeyes Louisiana Kitchen  Fried Chicken Joint   \n",
       "4  MTA Bus Operations Depot - LaGuardia          Bus Station   \n",
       "\n",
       "            root_type  rating weather  label  \n",
       "0           Residence    -1.0   Clear      6  \n",
       "1                Food     8.2  Clouds      6  \n",
       "2           Residence    -1.0  Clouds      6  \n",
       "3                Food     6.6   Clear      6  \n",
       "4  Travel & Transport    -1.0   Clear      6  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fs_193 = pd.read_csv('data/real_application/foursquare_NY/fs_ny_top_users_193.csv', sep=\";\")\n",
    "print(df_fs_193['label'].nunique())\n",
    "df_fs_193.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_summary = create_df_map_traj_user(df_fs_193)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of trajectories:  3079\n",
      "Number of unique POIs:  10808\n",
      "Number of unique POIs type:  491\n",
      "Number of unique POIs root_type:  10\n",
      "\n",
      "Basics:\n",
      "Number of users:  193\n",
      "Number of trajectories:  3079\n",
      "AVG number of trajectories per user is 15.95 with STD of 6.33\n",
      "AVG trajectory length per user is 20.52 with STD of 8.06\n",
      "\n",
      "More stats:\n",
      "Number of users:  193\n",
      "Number of trajectories:  3079\n",
      "\n",
      "Check-ins:\n",
      "Min number of check-ins in an user:  127\n",
      "Max number of check-ins in an user:  1952\n",
      "Number of average check-ins per user 346.870 with std of 284.283.\n",
      "Number of unique POIs in the dataset:  491\n",
      "\n",
      "Trajectories:\n",
      "Min number of trajectories in an user:  10\n",
      "Max number of trajectories in an user:  42\n",
      "Number of average trajectories per user 15.953 with std of 6.334.\n"
     ]
    }
   ],
   "source": [
    "print('Number of trajectories: ',df_fs_193['new_tid'].nunique())\n",
    "print('Number of unique POIs: ',df_fs_193['poi'].nunique())\n",
    "print('Number of unique POIs type: ',df_fs_193['type'].nunique())\n",
    "print('Number of unique POIs root_type: ',df_fs_193['root_type'].nunique())\n",
    "basic_stats(df_fs_193)\n",
    "df_main_info(df_fs_193)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processando DataFrame: 100%|\u001b[34m██████████████████████████████████████████████████\u001b[0m| 66946/66946 [00:01<00:00, 48244.92it/s]\u001b[0m"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences:  3079\n",
      "Process completed!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "file = \"data/real_application/foursquare_NY/preprocessed/fs_ny_top_193_week_sequences.dat\"\n",
    "create_sequence_file_data(df_fs_193,file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splice-junction Gene Sequencies data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of labels:  3\n"
     ]
    }
   ],
   "source": [
    "df_sjgs = pd.read_csv('data/real_application/gene_sequences/SJGS/splice_data.csv', sep=\";\")\n",
    "print('Number of labels: ',df_sjgs['label'].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processando DataFrame: 100%|\u001b[34m████████████████████████████████████████████████\u001b[0m| 191400/191400 [00:03<00:00, 53562.68it/s]\u001b[0m"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences:  3190\n",
      "Preprocess completed!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "file = \"data/real_application/gene_sequences/SJGS/preprocessed/sjgs.dat\"\n",
    "dimension = 'sequence'\n",
    "create_sequence_file_data(df_sjgs,file,dimension)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processando DataFrame: 100%|\u001b[34m████████████████████████████████████████████████\u001b[0m| 191400/191400 [00:03<00:00, 51331.52it/s]\u001b[0m"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sequences:  3190\n",
      "Preprocess completed!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "file = \"data/real_application/gene_sequences/SJGS/preprocessed/sjgs2.dat\"\n",
    "dimension = 'sequence'\n",
    "create_sequence_file_data_triple_sjgs(df_sjgs,file,dimension)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   new_tid                     type  label\n",
      "0        0       Seafood Restaurant      1\n",
      "1        0  New American Restaurant      1\n",
      "2        0    General Entertainment      1\n",
      "3        0      American Restaurant      1\n",
      "4        1      American Restaurant      1\n",
      "                               type\n",
      "label new_tid                      \n",
      "1     0          Seafood Restaurant\n",
      "      1         American Restaurant\n",
      "      2              Sandwich Place\n",
      "      3                 Coffee Shop\n",
      "      4                   Gift Shop\n",
      "...                             ...\n",
      "1083  30504                  Office\n",
      "      30505                 Gay Bar\n",
      "      30506                Bus Line\n",
      "      30507    Gym / Fitness Center\n",
      "      30508                  Office\n",
      "\n",
      "[30511 rows x 1 columns]\n",
      "               type\n",
      "label new_tid      \n",
      "1     0           4\n",
      "      1          12\n",
      "      2          13\n",
      "      3           3\n",
      "      4           3\n",
      "...             ...\n",
      "1083  30504       1\n",
      "      30505       4\n",
      "      30506       6\n",
      "      30507       2\n",
      "      30508       1\n",
      "\n",
      "[30511 rows x 1 columns]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>new_tid</th>\n",
       "      <th>type</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>55</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>39</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>19</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1079</td>\n",
       "      <td>21</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1080</td>\n",
       "      <td>19</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1081</td>\n",
       "      <td>10</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1082</td>\n",
       "      <td>10</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1083</td>\n",
       "      <td>40</td>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1083 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       new_tid  type  label\n",
       "label                      \n",
       "1           18    46      1\n",
       "2           24    63      1\n",
       "3           27    55      1\n",
       "4           39    49      1\n",
       "5           19    25      1\n",
       "...        ...   ...    ...\n",
       "1079        21    34      1\n",
       "1080        19    45      1\n",
       "1081        10    29      1\n",
       "1082        10    25      1\n",
       "1083        40    63      1\n",
       "\n",
       "[1083 rows x 3 columns]"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df_fs_yw = df_foursquare_yw.drop(columns=['checkin_id','venue_id','tid','lat_lon','split_year_week','time','rating',\n",
    "                                             'price','day','weather','poi','root_type'])\n",
    "print(new_df_fs_yw.head())\n",
    "a = new_df_fs_yw.groupby(['label','new_tid'])\n",
    "print(a.first())\n",
    "# print(a.nunique()[0:20])\n",
    "# print(a.nunique()[0:20].groupby('label').sum())\n",
    "print(a.nunique())\n",
    "# print(a.get_group(1)['new_tid'].nunique)\n",
    "b = new_df_fs_yw.groupby(['label'])\n",
    "b.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# support methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_sequence_file_data(df,pfile,dimension):\n",
    "    sequence = []\n",
    "    past_tid = None\n",
    "    curr_tid = None\n",
    "    num_of_seqs = 0\n",
    "#     f = open(\"data/real_application/foursquare_NY/fs_ny_week_sequences.dat\", \"w\")\n",
    "    f = open(pfile, \"w\")\n",
    "    \n",
    "    for i in tqdm(range(int(len(df))), colour='blue', desc='Processando DataFrame'):\n",
    "        curr_tid = df.loc[i,\"new_tid\"]\n",
    "        index = df.index[df.loc[i,\"new_tid\"]]\n",
    "        \n",
    "        if index == 0:\n",
    "            sequence.append(str(df.loc[i,dimension].replace(' ','_')))\n",
    "            past_tid = curr_tid\n",
    "        elif curr_tid == past_tid:\n",
    "            sequence.append(str(df.loc[i,dimension].replace(' ','_')))\n",
    "        elif curr_tid != past_tid:\n",
    "            tmp = ' '.join(sequence)\n",
    "            tmp += '\\n'\n",
    "            f.write(tmp)\n",
    "            num_of_seqs += 1\n",
    "            sequence = []\n",
    "            sequence.append(str(df.loc[i,dimension].replace(' ','_')))\n",
    "            past_tid = curr_tid\n",
    "        \n",
    "        if i == (len(df)-1):\n",
    "            tmp = ' '.join(sequence)\n",
    "            f.write(tmp)\n",
    "            num_of_seqs += 1\n",
    "            \n",
    "    f.close()\n",
    "    \n",
    "    print('Number of sequences: ',num_of_seqs)\n",
    "    print(\"Preprocess completed!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_sequence_file_data_triple_sjgs(df,pfile,dimension):\n",
    "    sequence = []\n",
    "    triple = \"\"\n",
    "    past_tid = None\n",
    "    curr_tid = None\n",
    "    num_of_seqs = 0\n",
    "#     f = open(\"data/real_application/foursquare_NY/fs_ny_week_sequences.dat\", \"w\")\n",
    "    f = open(pfile, \"w\")\n",
    "    \n",
    "    count_triple = 0\n",
    "    for i in tqdm(range(int(len(df))), colour='blue', desc='Processando DataFrame'):\n",
    "        curr_tid = df.loc[i,\"new_tid\"]\n",
    "        index = df.index[df.loc[i,\"new_tid\"]]\n",
    "        \n",
    "        if index == 0:\n",
    "#             sequence.append(str(df.loc[i,dimension].replace(' ','_')))\n",
    "            triple += str(df.loc[i,dimension].replace(' ','_'))\n",
    "            past_tid = curr_tid\n",
    "            count_triple += 1\n",
    "        elif curr_tid == past_tid and count_triple < 3:\n",
    "#             sequence.append(str(df.loc[i,dimension].replace(' ','_')))\n",
    "            triple += str(df.loc[i,dimension].replace(' ','_'))\n",
    "            count_triple += 1\n",
    "        elif curr_tid == past_tid and count_triple == 3:\n",
    "            sequence.append(triple)\n",
    "            triple = str(df.loc[i,dimension].replace(' ','_'))\n",
    "            count_triple = 1\n",
    "        elif curr_tid != past_tid:\n",
    "            tmp = ' '.join(sequence)\n",
    "            tmp += '\\n'\n",
    "            f.write(tmp)\n",
    "            num_of_seqs += 1\n",
    "            sequence = []\n",
    "#             sequence.append(str(df.loc[i,dimension].replace(' ','_')))\n",
    "            triple = str(df.loc[i,dimension].replace(' ','_'))\n",
    "            count_triple = 1\n",
    "            past_tid = curr_tid\n",
    "        \n",
    "        if i == (len(df)-1):\n",
    "            tmp = ' '.join(sequence)\n",
    "            f.write(tmp)\n",
    "            num_of_seqs += 1\n",
    "            \n",
    "    f.close()\n",
    "    \n",
    "    print('Number of sequences: ',num_of_seqs)\n",
    "    print(\"Preprocess completed!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "# basic_stats(df_fs_top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def basic_stats(df):\n",
    "    print('\\nBasics:')\n",
    "    num_of_users = df['label'].nunique()\n",
    "    num_of_trajs = df['new_tid'].nunique()\n",
    "    avg_trajs_per_user = df.groupby(['label']).nunique()['new_tid'].mean()\n",
    "    std_trajs_per_user = df.groupby(['label']).nunique()['new_tid'].std()\n",
    "    tmp = df.groupby(['label','new_tid'])\n",
    "    avg_traj_len_per_user = tmp.count()['type'].groupby('label').mean().mean()\n",
    "    std_traj_len_per_user = tmp.count()['type'].groupby('label').std().mean()\n",
    "    \n",
    "    \n",
    "    print('Number of users: ',num_of_users)\n",
    "    print('Number of trajectories: ',num_of_trajs)\n",
    "    print('AVG number of trajectories per user is {:.2f} with STD of {:.2f}'.format(avg_trajs_per_user,std_trajs_per_user))\n",
    "    print('AVG trajectory length per user is {:.2f} with STD of {:.2f}'.format(avg_traj_len_per_user,std_traj_len_per_user))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_main_info(df):\n",
    "    print('\\nMore stats:')\n",
    "    print('Number of users: ',df['label'].nunique())\n",
    "    print('Number of trajectories: ',df['new_tid'].nunique())\n",
    "    \n",
    "    print('\\nCheck-ins:')\n",
    "    # print(df_foursquare.groupby('label')['new_tid'].count())\n",
    "    print('Min number of check-ins in an user: ',df.groupby('label')['new_tid'].count().min())\n",
    "    print('Max number of check-ins in an user: ',df.groupby('label')['new_tid'].count().max())\n",
    "    print('Number of average check-ins per user {:.3f} with std of {:.3f}.'.format(df.groupby('label')['new_tid'].count().mean(),\n",
    "                                                                                   df.groupby('label')['new_tid'].count().std()))\n",
    "    print('Number of unique POIs in the dataset: ',df['type'].nunique())\n",
    "\n",
    "    print('\\nTrajectories:')\n",
    "    # print(df_foursquare.groupby('label')['new_tid'].nunique())\n",
    "    print('Min number of trajectories in an user: ',df.groupby('label')['new_tid'].nunique().min())\n",
    "    print('Max number of trajectories in an user: ',df.groupby('label')['new_tid'].nunique().max())\n",
    "    print('Number of average trajectories per user {:.3f} with std of {:.3f}.'.format(df.groupby('label')['new_tid'].nunique().mean(),\n",
    "                                                                                   df.groupby('label')['new_tid'].nunique().std()))\n",
    "\n",
    "    # df_foursquare\n",
    "    # print(df_foursquare[df_foursquare['label'==1]])\n",
    "    # df_foursquare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def create_df_map_traj_user(df):\n",
    "def create_df_map_traj_user(df=pd.DataFrame):\n",
    "    '''\n",
    "    Method to support the calculation of the quality result.\n",
    "    It returns a dataframe with the users and their trajectories with its respective length.\n",
    "    '''\n",
    "    try:\n",
    "        #     df = pd.read_csv('data/real_application/foursquare_NY/fs_ny_top_users_10.csv', sep=\";\")\n",
    "        df_map_traj_user = pd.DataFrame(columns=['Tid','Traj_length','User'])\n",
    "        tids = []\n",
    "        user = ''\n",
    "        traj_length = 0\n",
    "\n",
    "        sequence = []\n",
    "        past_tid = None\n",
    "        curr_tid = None\n",
    "        num_of_seqs = 0\n",
    "        map_element_id = 0\n",
    "        unique_elements = {}\n",
    "        map_id_to_element = {}\n",
    "\n",
    "        for i in range(len(df)):\n",
    "            curr_tid = df.loc[i,\"new_tid\"]\n",
    "            if curr_tid not in tids:\n",
    "                tids.append(int(curr_tid))\n",
    "                user = int(df.loc[i,\"label\"])\n",
    "                traj_length = len(df[df['new_tid'] == curr_tid])\n",
    "                # append rows to an empty DataFrame\n",
    "                df_map_traj_user = df_map_traj_user.append({'Tid' : curr_tid, 'Traj_length' : traj_length, 'User' : user},ignore_index = True)\n",
    "        df_map_traj_user['Tid'] = df_map_traj_user['Tid'].astype(int, errors='ignore')\n",
    "        df_map_traj_user['Traj_length'] = df_map_traj_user['Traj_length'].astype(int, errors='ignore')\n",
    "        df_map_traj_user['User'] = df_map_traj_user['User'].astype(int, errors='ignore')\n",
    "\n",
    "    #     print(df_map_traj_user.shape)\n",
    "    #     print(df_map_traj_user.head())\n",
    "    #     print('Todo DataFrame (traj_length):',' mean=',df_map_traj_user['Traj_length'].mean(),\n",
    "    #           ' std=',df_map_traj_user['Traj_length'].std())\n",
    "    #     u_185 = df_map_traj_user[df_map_traj_user['User']==185]\n",
    "    #     print(u_185)\n",
    "    #     print('User 185:',' mean=',u_185['Traj_length'].mean(),' std=',u_185['Traj_length'].std())\n",
    "    #     df_map_traj_user.groupby(['label']).nunique()['new_tid'].mean()\n",
    "        r = (df_map_traj_user.groupby(['User'])['Traj_length']\n",
    "             .agg([np.count_nonzero,np.mean,np.std])\n",
    "             .rename(columns={'count_nonzero':'Count_trajs',\n",
    "                              'mean':'AVG_traj_length',\n",
    "                              'std':'STD_traj_length'}))\n",
    "    #     print('Número médio de trajs por usuário = {:.2f} com DP = {:.2f}'.format(r['Count_trajs'].mean(),\n",
    "    #                                                                        r['Count_trajs'].std()))\n",
    "    #     print('Tamanho médio das trajs por usuário = {:.2f} com DP médio = {:.2f}'.format(r['AVG_traj_length'].mean(),\n",
    "    #                                                                        r['STD_traj_length'].mean()))\n",
    "        return df_map_traj_user\n",
    "    except:\n",
    "        raise('Please, check the input data format.')\n",
    "        return None"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}