dataset_generate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from utils.data_preprocess_utils import get_config\n",
    "import utils.dataset_generate_utils as gen\n",
    "import utils.data_preprocess_utils as pre\n",
    "import warnings\n",
    "import pandas as pd\n",
    "from easydict import EasyDict\n",
    "from concurrent.futures import ProcessPoolExecutor, as_completed\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "# ignore warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 1362/1461 [15:35:53<49:45, 30.16s/it]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error processing day 2022-10-01 00:00:00: File is too small to be a well-formed file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 1365/1461 [15:38:38<1:15:42, 47.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error processing day 2022-10-04 00:00:00: File is too small to be a well-formed file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 1387/1461 [15:50:03<39:06, 31.70s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error processing day 2022-10-26 00:00:00: [Errno 2] No such file or directory: '/root/autodl-tmp/data_processed/aisdk-2022-10-26.feather'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1461/1461 [16:28:44<00:00, 40.61s/it]  \n"
     ]
    }
   ],
   "source": [
    "\n",
    "def process_day(day):\n",
    "    try:\n",
    "        day_str = str(day).split(' ')[0]\n",
    "        df = pd.read_feather(files_dict[day_str])\n",
    "        df.sort_values(by=['MMSI', 'time'], inplace=True)\n",
    "        df['inout'] = 0\n",
    "        df['label'] = -1\n",
    "        df = df.groupby('MMSI').apply(gen.get_traj_point_type, cfg)\n",
    "        time_intervals = cfg.time_intervals\n",
    "        for time_interval in time_intervals:\n",
    "            day_dic = gen.generate_dataset(day, cfg, df, time_interval, keep_unlabeled=False)\n",
    "            file_name = cfg.out_dir + str(time_interval) + '/'+  day_str + '.npy'\n",
    "            np.save(file_name, day_dic)\n",
    "        \n",
    "    except Exception as e:\n",
    "        print(f\"Error processing day {day}: {e}\")\n",
    "    \n",
    "# 6 12 24\n",
    "date_range = pd.date_range(start='2019-01-01', end='2022-12-31', freq='1D')\n",
    "cfg = EasyDict(get_config('./cfg/dataset_generate_cfg.yaml'))\n",
    "files = pre.get_all_files(cfg.in_dir)\n",
    "files_dict = pre.files2dict(files)\n",
    "time_intervals = cfg.time_intervals\n",
    "for time_interval in time_intervals:\n",
    "    os.makedirs(cfg.out_dir + str(time_interval), exist_ok=True)\n",
    "with ProcessPoolExecutor(max_workers=8) as executor:\n",
    "    # Create a list of futures for each day\n",
    "    futures = [executor.submit(process_day, day) for day in date_range]\n",
    "\n",
    "    # Display progress using tqdm\n",
    "    for _ in tqdm(as_completed(futures), total=len(futures)):\n",
    "        pass  # Do nothing here as we are using tqdm for progress display\n",
    "\n",
    "# Ensure all tasks are completed\n",
    "for future in futures:\n",
    "    future.result()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rapids-23.10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}