2024-01-02-process-run-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import polars as pl\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "task = \"imagenet (a)\"\n",
    "# task = \"imagenet (b)\"\n",
    "# task = \"segmentation\"\n",
    "\n",
    "c0 = \"accuracy\"\n",
    "c1 = \"multiply-adds\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General 'knowledge'\n",
    "improvement_direction = {\n",
    "    \"accuracy\": 1,\n",
    "    \"loss\": -1,\n",
    "    \"loss-clip\": -1,\n",
    "    \"total bytes\": -1,\n",
    "    \"total_memory_bytes\": -1, # dict name\n",
    "    \"multiply-adds\": -1,\n",
    "    \"total_mult_adds\": -1, # dict name\n",
    "    # \"genotype\": 0, # -- not a criterion\n",
    "}\n",
    "best_possible_value = {\n",
    "    \"accuracy\": 1.0,\n",
    "    \"loss\": 0.0,\n",
    "    \"loss-clip\": 0.0,\n",
    "    \"total bytes\": 0.0,\n",
    "    \"total_memory_bytes\": 0.0,\n",
    "    \"multiply-adds\": 0.0,\n",
    "    \"total_mult_adds\": 0.0, \n",
    "}\n",
    "\n",
    "# Task specific settings\n",
    "if task == \"imagenet (a)\":\n",
    "    # imagenet-a\n",
    "    folder = Path(\"./2024-01-02-results/imagenet_a/\")\n",
    "    assert folder.exists()\n",
    "    run_folder = folder / \"exp-imagenet-a\"\n",
    "    assert run_folder.exists()\n",
    "    files = list(run_folder.glob(\"*.arrow\"))\n",
    "    reference_file = folder / \"stitched-imagenet-a-reference.arrow\"\n",
    "    # some task-specific tidbits\n",
    "    min_accuracy = 0.7\n",
    "    best_possible_value[\"accuracy\"] = 0.8\n",
    "elif task == \"imagenet (b)\":\n",
    "    # imagenet-b\n",
    "    folder = Path(\"./2024-01-04\")\n",
    "    assert folder.exists()\n",
    "    run_folder = folder / \"exp-imagenet-b\"\n",
    "    assert run_folder.exists()\n",
    "    files = list(run_folder.glob(\"*.arrow\"))\n",
    "    reference_file = folder / \"stitched-imagenet-b-a-resnet50-b-resnext50_32x4d-reference.arrow\"\n",
    "    # some task-specific tidbits\n",
    "    min_accuracy = 0.7\n",
    "    best_possible_value[\"accuracy\"] = 0.8\n",
    "\n",
    "elif task == \"segmentation\":\n",
    "    folder = Path(\"./2024-01-02-results/segmentation/\")\n",
    "    assert folder.exists()\n",
    "    run_folder = folder / \"exp-voc\"\n",
    "    assert run_folder.exists()\n",
    "    files = list(run_folder.glob(\"*.arrow\"))\n",
    "    reference_file = folder / \"stitched-voc-reference.arrow\"\n",
    "    # some task-specific tidbits\n",
    "    min_accuracy = 0.90\n",
    "else:\n",
    "    raise ValueError(\"Unknown task\")\n",
    "\n",
    "if c0 == \"accuracy\":\n",
    "    min_performance = min_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load reference front\n",
    "reference_samples = pl.read_ipc(reference_file).rename({\n",
    "    \"total_memory_bytes\": \"total bytes\",\n",
    "    \"total_mult_adds\": \"multiply-adds\",\n",
    "}).with_columns(\n",
    "    pl.col(\"genotype\").list.last().map_dict({\n",
    "        0: \"a\",\n",
    "        1: \"b\",\n",
    "        2: \"ensemble\",\n",
    "    }).alias(\"set\"),\n",
    "    pl.lit(reference_file.name).alias(\"file\")\n",
    ")\n",
    "\n",
    "# Collect some statistics\n",
    "run_info = []\n",
    "common_columns = None\n",
    "\n",
    "def load_run(filepath):\n",
    "    global run_info\n",
    "    global common_columns\n",
    "    # As grouping label - use the file path - which is unique across any run.\n",
    "    file_ref = filepath.name\n",
    "\n",
    "    # Determine legend label - i.e. the approach used\n",
    "    set_name = \"unk\"\n",
    "    filepath_split = Path(filepath).name.split(\"-\")\n",
    "    if len(filepath_split) > 2:\n",
    "        set_name = filepath_split[1]\n",
    "\n",
    "    # Load data - note: here we always assume the files have already been converted to arrow\n",
    "    # e.g. using jsonl-to-arrow.py.\n",
    "    approach_samples = pl.read_ipc(filepath).lazy().with_row_count(name=\"#eval\")\n",
    "    num_samples_evaluated = approach_samples.select(pl.count()).collect().item()\n",
    "\n",
    "    # Clean-up data\n",
    "    approach_samples = approach_samples.drop_nulls(\"loss\").rename({\n",
    "        \"total_memory_bytes\": \"total bytes\",\n",
    "        \"total_mult_adds\": \"multiply-adds\",\n",
    "    }).with_columns([\n",
    "        pl.lit(set_name).alias(\"set\"),\n",
    "        pl.lit(file_ref).alias(\"file\"),\n",
    "    ])\n",
    "    num_samples_evaluated_active = approach_samples.select(pl.count()).collect().item()\n",
    "    num_samples_evaluated_inactive = num_samples_evaluated - num_samples_evaluated_active\n",
    "\n",
    "    # Collect data on how many evaluations were short-circuited.\n",
    "    run_info.append({\n",
    "        \"file_ref\": file_ref,\n",
    "        \"set_name\": set_name,\n",
    "        \"num_samples_evaluated\": num_samples_evaluated,\n",
    "        \"num_samples_evaluated_active\": num_samples_evaluated_active,\n",
    "        \"num_samples_evaluated_inactive\": num_samples_evaluated_inactive,\n",
    "    })\n",
    "\n",
    "    common_columns = list(set(approach_samples.columns).intersection(reference_samples.columns))\n",
    "    approach_samples = approach_samples.select(common_columns)\n",
    "\n",
    "    if min_performance is not None:\n",
    "        approach_samples = approach_samples.filter(improvement_direction[c0] * pl.col(c0) > improvement_direction[c0] * min_performance)\n",
    "\n",
    "    return approach_samples\n",
    "\n",
    "runs_data = pl.concat(load_run(f) for f in files)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_info = pl.DataFrame(run_info)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute some preliminary statistics:\n",
    "# - How much of the budget is used?\n",
    "# - How often is the inactive variable short-circuit activated?\n",
    "task_ = task.replace(\" \", \"_\").replace(\"(\", \"\").replace(\")\", \"\")\n",
    "run_summary = run_info.lazy().with_columns(\n",
    "    (pl.col(\"num_samples_evaluated_active\") / pl.col(\"num_samples_evaluated\")).alias(\"num_samples_evaluated_active_ratio\")\n",
    ").group_by(\"set_name\").agg([\n",
    "    pl.col(\"^num_samples.*$\").median().map_alias(lambda x: x + \"_median\"),\n",
    "    pl.col(\"^num_samples.*$\").quantile(0.20).map_alias(lambda x: x + \"_q20\"),\n",
    "    pl.col(\"^num_samples.*$\").quantile(0.80).map_alias(lambda x: x + \"_q80\"),\n",
    "]).sort(\"set_name\").collect()\n",
    "run_summary.write_csv(f\"{task}_activity_stats.csv\")\n",
    "run_summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare for front computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensure reference has common samples, too.\n",
    "reference_samples = reference_samples.select(common_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tools for computing fronts\n",
    "def maybe_over(a, o):\n",
    "    if len(o) == 0: return a\n",
    "    else: return a.over(o)\n",
    "\n",
    "def compute_pareto(df, group_vars):\n",
    "    return (df.sort(c0, descending=improvement_direction[c0] > 0)\n",
    "        .with_columns((pl.col(c1) * -improvement_direction[c1]).alias(\"c1-min\"))\n",
    "        .with_columns(maybe_over((pl.col(\"c1-min\")).cummin(), group_vars).alias(\"mv\"))\n",
    "        .with_columns((maybe_over(pl.col(\"c1-min\") < pl.col(\"mv\").shift(1), group_vars).alias(\"is pareto\")).fill_null(True))\n",
    "        .filter(pl.col(\"is pareto\"))\n",
    "    )\n",
    "\n",
    "def compute_2d_hv(df_pareto, ref, axis_scale, group_vars):\n",
    "    # note - df_pareto is a df created using compute_pareto\n",
    "    dhva = (df_pareto.sort(c0, descending=improvement_direction[c0] < 0)\n",
    "        # Samples worse than reference point do not contribute.\n",
    "        # .filter(improvement_direction[c0] * pl.col(c0) > improvement_direction[c0] * ref[0])\n",
    "        # .filter(improvement_direction[c1] * pl.col(c1) > improvement_direction[c1] * ref[1])\n",
    "        .with_columns(\n",
    "        [\n",
    "            maybe_over( improvement_direction[c0] * (pl.col(c0) - pl.col(c0).shift(1).fill_null(ref[0])) / axis_scale[0], group_vars).alias(\"slice_width\"),\n",
    "            maybe_over( improvement_direction[c1] * (pl.col(c1) - ref[1]) / axis_scale[1], group_vars).alias(\"slice_height\"),\n",
    "        ])\n",
    "        .select([pl.col(group_vars), (pl.col(\"slice_width\") * pl.col(\"slice_height\")).alias(\"hv_contrib\")])\n",
    "        .group_by(group_vars).agg(pl.col(\"hv_contrib\").sum()))\n",
    "    return dhva"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute front over all evaluated solutions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Front over all samples over all runs & reference points\n",
    "all_data = pl.concat([\n",
    "    runs_data,\n",
    "    reference_samples.lazy(),\n",
    "], how=\"diagonal_relaxed\")\n",
    "total_front = compute_pareto(all_data, []).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sidenote - only used for imagenet a for training experiment.\n",
    "# total_front.write_ipc(\"imagenet-a-all-evals-front.arrow\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.step(total_front[c0], total_front[c1], where=\"post\", alpha=0.5) # label=sn \n",
    "\n",
    "plt.scatter(total_front[c0], total_front[c1], label=\"best\", alpha=0.5, s=8.0)\n",
    "plt.scatter(reference_samples[c0], reference_samples[c1], label=\"reference\", alpha=0.5, s=18.0, marker='x')\n",
    "\n",
    "# Complete the bounding box for the front\n",
    "ax = plt.gca()\n",
    "hline_p = ax.transAxes.inverted().transform(ax.transData.transform((total_front[-1, c0], 0)))[0]\n",
    "vline_p = ax.transAxes.inverted().transform(ax.transData.transform((0, total_front[0, c1])))[1]\n",
    "plt.axhline(total_front[-1, c1], xmin=0 if improvement_direction[c0] > 0 else 1, xmax=hline_p, alpha=0.5)\n",
    "plt.axvline(total_front[0, c0], ymin=0 if improvement_direction[c1] > 0 else 1, ymax=vline_p, alpha=0.5)\n",
    "\n",
    "plt.title(f\"Approximation front over all runs - {task}\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute front per run\n",
    "With respect to a reference depending on all evaluated solutions / predetermined bounds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute reference point & scale for hypervolume\n",
    "# For the reference point we use the nadir: the worst combination of objectives.\n",
    "# Note, that this is computed /after/ a minimal fitness filter.\n",
    "worst_perf_c0, worst_perf_c1 = all_data.select(\n",
    "    (improvement_direction[c0] * (improvement_direction[c0] * pl.col(c0)).min()).alias(c0),\n",
    "    (improvement_direction[c1] * (improvement_direction[c1] * pl.col(c1)).min()).alias(c1)\n",
    "    ).collect().row(0)\n",
    "# Override, if we have a filter threshold.\n",
    "if min_performance is not None:\n",
    "    worst_perf_c0 = min_performance\n",
    "scale_c0 = -improvement_direction[c0] * (worst_perf_c0 - best_possible_value[c0])\n",
    "scale_c1 = -improvement_direction[c1] * (worst_perf_c1 - best_possible_value[c1])\n",
    "\n",
    "hv_reference_point = (worst_perf_c0, worst_perf_c1)\n",
    "hv_scale = (scale_c0, scale_c1)\n",
    "\n",
    "print(f\"Reference point: ({worst_perf_c0}, {worst_perf_c1}) - Scale: ({scale_c0}, {scale_c1})\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "per_run_front = compute_pareto(runs_data.lazy(), [\"file\"]).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "per_run_front.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colormap = mpl.colormaps[\"Set1\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "per_run_alpha = 0.3\n",
    "for c, (approach, multifront) in zip(colormap.colors, per_run_front.sort(\"file\").group_by(\"set\", maintain_order=True)):\n",
    "    is_first = True\n",
    "    for _file_path, front in multifront.group_by(\"file\"):\n",
    "        plt.step(front[c0], front[c1], where=\"post\", alpha=per_run_alpha, color=c) # label=sn \n",
    "        plt.scatter(front[c0], front[c1], label=approach if is_first else None, alpha=per_run_alpha, s=8.0, color=c)\n",
    "        is_first = False\n",
    "\n",
    "# Complete the bounding box for each front\n",
    "ax = plt.gca()\n",
    "for c, (approach, multifront) in zip(colormap.colors, per_run_front.sort(\"file\").group_by(\"set\", maintain_order=True)):\n",
    "    is_first = True\n",
    "    for _file_path, front in multifront.group_by(\"file\"):\n",
    "        hline_p = ax.transAxes.inverted().transform(ax.transData.transform((front[-1, c0], 0)))[0]\n",
    "        vline_p = ax.transAxes.inverted().transform(ax.transData.transform((0, front[0, c1])))[1]\n",
    "        plt.axhline(front[-1, c1], xmin=0 if improvement_direction[c0] > 0 else 1, xmax=hline_p, alpha=per_run_alpha, color=c)\n",
    "        plt.axvline(front[0, c0], ymin=0 if improvement_direction[c1] > 0 else 1, ymax=vline_p, alpha=per_run_alpha, color=c)\n",
    "\n",
    "plt.scatter(reference_samples[c0], reference_samples[c1], label=\"reference\", alpha=1.0, s=32.0, color=\"orange\", marker='x')\n",
    "plt.title(f\"Individual approximation fronts - {task}\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(per_run_front)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store a list of solutions to evaluate as part of a 'front'\n",
    "task_ = task.replace(\" \", \"_\").replace(\"(\", \"\").replace(\")\", \"\")\n",
    "per_run_out = f\"{task_}-front.arrow\"\n",
    "pl.concat([per_run_front.lazy().select(reference_samples.columns), reference_samples.lazy().with_columns([pl.lit(\"reference\").alias(\"file\"), pl.lit(\"reference\").alias(\"set\")])]).collect().write_ipc(per_run_out)\n",
    "per_run_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute hypervolume per run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hv_per_run = (compute_2d_hv(\n",
    "                pl.concat([per_run_front.lazy(), reference_samples.lazy().with_columns([pl.lit(\"reference\").alias(\"file\"), pl.lit(\"reference\").alias(\"set\")])]), \n",
    "                hv_reference_point,\n",
    "                hv_scale, [\"file\", \"set\"]).sort(\"hv_contrib\")\n",
    "              .with_columns(pl.col(\"set\").replace({\"SGA\": \"GA\"}))\n",
    "              .collect())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hv_per_run.write_csv(f\"./2024-01-02-hypervolumes-{task_}.csv\")\n",
    "hv_per_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hv_summary = (hv_per_run.lazy().group_by(\"set\").agg(\n",
    "    pl.col(\"hv_contrib\").mean().alias(\"hv_mean\"),\n",
    "    pl.col(\"hv_contrib\").std().alias(\"hv_std\"),\n",
    "    pl.col(\"hv_contrib\").quantile(0.1).alias(\"hv_q10\"),\n",
    "    pl.col(\"hv_contrib\").median().alias(\"hv_median\"),\n",
    "    pl.col(\"hv_contrib\").quantile(0.9).alias(\"hv_q90\"),\n",
    ")).sort(\"set\").collect()\n",
    "hv_summary.write_csv(f\"./2024-01-02-hypervolumes-summary-{task_}.csv\")\n",
    "hv_summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot front with median hypervolume per approach\n",
    "As to showcase a representative front."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine the run closest to the median hypervolume for each configuration.\n",
    "middle_runs = (hv_per_run.lazy().filter(pl.col(\"file\") != \"reference\" ).with_columns(((pl.col(\"hv_contrib\").arg_sort() - (pl.count() - 1) / 2).abs() <= 0.5).over(\"set\").alias(\"is_middle\")).collect())\n",
    "middle_runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "per_run_alpha = 0.3\n",
    "\n",
    "middle_run_fronts = middle_runs.lazy().filter(\"is_middle\").join(per_run_front.lazy(), on=\"file\").sort(\"file\").collect().group_by(\"set\", maintain_order=True)\n",
    "\n",
    "for c, (approach, multifront) in zip(colormap.colors, middle_run_fronts):\n",
    "    is_first = True\n",
    "    for _file_path, front in multifront.group_by(\"file\"):\n",
    "        plt.step(front[c0], front[c1], where=\"post\", alpha=per_run_alpha, color=c) # label=sn \n",
    "        plt.scatter(front[c0], front[c1], label=approach if is_first else None, alpha=per_run_alpha, s=8.0, color=c)\n",
    "        is_first = False\n",
    "\n",
    "# Complete the bounding box for each front\n",
    "ax = plt.gca()\n",
    "for c, (approach, multifront) in zip(colormap.colors, middle_run_fronts):\n",
    "    is_first = True\n",
    "    for _file_path, front in multifront.group_by(\"file\"):\n",
    "        hline_p = ax.transAxes.inverted().transform(ax.transData.transform((front[-1, c0], 0)))[0]\n",
    "        vline_p = ax.transAxes.inverted().transform(ax.transData.transform((0, front[0, c1])))[1]\n",
    "        plt.axhline(front[-1, c1], xmin=0 if improvement_direction[c0] > 0 else 1, xmax=hline_p, alpha=per_run_alpha, color=c)\n",
    "        plt.axvline(front[0, c0], ymin=0 if improvement_direction[c1] > 0 else 1, ymax=vline_p, alpha=per_run_alpha, color=c)\n",
    "\n",
    "\n",
    "plt.scatter(reference_samples[c0], reference_samples[c1], label=\"reference\", alpha=1.0, s=32.0, color=\"orange\", marker='x')\n",
    "plt.title(f\"fronts of run with Median HV - {task}\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine the run closest to the median hypervolume for each configuration.\n",
    "picked_runs = (hv_per_run.lazy().filter(pl.col(\"file\") != \"reference\" ).with_columns(((pl.col(\"hv_contrib\").arg_sort() - (pl.count() - 1)).abs() <= 0.5).over(\"set\").alias(\"is_chosen\")).collect())\n",
    "per_run_alpha = 0.3\n",
    "\n",
    "picked_runs_fronts = picked_runs.lazy().filter(\"is_chosen\").join(per_run_front.lazy(), on=\"file\").sort(\"file\").collect().group_by(\"set\", maintain_order=True)\n",
    "\n",
    "for c, (approach, multifront) in zip(colormap.colors, picked_runs_fronts):\n",
    "    is_first = True\n",
    "    for _file_path, front in multifront.group_by(\"file\"):\n",
    "        plt.step(front[c0], front[c1], where=\"post\", alpha=per_run_alpha, color=c) # label=sn \n",
    "        plt.scatter(front[c0], front[c1], label=approach if is_first else None, alpha=per_run_alpha, s=8.0, color=c)\n",
    "        is_first = False\n",
    "\n",
    "# Complete the bounding box for each front\n",
    "ax = plt.gca()\n",
    "for c, (approach, multifront) in zip(colormap.colors, picked_runs_fronts):\n",
    "    is_first = True\n",
    "    for _file_path, front in multifront.group_by(\"file\"):\n",
    "        hline_p = ax.transAxes.inverted().transform(ax.transData.transform((front[-1, c0], 0)))[0]\n",
    "        vline_p = ax.transAxes.inverted().transform(ax.transData.transform((0, front[0, c1])))[1]\n",
    "        plt.axhline(front[-1, c1], xmin=0 if improvement_direction[c0] > 0 else 1, xmax=hline_p, alpha=per_run_alpha, color=c)\n",
    "        plt.axvline(front[0, c0], ymin=0 if improvement_direction[c1] > 0 else 1, ymax=vline_p, alpha=per_run_alpha, color=c)\n",
    "\n",
    "\n",
    "plt.scatter(reference_samples[c0], reference_samples[c1], label=\"reference\", alpha=1.0, s=32.0, color=\"orange\", marker='x')\n",
    "plt.title(f\"fronts of run - selected based on HV, somehow - {task}\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nicer graphs with R"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import rpy2\n",
    "# Configure for notebook use.\n",
    "import rpy2.ipython.html\n",
    "rpy2.ipython.html.init_printing()\n",
    "%load_ext rpy2.ipython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%R\n",
    "library(ggplot2)\n",
    "library(dplyr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dataframe for plotting in R\n",
    "pd_per_run_front = (per_run_front.lazy()\n",
    "    # Add a tag so that we can track which samples were original - and which ones were added for sake\n",
    "    # of continuing the lines.\n",
    "    .with_columns(pl.lit(1.0).alias(\"is_original\"))\n",
    "    # For each run include an additional two rows:\n",
    "    # Repeat best per objective, but replace the other objective with -Inf - as to plot towards the axes.\n",
    "    .merge_sorted(per_run_front.lazy()\n",
    "                  .with_columns([(pl.col(c0) * improvement_direction[c0]).alias(\"c0-n\"),\n",
    "                                 pl.lit(-np.Inf * improvement_direction[c1]).alias(c1),\n",
    "                                 pl.lit(0.0).alias(\"is_original\")])\n",
    "                  .group_by(\"file\", maintain_order=True)\n",
    "                  .agg(pl.all().sort_by(\"c0-n\").last())\n",
    "                  .select(per_run_front.columns + [\"is_original\"]), \"file\")\n",
    "    .merge_sorted(per_run_front.lazy()\n",
    "                  .with_columns([(pl.col(c1) * improvement_direction[c1]).alias(\"c1-n\"),\n",
    "                                 pl.lit(-np.Inf * improvement_direction[c0]).alias(c0),\n",
    "                                 pl.lit(0.0).alias(\"is_original\")])\n",
    "                  .group_by(\"file\", maintain_order=True)\n",
    "                  .agg(pl.all().sort_by(\"c1-n\").last())\n",
    "                  .select(per_run_front.columns + [\"is_original\"]), \"file\")\n",
    "    # Add c0 and c1 as a named column\n",
    "    .with_columns([pl.col(c0).alias(\"c0\"), pl.col(c1).alias(\"c1\")])\n",
    "    # Collect & convert to pandas in order to transfer.\n",
    "    .collect().to_pandas())\n",
    "\n",
    "# Convert reference points\n",
    "pd_reference_samples = (reference_samples.lazy()\n",
    "    # Add c0 and c1 as a named column\n",
    "    .with_columns([pl.col(c0).alias(\"c0\"), pl.col(c1).alias(\"c1\")])\n",
    "    # Collect & convert to pandas in order to transfer.\n",
    "    .collect().to_pandas())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%R -i pd_per_run_front -i pd_reference_samples -i c0 -i c1\n",
    "\n",
    "# Remove additional samples added to continue the lines to the axis edge\n",
    "pd_per_run_front_excl_edges <- pd_per_run_front |> filter(`is_original` > 0.5)\n",
    "\n",
    "ggplot(pd_per_run_front, aes(x = `c0`, y=`c1`, color=`set`, group=`file`)) +\n",
    "    geom_step(alpha=0.3, direction = \"vh\") +\n",
    "    geom_point(data = pd_per_run_front_excl_edges, alpha=0.3) +\n",
    "    geom_point(data = pd_reference_samples, color=\"orange\", group=\"reference\", shape=4, size=2, stroke=2) +\n",
    "    labs(x = c0, y = c1, color = \"approach\") +\n",
    "    theme_bw() +\n",
    "    theme(\n",
    "      legend.position=\"bottom\",\n",
    "      axis.text.x = element_text(angle = 45, vjust = 1, hjust=1),\n",
    "      plot.background = element_rect(fill='transparent', color=NA),\n",
    "      strip.background = element_blank())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "recombnet",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}