demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Self-Supervised Syllable Discovery Based on Speaker-Disentangled HuBERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only required for Google Colab\n",
    "!git clone https://github.com/ryota-komatsu/speaker_disentangled_hubert\n",
    "%cd speaker_disentangled_hubert"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -r requirements/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd src/s5hubert/mincut\n",
    "!python setup.py build_ext --inplace\n",
    "%cd -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://github.com/cheoljun95/sdhubert.git src/sdhubert"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download LibriSpeech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget -t 0 -c -P data/LibriSpeech https://www.openslr.org/resources/12/test-clean.tar.gz\n",
    "!tar zxvf data/LibriSpeech/test-clean.tar.gz -C data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torchaudio\n",
    "\n",
    "from src.s5hubert import S5HubertForSyllableDiscovery"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "syllable_alignment_path = \"src/sdhubert/files/librispeech_syllable_test.json\"\n",
    "wav_name = \"test-clean/61/70968/61-70968-0021.flac\"\n",
    "wav_path = \"data/LibriSpeech/test-clean/61/70968/61-70968-0021.flac\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = S5HubertForSyllableDiscovery.from_pretrained(\"ryota-komatsu/s5-hubert\").cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load audio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "waveform, sr = torchaudio.load(wav_path)\n",
    "waveform = torchaudio.functional.resample(waveform, sr, 16000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = model(waveform.cuda())\n",
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame_boundary = outputs[\"frame_boundary\"]\n",
    "frame_similarity = outputs[\"frame_similarity\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load ground truth syllable alignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "refs = {}\n",
    "with open(syllable_alignment_path) as f:\n",
    "    syllables = json.load(f)\n",
    "    for item in syllables.values():\n",
    "        boundary = []\n",
    "        labels = []\n",
    "        ticks = []\n",
    "        for syllable in item[\"syllables\"]:\n",
    "            start = round(float(syllable[\"start\"]) / 0.02)\n",
    "            end = round(float(syllable[\"end\"]) / 0.02)\n",
    "\n",
    "            boundary.append([start, end])\n",
    "            labels.append(syllable[\"label\"])\n",
    "            ticks.append((start + end) / 2)\n",
    "        refs.update(\n",
    "            {\n",
    "                item[\"file_name\"]: {\n",
    "                    \"boundary\": np.unique(boundary),\n",
    "                    \"labels\": labels,\n",
    "                    \"ticks\": ticks,\n",
    "                }\n",
    "            }\n",
    "        )\n",
    "\n",
    "ref_boundary = refs[wav_name][\"boundary\"]\n",
    "labels = refs[wav_name][\"labels\"]\n",
    "ticks = refs[wav_name][\"ticks\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.imshow(frame_similarity)\n",
    "plt.vlines(ref_boundary, 0, frame_similarity.shape[1] - 1, colors=\"red\", label=\"ground truth\")\n",
    "plt.vlines(\n",
    "    frame_boundary[:, 0], 0, frame_similarity.shape[1] - 1, colors=\"white\", linestyles=\"dotted\", label=\"prediction\"\n",
    ")\n",
    "plt.xticks(ticks=ticks, labels=labels, rotation=-60, color=\"red\")\n",
    "plt.yticks([], [])\n",
    "plt.legend()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py310",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}