notebooks/sample_submit_dsm2_historical.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dmsbatch import create_batch_client, create_blob_client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First create a batch client from the config file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = create_batch_client('../tests/data/dmsbatch.config')\n",
    "blob_client = create_blob_client('../tests/data/dmsbatch.config')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Application packages\n",
    "To copy large files and programs it is best to zip (or targz) them and upload them as application packages\n",
    "\n",
    "Application packages are setup separately in either azure management apis or from the web console or cli tool\n",
    "\n",
    "These are referenced here by their name and version\n",
    "e.g. DSM2, python and other programs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "app_pkgs = [('dsm2', '8.2.c5aacef7', 'DSM2-8.2.c5aacef7-win32/bin'),\n",
    "            ('vista', '1.0-v2019-05-28', 'bin'),\n",
    "            ('unzip', '5.51-1', 'bin')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create or resize existing pool\n",
    "If the pool doesn't exist it will create it\n",
    "If the pool exists, it will resize to the second arg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.create_or_resize_pool('dsm2winpool',\n",
    "                   1,\n",
    "                   app_packages=[(app,version) for app,version,_ in app_pkgs], \n",
    "                   vm_size='standard_f2s_v2', \n",
    "                   tasks_per_vm=2,\n",
    "                   os_image_data=('microsoftwindowsserver', 'windowsserver', '2019-datacenter-core')\n",
    "                  )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create job on pool or fail if it exists\n",
    "Jobs are containers of tasks (things that run on nodes (machines) in the pool). If this exists, the next line will fail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    client.create_job('dsm2jobs','dsm2winpool')\n",
    "except:\n",
    "    print('Job already exists, but we will continue')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Upload input files\n",
    "Zip the input files and add them to storage container (in this case the auto storage associated with the batch account)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, datetime\n",
    "userid = os.getlogin()\n",
    "tsnow = str(datetime.datetime.now().timestamp()).split('.')[0]\n",
    "task_name = f'hydro_run_{tsnow}_{userid}'\n",
    "print(task_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "local_dir = '../tests/data/dsm2v821'\n",
    "input_file=blob_client.zip_and_upload('dsm2jobs',f'{task_name}/dsm2v821.zip',local_dir,30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file = client.create_input_file_spec('dsm2jobs',blob_prefix=f'{task_name}/dsm2v821.zip',file_path='.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dmsbatch\n",
    "permissions = dmsbatch.commands.BlobSasPermissions(write=True)\n",
    "# |helpers.azureblob.BlobPermissions.ADD|helpers.azureblob.BlobPermissions.CREATE\n",
    "output_dir_sas_url = blob_client.get_container_sas_url('dsm2jobs', permissions)\n",
    "print(output_dir_sas_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "std_out_files = client.create_output_file_spec('../std*.txt', output_dir_sas_url, blob_path=f'{task_name}')\n",
    "output_dir = client.create_output_file_spec('**/output/*',output_dir_sas_url, blob_path=f'{task_name}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a task\n",
    "This uses the application package as pre -set up. If not, create one https://docs.microsoft.com/en-us/azure/batch/batch-application-packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tsnow = str(datetime.datetime.now().timestamp()).split('.')[0]\n",
    "#task_name = f'hydro_run_{tsnow}'\n",
    "cmd_string = client.wrap_cmd_with_app_path(f'cd {task_name} & unzip dsm2v821.zip & del dsm2v821.zip & cd study_templates/historical & hydro hydro.inp & qual qual_ec.inp',app_pkgs)\n",
    "print(cmd_string)\n",
    "hydro_task = client.create_task(task_name,cmd_string, resource_files=[input_file],output_files=[std_out_files,output_dir])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "task_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Next submit the task and wait "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#client.submit_tasks_and_wait('dsm2jobs',[hydro_task],poll_secs=120,wait_time_mins=300)\n",
    "client.submit_tasks('dsm2jobs',[hydro_task])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.wait_for_tasks_to_complete('dsm2jobs',timeout=datetime.timedelta(minutes=100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finally resize the pool to 0 to save costs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.resize_pool('dsm2winpool',0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "9e539dc02fc56d55a22d79a0646788fa38cb6ebb3e85aa69306aae5c2f643a8f"
  },
  "kernelspec": {
   "display_name": "Python [conda env:dev_azure]",
   "language": "python",
   "name": "conda-env-dev_azure-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}