4-WH-Score-Estimator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training a Score Estimator (SALLY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MadMiner version: 0.4.9\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "madminer_src_path = \"/home/shomiller/madminer\"\n",
    "sys.path.append(madminer_src_path)\n",
    "\n",
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "import logging\n",
    "\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib\n",
    "from matplotlib import pyplot as plt\n",
    "from scipy.optimize import curve_fit\n",
    "% matplotlib inline\n",
    "\n",
    "from madminer.sampling import SampleAugmenter\n",
    "from madminer import sampling\n",
    "from madminer.ml import ScoreEstimator, Ensemble\n",
    "\n",
    "import madminer.__version__\n",
    "print( 'MadMiner version: {}'.format(madminer.__version__) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# MadMiner output\n",
    "logging.basicConfig(\n",
    "    format='%(asctime)-5.5s %(name)-20.20s %(levelname)-7.7s %(message)s',\n",
    "    datefmt='%H:%M',\n",
    "    level=logging.INFO\n",
    ")\n",
    "\n",
    "# Output of all other modules (e.g. matplotlib)\n",
    "for key in logging.Logger.manager.loggerDict:\n",
    "    if \"madminer\" not in key:\n",
    "        logging.getLogger(key).setLevel(logging.WARNING)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we define a function `augment_and_train`, which creates the augmented (unweighted) training and test samples, then runs the score estimator (using the `SALLY` method) to create a model for a given dataset. With the form of all our datafiles from the previous notebooks, we can run this with just two arguments, `channel` (e.g., `wph_mu`) and `observables` (e.g, `full` or `met`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def augment_and_train(channel, observables, nsamples, is_signal_only=False):\n",
    "\n",
    "    n_estimators = 5\n",
    "\n",
    "    print('Creating Training Samples...\\n')\n",
    "\n",
    "    if observables == 'ptw' or '2d':\n",
    "        sampler_obs = 'met'\n",
    "    else:\n",
    "        sampler_obs = observables\n",
    "    \n",
    "    # Make (unweighted) training and test samples with augmented data\n",
    "    \n",
    "    if is_signal_only:\n",
    "        sampler = SampleAugmenter('data/{}/signal/{}_lhedata_{}.h5'.format(sampler_obs, channel, sampler_obs))\n",
    "    else:\n",
    "        sampler = SampleAugmenter('data/{}/{}_lhedata_{}.h5'.format(sampler_obs, channel, sampler_obs))\n",
    "    \n",
    "\n",
    "    #create training samples (the same number as the number of estimators we want)\n",
    "    for i in range(n_estimators):\n",
    "        x, theta, t_xz, _ = sampler.sample_train_local(\n",
    "            theta=sampling.benchmark('sm'),\n",
    "            n_samples=int(nsamples/2.),\n",
    "            folder='./samples/{}/samples_{}_{}'.format(observables, channel, observables),\n",
    "            filename='train_score_{}'.format(i), \n",
    "            sample_only_from_closest_benchmark=False,\n",
    "        )\n",
    "    \n",
    "    print('Creating Testing Samples...\\n')\n",
    "    \n",
    "    #create test sample\n",
    "    _ = sampler.sample_test(\n",
    "        theta=sampling.benchmark('sm'),\n",
    "        n_samples=int(nsamples/2.),\n",
    "        folder='./samples/{}/samples_{}_{}'.format(observables, channel, observables),\n",
    "        filename='test',\n",
    "        sample_only_from_closest_benchmark=False,\n",
    "    )\n",
    "    \n",
    "    #Choose which features to train on \n",
    "    #  if 'met' or 'full', we use all of them (None), \n",
    "    #  otherwise we select the correct indices\n",
    "    if observables == 'met' or 'full':\n",
    "        my_features = None\n",
    "    elif observables == 'ptw':\n",
    "        my_features = [18]\n",
    "    elif observables == '2d':\n",
    "        my_features = [18, 39]\n",
    "    \n",
    "    #Create a list of ScoreEstimator objects to add to the ensemble\n",
    "    estimators = [ ScoreEstimator(features=my_features, n_hidden=(50,)) for _ in range(n_estimators) ]\n",
    "    ensemble = Ensemble(estimators)\n",
    "    \n",
    "    print('Training Ensemble...\\n')\n",
    "    \n",
    "    # Run the Training\n",
    "    ensemble.train_all(\n",
    "        method='sally',\n",
    "        x=[ 'samples/{}/samples_{}_{}/x_train_score_{}.npy'.format(observables, channel, observables, i) for i in range(n_estimators) ],\n",
    "        t_xz=[ 'samples/{}/samples_{}_{}/t_xz_train_score_{}.npy'.format(observables, channel, observables, i) for i in range(n_estimators) ],\n",
    "    )\n",
    "    \n",
    "    #Finally, save our SALLY model to a file we can load later\n",
    "    ensemble.save('models/{}/sally_ensemble_{}_{}'.format(observables, channel, observables))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Run Augmentation and Training (with Backgrounds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Full Observables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_mu_wbkgs','full',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_e_wbkgs','full',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_wbkgs','full',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_wbkgs','full',100*50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### MET Observables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_mu_wbkgs','met',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_e_wbkgs','met',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_wbkgs','met',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_wbkgs','met',100*50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### $p_{T,W}$ Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_mu_wbkgs','ptw',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_e_wbkgs','ptw',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_wbkgs','ptw',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_wbkgs','ptw',100*50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### $p_{T,W}$ and $m_{T,\\mathrm{tot}}$ Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_mu_wbkgs','short_2d',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_e_wbkgs','short_2d',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_wbkgs','short_2d',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_wbkgs','short_2d',100*50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Background Free Runs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Full Observables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13:19 madminer.analysis    INFO    Loading data from data/full/signal/wph_mu_smeftsim_lhedata_full.h5\n",
      "13:19 madminer.analysis    INFO    Found 4 parameters\n",
      "13:19 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "13:19 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "13:19 madminer.analysis    INFO    Found 60 observables\n",
      "13:19 madminer.analysis    INFO    Found 1000000 events\n",
      "13:19 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "13:19 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "13:19 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "13:19 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "13:19 madminer.sampling    INFO    Starting sampling serially\n",
      "13:19 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13:19 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:19 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:19 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "13:19 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "13:19 madminer.sampling    INFO    Starting sampling serially\n",
      "13:19 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "13:19 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:19 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:19 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "13:19 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "13:19 madminer.sampling    INFO    Starting sampling serially\n",
      "13:19 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "13:19 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:19 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:19 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "13:19 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "13:19 madminer.sampling    INFO    Starting sampling serially\n",
      "13:19 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "13:19 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:19 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:19 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "13:19 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "13:19 madminer.sampling    INFO    Starting sampling serially\n",
      "13:19 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "13:20 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:20 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "13:20 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "13:20 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "13:20 madminer.sampling    INFO    Starting sampling serially\n",
      "13:20 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13:20 madminer.sampling    INFO    Effective number of samples: mean 199716.36363, with individual thetas ranging from 199716.36363 to 199716.36363\n",
      "13:20 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "13:20 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "13:20 madminer.ml          INFO    Starting training\n",
      "13:20 madminer.ml          INFO      Batch size:             128\n",
      "13:20 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "13:20 madminer.ml          INFO      Epochs:                 50\n",
      "13:20 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "13:20 madminer.ml          INFO      Validation split:       0.25\n",
      "13:20 madminer.ml          INFO      Early stopping:         True\n",
      "13:20 madminer.ml          INFO      Scale inputs:           True\n",
      "13:20 madminer.ml          INFO      Shuffle labels          False\n",
      "13:20 madminer.ml          INFO      Samples:                all\n",
      "13:20 madminer.ml          INFO    Loading training data\n",
      "13:20 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/x_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13:20 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/t_xz_train_score_0.npy into RAM\n",
      "13:20 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "13:20 madminer.ml          INFO    Rescaling inputs\n",
      "13:20 madminer.ml          INFO    Creating model\n",
      "13:20 madminer.ml          INFO    Training model\n",
      "13:20 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "13:21 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00017 (mse_score:  0.000)\n",
      "13:21 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "13:21 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00016 (mse_score:  0.000)\n",
      "13:21 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "13:22 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "13:22 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "13:23 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00015 (mse_score:  0.000)\n",
      "13:23 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "13:24 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00015 (mse_score:  0.000)\n",
      "13:24 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:25 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00015 (mse_score:  0.000)\n",
      "13:25 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:25 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "13:25 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:26 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "13:26 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:27 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "13:27 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:28 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00014 (mse_score:  0.000)\n",
      "13:28 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:28 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00014 (mse_score:  0.000)\n",
      "13:28 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:29 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00014 (mse_score:  0.000)\n",
      "13:29 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:30 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00014 (mse_score:  0.000)\n",
      "13:30 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:31 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00014 (mse_score:  0.000)\n",
      "13:31 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:32 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00014 (mse_score:  0.000)\n",
      "13:32 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:32 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00014 (mse_score:  0.000)\n",
      "13:32 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:33 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "13:33 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "13:33 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "13:33 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "13:33 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "13:33 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "13:33 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "13:33 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "13:33 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "13:33 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "13:33 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "13:33 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "13:33 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "13:33 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "13:33 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "13:33 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "13:33 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "13:33 madminer.ml          INFO    Starting training\n",
      "13:33 madminer.ml          INFO      Batch size:             128\n",
      "13:33 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "13:33 madminer.ml          INFO      Epochs:                 50\n",
      "13:33 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "13:33 madminer.ml          INFO      Validation split:       0.25\n",
      "13:33 madminer.ml          INFO      Early stopping:         True\n",
      "13:33 madminer.ml          INFO      Scale inputs:           True\n",
      "13:33 madminer.ml          INFO      Shuffle labels          False\n",
      "13:33 madminer.ml          INFO      Samples:                all\n",
      "13:33 madminer.ml          INFO    Loading training data\n",
      "13:33 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/x_train_score_1.npy into RAM\n",
      "13:33 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/t_xz_train_score_1.npy into RAM\n",
      "13:33 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "13:33 madminer.ml          INFO    Rescaling inputs\n",
      "13:33 madminer.ml          INFO    Creating model\n",
      "13:33 madminer.ml          INFO    Training model\n",
      "13:33 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "13:34 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00017 (mse_score:  0.000)\n",
      "13:34 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "13:35 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00015 (mse_score:  0.000)\n",
      "13:35 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "13:35 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "13:35 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:36 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00015 (mse_score:  0.000)\n",
      "13:36 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:37 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00014 (mse_score:  0.000)\n",
      "13:37 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:38 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "13:38 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:38 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "13:38 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:39 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "13:39 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:40 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "13:40 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13:41 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00014 (mse_score:  0.000)\n",
      "13:41 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "13:42 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00014 (mse_score:  0.000)\n",
      "13:42 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "13:42 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00014 (mse_score:  0.000)\n",
      "13:42 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "13:43 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00013 (mse_score:  0.000)\n",
      "13:43 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "13:44 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00013 (mse_score:  0.000)\n",
      "13:44 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "13:45 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00013 (mse_score:  0.000)\n",
      "13:45 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "13:46 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00013 (mse_score:  0.000)\n",
      "13:46 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "13:46 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "13:46 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "13:46 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "13:46 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "13:46 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "13:46 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "13:46 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "13:46 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "13:46 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "13:46 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "13:46 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "13:46 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "13:46 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "13:46 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "13:46 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "13:46 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "13:46 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "13:46 madminer.ml          INFO    Starting training\n",
      "13:46 madminer.ml          INFO      Batch size:             128\n",
      "13:46 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "13:46 madminer.ml          INFO      Epochs:                 50\n",
      "13:46 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "13:46 madminer.ml          INFO      Validation split:       0.25\n",
      "13:46 madminer.ml          INFO      Early stopping:         True\n",
      "13:46 madminer.ml          INFO      Scale inputs:           True\n",
      "13:46 madminer.ml          INFO      Shuffle labels          False\n",
      "13:46 madminer.ml          INFO      Samples:                all\n",
      "13:46 madminer.ml          INFO    Loading training data\n",
      "13:46 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/x_train_score_2.npy into RAM\n",
      "13:46 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/t_xz_train_score_2.npy into RAM\n",
      "13:46 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "13:46 madminer.ml          INFO    Rescaling inputs\n",
      "13:46 madminer.ml          INFO    Creating model\n",
      "13:46 madminer.ml          INFO    Training model\n",
      "13:46 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "13:47 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00016 (mse_score:  0.000)\n",
      "13:47 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "13:48 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00015 (mse_score:  0.000)\n",
      "13:48 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "13:49 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "13:49 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "13:49 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00014 (mse_score:  0.000)\n",
      "13:49 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:50 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00014 (mse_score:  0.000)\n",
      "13:50 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:51 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "13:51 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:52 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "13:52 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:52 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "13:52 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:53 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "13:53 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:54 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00014 (mse_score:  0.000)\n",
      "13:54 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:55 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00014 (mse_score:  0.000)\n",
      "13:55 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:56 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00014 (mse_score:  0.000)\n",
      "13:56 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:56 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00014 (mse_score:  0.000)\n",
      "13:56 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:57 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00014 (mse_score:  0.000)\n",
      "13:57 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:58 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00014 (mse_score:  0.000)\n",
      "13:58 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:59 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00014 (mse_score:  0.000)\n",
      "13:59 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "13:59 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00014 compared to final loss  0.00014\n",
      "13:59 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "13:59 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "13:59 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                                set lr:   0.00h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13:59 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "13:59 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "13:59 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "13:59 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "13:59 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "13:59 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "13:59 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "13:59 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "13:59 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "13:59 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "13:59 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "13:59 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "13:59 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "13:59 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "13:59 madminer.ml          INFO    Starting training\n",
      "13:59 madminer.ml          INFO      Batch size:             128\n",
      "13:59 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "13:59 madminer.ml          INFO      Epochs:                 50\n",
      "13:59 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "13:59 madminer.ml          INFO      Validation split:       0.25\n",
      "13:59 madminer.ml          INFO      Early stopping:         True\n",
      "13:59 madminer.ml          INFO      Scale inputs:           True\n",
      "13:59 madminer.ml          INFO      Shuffle labels          False\n",
      "13:59 madminer.ml          INFO      Samples:                all\n",
      "13:59 madminer.ml          INFO    Loading training data\n",
      "13:59 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/x_train_score_3.npy into RAM\n",
      "13:59 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/t_xz_train_score_3.npy into RAM\n",
      "13:59 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "13:59 madminer.ml          INFO    Rescaling inputs\n",
      "13:59 madminer.ml          INFO    Creating model\n",
      "13:59 madminer.ml          INFO    Training model\n",
      "13:59 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "14:00 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00017 (mse_score:  0.000)\n",
      "14:00 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "14:01 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00016 (mse_score:  0.000)\n",
      "14:01 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "14:02 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "14:02 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:03 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00015 (mse_score:  0.000)\n",
      "14:03 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:03 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00014 (mse_score:  0.000)\n",
      "14:03 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:04 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "14:04 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:05 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "14:05 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:06 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "14:06 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:07 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "14:07 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:07 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00014 (mse_score:  0.000)\n",
      "14:07 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:08 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00014 (mse_score:  0.000)\n",
      "14:08 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:09 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00014 (mse_score:  0.000)\n",
      "14:09 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:10 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00014 (mse_score:  0.000)\n",
      "14:10 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:11 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00014 (mse_score:  0.000)\n",
      "14:11 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:11 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00014 (mse_score:  0.000)\n",
      "14:11 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:12 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00014 (mse_score:  0.000)\n",
      "14:12 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:13 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "14:13 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "14:13 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "14:13 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "14:13 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "14:13 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "14:13 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "14:13 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "14:13 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "14:13 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "14:13 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "14:13 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "14:13 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "14:13 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "14:13 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "14:13 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "14:13 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "14:13 madminer.ml          INFO    Starting training\n",
      "14:13 madminer.ml          INFO      Batch size:             128\n",
      "14:13 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "14:13 madminer.ml          INFO      Epochs:                 50\n",
      "14:13 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "14:13 madminer.ml          INFO      Validation split:       0.25\n",
      "14:13 madminer.ml          INFO      Early stopping:         True\n",
      "14:13 madminer.ml          INFO      Scale inputs:           True\n",
      "14:13 madminer.ml          INFO      Shuffle labels          False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14:13 madminer.ml          INFO      Samples:                all\n",
      "14:13 madminer.ml          INFO    Loading training data\n",
      "14:13 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/x_train_score_4.npy into RAM\n",
      "14:13 madminer.utils.vario INFO      Loading samples/full/samples_wph_mu_smeftsim_full/t_xz_train_score_4.npy into RAM\n",
      "14:13 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "14:13 madminer.ml          INFO    Rescaling inputs\n",
      "14:13 madminer.ml          INFO    Creating model\n",
      "14:13 madminer.ml          INFO    Training model\n",
      "14:13 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "14:13 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00017 (mse_score:  0.000)\n",
      "14:13 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "14:14 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00016 (mse_score:  0.000)\n",
      "14:14 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:15 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "14:15 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:16 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00015 (mse_score:  0.000)\n",
      "14:16 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:17 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00015 (mse_score:  0.000)\n",
      "14:17 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:17 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "14:17 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:18 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "14:18 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:19 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "14:19 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:20 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "14:20 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:21 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00014 (mse_score:  0.000)\n",
      "14:21 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:21 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00014 (mse_score:  0.000)\n",
      "14:21 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:22 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00014 (mse_score:  0.000)\n",
      "14:22 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:23 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00014 (mse_score:  0.000)\n",
      "14:23 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:24 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00014 (mse_score:  0.000)\n",
      "14:24 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:25 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00014 (mse_score:  0.000)\n",
      "14:25 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:25 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00014 (mse_score:  0.000)\n",
      "14:25 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:26 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00013 compared to final loss  0.00013\n",
      "14:26 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "14:26 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "14:26 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "14:26 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "14:26 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "14:26 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "14:26 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "14:26 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "14:26 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "14:26 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "14:26 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "14:26 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "14:26 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "14:26 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "14:26 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "14:26 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_mu_smeftsim_full/estimator_0\n",
      "14:26 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_mu_smeftsim_full/estimator_1\n",
      "14:26 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_mu_smeftsim_full/estimator_2\n",
      "14:26 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_mu_smeftsim_full/estimator_3\n",
      "14:26 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_mu_smeftsim_full/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wph_mu_smeftsim','full',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14:26 madminer.analysis    INFO    Loading data from data/full/signal/wph_e_smeftsim_lhedata_full.h5\n",
      "14:26 madminer.analysis    INFO    Found 4 parameters\n",
      "14:26 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "14:26 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "14:26 madminer.analysis    INFO    Found 60 observables\n",
      "14:26 madminer.analysis    INFO    Found 1000000 events\n",
      "14:26 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "14:26 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "14:26 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "14:26 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "14:26 madminer.sampling    INFO    Starting sampling serially\n",
      "14:26 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14:26 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "14:26 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "14:26 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "14:26 madminer.sampling    INFO    Starting sampling serially\n",
      "14:26 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "14:26 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "14:26 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "14:26 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "14:26 madminer.sampling    INFO    Starting sampling serially\n",
      "14:26 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "14:26 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "14:27 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "14:27 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "14:27 madminer.sampling    INFO    Starting sampling serially\n",
      "14:27 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "14:27 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "14:27 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "14:27 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "14:27 madminer.sampling    INFO    Starting sampling serially\n",
      "14:27 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "14:27 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "14:27 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "14:27 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "14:27 madminer.sampling    INFO    Starting sampling serially\n",
      "14:27 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14:27 madminer.sampling    INFO    Effective number of samples: mean 199716.36363, with individual thetas ranging from 199716.36363 to 199716.36363\n",
      "14:27 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "14:27 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "14:27 madminer.ml          INFO    Starting training\n",
      "14:27 madminer.ml          INFO      Batch size:             128\n",
      "14:27 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "14:27 madminer.ml          INFO      Epochs:                 50\n",
      "14:27 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "14:27 madminer.ml          INFO      Validation split:       0.25\n",
      "14:27 madminer.ml          INFO      Early stopping:         True\n",
      "14:27 madminer.ml          INFO      Scale inputs:           True\n",
      "14:27 madminer.ml          INFO      Shuffle labels          False\n",
      "14:27 madminer.ml          INFO      Samples:                all\n",
      "14:27 madminer.ml          INFO    Loading training data\n",
      "14:27 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/x_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14:27 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/t_xz_train_score_0.npy into RAM\n",
      "14:27 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "14:27 madminer.ml          INFO    Rescaling inputs\n",
      "14:27 madminer.ml          INFO    Creating model\n",
      "14:27 madminer.ml          INFO    Training model\n",
      "14:27 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "14:28 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00016 (mse_score:  0.000)\n",
      "14:28 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "14:29 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00016 (mse_score:  0.000)\n",
      "14:29 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "14:30 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "14:30 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "14:30 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00014 (mse_score:  0.000)\n",
      "14:30 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:31 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00014 (mse_score:  0.000)\n",
      "14:31 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:32 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "14:32 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:33 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "14:33 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:33 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "14:33 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:34 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00013 (mse_score:  0.000)\n",
      "14:34 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:35 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00013 (mse_score:  0.000)\n",
      "14:35 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:36 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00013 (mse_score:  0.000)\n",
      "14:36 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:37 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00013 (mse_score:  0.000)\n",
      "14:37 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:37 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00013 (mse_score:  0.000)\n",
      "14:37 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:38 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00013 (mse_score:  0.000)\n",
      "14:38 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:39 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00013 (mse_score:  0.000)\n",
      "14:39 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:40 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00013 (mse_score:  0.000)\n",
      "14:40 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:40 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "14:40 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "14:40 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "14:40 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "14:40 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "14:40 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "14:40 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "14:40 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "14:40 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "14:40 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "14:40 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "14:40 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "14:40 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "14:40 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "14:40 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "14:40 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "14:40 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "14:40 madminer.ml          INFO    Starting training\n",
      "14:40 madminer.ml          INFO      Batch size:             128\n",
      "14:40 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "14:40 madminer.ml          INFO      Epochs:                 50\n",
      "14:40 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "14:40 madminer.ml          INFO      Validation split:       0.25\n",
      "14:40 madminer.ml          INFO      Early stopping:         True\n",
      "14:40 madminer.ml          INFO      Scale inputs:           True\n",
      "14:40 madminer.ml          INFO      Shuffle labels          False\n",
      "14:40 madminer.ml          INFO      Samples:                all\n",
      "14:40 madminer.ml          INFO    Loading training data\n",
      "14:40 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/x_train_score_1.npy into RAM\n",
      "14:40 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/t_xz_train_score_1.npy into RAM\n",
      "14:40 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "14:40 madminer.ml          INFO    Rescaling inputs\n",
      "14:40 madminer.ml          INFO    Creating model\n",
      "14:40 madminer.ml          INFO    Training model\n",
      "14:40 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "14:41 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00017 (mse_score:  0.000)\n",
      "14:41 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "14:42 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00015 (mse_score:  0.000)\n",
      "14:42 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:43 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "14:43 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:44 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00014 (mse_score:  0.000)\n",
      "14:44 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:44 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00014 (mse_score:  0.000)\n",
      "14:44 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:45 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "14:45 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:46 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "14:46 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:47 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "14:47 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:47 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "14:47 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14:48 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00014 (mse_score:  0.000)\n",
      "14:48 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:49 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00014 (mse_score:  0.000)\n",
      "14:49 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:50 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00014 (mse_score:  0.000)\n",
      "14:50 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:51 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00014 (mse_score:  0.000)\n",
      "14:51 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:51 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00014 (mse_score:  0.000)\n",
      "14:51 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:52 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00014 (mse_score:  0.000)\n",
      "14:52 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:53 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00014 (mse_score:  0.000)\n",
      "14:53 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "14:54 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00013 compared to final loss  0.00013\n",
      "14:54 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "14:54 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "14:54 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "14:54 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "14:54 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "14:54 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "14:54 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "14:54 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "14:54 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "14:54 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "14:54 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "14:54 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "14:54 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "14:54 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "14:54 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "14:54 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "14:54 madminer.ml          INFO    Starting training\n",
      "14:54 madminer.ml          INFO      Batch size:             128\n",
      "14:54 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "14:54 madminer.ml          INFO      Epochs:                 50\n",
      "14:54 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "14:54 madminer.ml          INFO      Validation split:       0.25\n",
      "14:54 madminer.ml          INFO      Early stopping:         True\n",
      "14:54 madminer.ml          INFO      Scale inputs:           True\n",
      "14:54 madminer.ml          INFO      Shuffle labels          False\n",
      "14:54 madminer.ml          INFO      Samples:                all\n",
      "14:54 madminer.ml          INFO    Loading training data\n",
      "14:54 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/x_train_score_2.npy into RAM\n",
      "14:54 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/t_xz_train_score_2.npy into RAM\n",
      "14:54 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "14:54 madminer.ml          INFO    Rescaling inputs\n",
      "14:54 madminer.ml          INFO    Creating model\n",
      "14:54 madminer.ml          INFO    Training model\n",
      "14:54 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "14:54 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00016 (mse_score:  0.000)\n",
      "14:54 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "14:55 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00015 (mse_score:  0.000)\n",
      "14:55 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "14:56 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00014 (mse_score:  0.000)\n",
      "14:56 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:57 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00014 (mse_score:  0.000)\n",
      "14:57 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:57 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00014 (mse_score:  0.000)\n",
      "14:57 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "14:58 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "14:58 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "14:59 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "14:59 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:00 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "15:00 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:01 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "15:01 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:01 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00013 (mse_score:  0.000)\n",
      "15:01 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:02 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00013 (mse_score:  0.000)\n",
      "15:02 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:03 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00013 (mse_score:  0.000)\n",
      "15:03 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:04 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00013 (mse_score:  0.000)\n",
      "15:04 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:05 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00013 (mse_score:  0.000)\n",
      "15:05 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:05 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00013 (mse_score:  0.000)\n",
      "15:05 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:06 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00013 (mse_score:  0.000)\n",
      "15:06 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:07 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "15:07 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "15:07 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "15:07 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                                set lr:   0.00h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15:07 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "15:07 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "15:07 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "15:07 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "15:07 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "15:07 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "15:07 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "15:07 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "15:07 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "15:07 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "15:07 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "15:07 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "15:07 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "15:07 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "15:07 madminer.ml          INFO    Starting training\n",
      "15:07 madminer.ml          INFO      Batch size:             128\n",
      "15:07 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "15:07 madminer.ml          INFO      Epochs:                 50\n",
      "15:07 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "15:07 madminer.ml          INFO      Validation split:       0.25\n",
      "15:07 madminer.ml          INFO      Early stopping:         True\n",
      "15:07 madminer.ml          INFO      Scale inputs:           True\n",
      "15:07 madminer.ml          INFO      Shuffle labels          False\n",
      "15:07 madminer.ml          INFO      Samples:                all\n",
      "15:07 madminer.ml          INFO    Loading training data\n",
      "15:07 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/x_train_score_3.npy into RAM\n",
      "15:07 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/t_xz_train_score_3.npy into RAM\n",
      "15:07 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "15:07 madminer.ml          INFO    Rescaling inputs\n",
      "15:07 madminer.ml          INFO    Creating model\n",
      "15:07 madminer.ml          INFO    Training model\n",
      "15:07 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "15:07 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00016 (mse_score:  0.000)\n",
      "15:07 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:08 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00015 (mse_score:  0.000)\n",
      "15:08 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "15:09 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "15:09 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:10 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00014 (mse_score:  0.000)\n",
      "15:10 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:11 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00014 (mse_score:  0.000)\n",
      "15:11 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:11 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "15:11 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:12 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "15:12 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:13 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "15:13 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:14 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "15:14 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:15 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00014 (mse_score:  0.000)\n",
      "15:15 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:15 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00014 (mse_score:  0.000)\n",
      "15:15 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:16 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00014 (mse_score:  0.000)\n",
      "15:16 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:17 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00014 (mse_score:  0.000)\n",
      "15:17 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:18 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00013 (mse_score:  0.000)\n",
      "15:18 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:19 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00013 (mse_score:  0.000)\n",
      "15:19 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:19 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00013 (mse_score:  0.000)\n",
      "15:19 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:20 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00013 compared to final loss  0.00013\n",
      "15:20 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "15:20 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "15:20 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "15:20 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "15:20 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "15:20 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "15:20 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "15:20 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "15:20 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "15:20 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "15:20 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "15:20 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "15:20 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "15:20 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "15:20 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "15:20 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "15:20 madminer.ml          INFO    Starting training\n",
      "15:20 madminer.ml          INFO      Batch size:             128\n",
      "15:20 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "15:20 madminer.ml          INFO      Epochs:                 50\n",
      "15:20 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "15:20 madminer.ml          INFO      Validation split:       0.25\n",
      "15:20 madminer.ml          INFO      Early stopping:         True\n",
      "15:20 madminer.ml          INFO      Scale inputs:           True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15:20 madminer.ml          INFO      Shuffle labels          False\n",
      "15:20 madminer.ml          INFO      Samples:                all\n",
      "15:20 madminer.ml          INFO    Loading training data\n",
      "15:20 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/x_train_score_4.npy into RAM\n",
      "15:20 madminer.utils.vario INFO      Loading samples/full/samples_wph_e_smeftsim_full/t_xz_train_score_4.npy into RAM\n",
      "15:20 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "15:20 madminer.ml          INFO    Rescaling inputs\n",
      "15:20 madminer.ml          INFO    Creating model\n",
      "15:20 madminer.ml          INFO    Training model\n",
      "15:20 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "15:21 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00017 (mse_score:  0.000)\n",
      "15:21 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:21 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00016 (mse_score:  0.000)\n",
      "15:21 madminer.utils.ml.tr INFO               val. loss   0.00015 (mse_score:  0.000)\n",
      "15:22 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00015 (mse_score:  0.000)\n",
      "15:22 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:23 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00015 (mse_score:  0.000)\n",
      "15:23 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:24 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00015 (mse_score:  0.000)\n",
      "15:24 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:25 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00014 (mse_score:  0.000)\n",
      "15:25 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:25 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00014 (mse_score:  0.000)\n",
      "15:25 madminer.utils.ml.tr INFO               val. loss   0.00014 (mse_score:  0.000)\n",
      "15:26 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00014 (mse_score:  0.000)\n",
      "15:26 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:27 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00014 (mse_score:  0.000)\n",
      "15:27 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:28 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00014 (mse_score:  0.000)\n",
      "15:28 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:29 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00014 (mse_score:  0.000)\n",
      "15:29 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:29 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00014 (mse_score:  0.000)\n",
      "15:29 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:30 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00014 (mse_score:  0.000)\n",
      "15:30 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:31 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00014 (mse_score:  0.000)\n",
      "15:31 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:32 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00014 (mse_score:  0.000)\n",
      "15:32 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:32 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00014 (mse_score:  0.000)\n",
      "15:32 madminer.utils.ml.tr INFO               val. loss   0.00013 (mse_score:  0.000)\n",
      "15:33 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00013 compared to final loss  0.00013\n",
      "15:33 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "15:33 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "15:33 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "15:33 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "15:33 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "15:33 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "15:33 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "15:33 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "15:33 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "15:33 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "15:33 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "15:33 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "15:33 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "15:33 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "15:33 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "15:33 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_e_smeftsim_full/estimator_0\n",
      "15:33 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_e_smeftsim_full/estimator_1\n",
      "15:33 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_e_smeftsim_full/estimator_2\n",
      "15:33 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_e_smeftsim_full/estimator_3\n",
      "15:33 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wph_e_smeftsim_full/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wph_e_smeftsim','full',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15:33 madminer.analysis    INFO    Loading data from data/full/signal/wmh_mu_smeftsim_lhedata_full.h5\n",
      "15:33 madminer.analysis    INFO    Found 4 parameters\n",
      "15:33 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "15:33 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "15:33 madminer.analysis    INFO    Found 60 observables\n",
      "15:33 madminer.analysis    INFO    Found 1000000 events\n",
      "15:33 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "15:33 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "15:33 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "15:33 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "15:33 madminer.sampling    INFO    Starting sampling serially\n",
      "15:33 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15:33 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "15:33 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "15:33 madminer.sampling    INFO    Starting sampling serially\n",
      "15:33 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "15:33 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "15:33 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "15:33 madminer.sampling    INFO    Starting sampling serially\n",
      "15:33 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "15:34 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "15:34 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "15:34 madminer.sampling    INFO    Starting sampling serially\n",
      "15:34 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "15:34 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "15:34 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "15:34 madminer.sampling    INFO    Starting sampling serially\n",
      "15:34 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "15:34 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "15:34 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "15:34 madminer.sampling    INFO    Starting sampling serially\n",
      "15:34 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15:34 madminer.sampling    INFO    Effective number of samples: mean 199827.145512, with individual thetas ranging from 199827.145512 to 199827.145512\n",
      "15:34 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "15:34 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "15:34 madminer.ml          INFO    Starting training\n",
      "15:34 madminer.ml          INFO      Batch size:             128\n",
      "15:34 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "15:34 madminer.ml          INFO      Epochs:                 50\n",
      "15:34 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "15:34 madminer.ml          INFO      Validation split:       0.25\n",
      "15:34 madminer.ml          INFO      Early stopping:         True\n",
      "15:34 madminer.ml          INFO      Scale inputs:           True\n",
      "15:34 madminer.ml          INFO      Shuffle labels          False\n",
      "15:34 madminer.ml          INFO      Samples:                all\n",
      "15:34 madminer.ml          INFO    Loading training data\n",
      "15:34 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/x_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15:34 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/t_xz_train_score_0.npy into RAM\n",
      "15:34 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "15:34 madminer.ml          INFO    Rescaling inputs\n",
      "15:34 madminer.ml          INFO    Creating model\n",
      "15:34 madminer.ml          INFO    Training model\n",
      "15:34 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "15:35 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00018 (mse_score:  0.000)\n",
      "15:35 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "15:36 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "15:36 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "15:37 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00017 (mse_score:  0.000)\n",
      "15:37 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:37 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00017 (mse_score:  0.000)\n",
      "15:37 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:38 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "15:38 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:39 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "15:39 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:40 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "15:40 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:40 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00016 (mse_score:  0.000)\n",
      "15:40 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:41 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00016 (mse_score:  0.000)\n",
      "15:41 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:42 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00016 (mse_score:  0.000)\n",
      "15:42 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:43 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00016 (mse_score:  0.000)\n",
      "15:43 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:44 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00016 (mse_score:  0.000)\n",
      "15:44 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:44 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00016 (mse_score:  0.000)\n",
      "15:44 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:45 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00016 (mse_score:  0.000)\n",
      "15:45 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:46 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00016 (mse_score:  0.000)\n",
      "15:46 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:47 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00016 (mse_score:  0.000)\n",
      "15:47 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:47 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "15:47 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "15:47 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "15:47 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "15:47 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "15:47 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "15:47 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "15:47 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "15:47 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "15:47 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "15:47 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "15:47 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "15:47 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "15:47 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "15:47 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "15:47 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "15:47 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "15:47 madminer.ml          INFO    Starting training\n",
      "15:47 madminer.ml          INFO      Batch size:             128\n",
      "15:47 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "15:47 madminer.ml          INFO      Epochs:                 50\n",
      "15:47 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "15:47 madminer.ml          INFO      Validation split:       0.25\n",
      "15:47 madminer.ml          INFO      Early stopping:         True\n",
      "15:47 madminer.ml          INFO      Scale inputs:           True\n",
      "15:47 madminer.ml          INFO      Shuffle labels          False\n",
      "15:47 madminer.ml          INFO      Samples:                all\n",
      "15:47 madminer.ml          INFO    Loading training data\n",
      "15:47 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/x_train_score_1.npy into RAM\n",
      "15:47 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/t_xz_train_score_1.npy into RAM\n",
      "15:47 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "15:47 madminer.ml          INFO    Rescaling inputs\n",
      "15:47 madminer.ml          INFO    Creating model\n",
      "15:47 madminer.ml          INFO    Training model\n",
      "15:47 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "15:48 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "15:48 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "15:49 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "15:49 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:50 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "15:50 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "15:50 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "15:50 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:51 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "15:51 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:52 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "15:52 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:53 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "15:53 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:54 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "15:54 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:54 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "15:54 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15:55 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "15:55 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:56 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "15:56 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:57 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "15:57 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:58 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "15:58 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:58 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "15:58 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "15:59 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "15:59 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:00 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "16:00 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:00 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00016 compared to final loss  0.00016\n",
      "16:00 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "16:00 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "16:00 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "16:00 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "16:00 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "16:00 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "16:00 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "16:00 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "16:00 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "16:00 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "16:00 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "16:00 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "16:00 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "16:00 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "16:00 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "16:00 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "16:00 madminer.ml          INFO    Starting training\n",
      "16:00 madminer.ml          INFO      Batch size:             128\n",
      "16:00 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "16:00 madminer.ml          INFO      Epochs:                 50\n",
      "16:00 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "16:00 madminer.ml          INFO      Validation split:       0.25\n",
      "16:00 madminer.ml          INFO      Early stopping:         True\n",
      "16:00 madminer.ml          INFO      Scale inputs:           True\n",
      "16:00 madminer.ml          INFO      Shuffle labels          False\n",
      "16:00 madminer.ml          INFO      Samples:                all\n",
      "16:00 madminer.ml          INFO    Loading training data\n",
      "16:00 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/x_train_score_2.npy into RAM\n",
      "16:00 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/t_xz_train_score_2.npy into RAM\n",
      "16:00 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "16:00 madminer.ml          INFO    Rescaling inputs\n",
      "16:00 madminer.ml          INFO    Creating model\n",
      "16:00 madminer.ml          INFO    Training model\n",
      "16:00 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "16:01 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00018 (mse_score:  0.000)\n",
      "16:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:02 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "16:02 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:03 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00017 (mse_score:  0.000)\n",
      "16:03 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:04 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00017 (mse_score:  0.000)\n",
      "16:04 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:04 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "16:04 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:05 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "16:05 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:06 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "16:06 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:07 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "16:07 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:08 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "16:08 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:08 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00016 (mse_score:  0.000)\n",
      "16:08 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:09 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00016 (mse_score:  0.000)\n",
      "16:09 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:10 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00016 (mse_score:  0.000)\n",
      "16:10 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:11 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00016 (mse_score:  0.000)\n",
      "16:11 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:12 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00016 (mse_score:  0.000)\n",
      "16:12 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:12 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00016 (mse_score:  0.000)\n",
      "16:12 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:13 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00016 (mse_score:  0.000)\n",
      "16:13 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:14 madminer.utils.ml.tr INFO    Early stopping after epoch 46, with loss  0.00016 compared to final loss  0.00016\n",
      "16:14 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "16:14 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "16:14 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16:14 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "16:14 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "16:14 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "16:14 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "16:14 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "16:14 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "16:14 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "16:14 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "16:14 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "16:14 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "16:14 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "16:14 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "16:14 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "16:14 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "16:14 madminer.ml          INFO    Starting training\n",
      "16:14 madminer.ml          INFO      Batch size:             128\n",
      "16:14 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "16:14 madminer.ml          INFO      Epochs:                 50\n",
      "16:14 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "16:14 madminer.ml          INFO      Validation split:       0.25\n",
      "16:14 madminer.ml          INFO      Early stopping:         True\n",
      "16:14 madminer.ml          INFO      Scale inputs:           True\n",
      "16:14 madminer.ml          INFO      Shuffle labels          False\n",
      "16:14 madminer.ml          INFO      Samples:                all\n",
      "16:14 madminer.ml          INFO    Loading training data\n",
      "16:14 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/x_train_score_3.npy into RAM\n",
      "16:14 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/t_xz_train_score_3.npy into RAM\n",
      "16:14 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "16:14 madminer.ml          INFO    Rescaling inputs\n",
      "16:14 madminer.ml          INFO    Creating model\n",
      "16:14 madminer.ml          INFO    Training model\n",
      "16:14 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "16:14 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "16:14 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:15 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "16:15 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:16 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00017 (mse_score:  0.000)\n",
      "16:16 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:17 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00017 (mse_score:  0.000)\n",
      "16:17 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:18 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "16:18 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:18 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "16:18 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:19 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "16:19 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:20 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "16:20 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:21 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "16:21 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:21 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "16:21 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:22 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "16:22 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:23 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "16:23 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:24 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "16:24 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:25 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00016 (mse_score:  0.000)\n",
      "16:25 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:25 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00016 (mse_score:  0.000)\n",
      "16:25 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:26 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00016 (mse_score:  0.000)\n",
      "16:26 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:27 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "16:27 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "16:27 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "16:27 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "16:27 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "16:27 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "16:27 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "16:27 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "16:27 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "16:27 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "16:27 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "16:27 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "16:27 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "16:27 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "16:27 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "16:27 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "16:27 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "16:27 madminer.ml          INFO    Starting training\n",
      "16:27 madminer.ml          INFO      Batch size:             128\n",
      "16:27 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "16:27 madminer.ml          INFO      Epochs:                 50\n",
      "16:27 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "16:27 madminer.ml          INFO      Validation split:       0.25\n",
      "16:27 madminer.ml          INFO      Early stopping:         True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16:27 madminer.ml          INFO      Scale inputs:           True\n",
      "16:27 madminer.ml          INFO      Shuffle labels          False\n",
      "16:27 madminer.ml          INFO      Samples:                all\n",
      "16:27 madminer.ml          INFO    Loading training data\n",
      "16:27 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/x_train_score_4.npy into RAM\n",
      "16:27 madminer.utils.vario INFO      Loading samples/full/samples_wmh_mu_smeftsim_full/t_xz_train_score_4.npy into RAM\n",
      "16:27 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "16:27 madminer.ml          INFO    Rescaling inputs\n",
      "16:27 madminer.ml          INFO    Creating model\n",
      "16:27 madminer.ml          INFO    Training model\n",
      "16:27 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "16:28 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00018 (mse_score:  0.000)\n",
      "16:28 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:28 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "16:28 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:29 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00017 (mse_score:  0.000)\n",
      "16:29 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:30 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00017 (mse_score:  0.000)\n",
      "16:30 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:31 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "16:31 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:31 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "16:31 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:32 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "16:32 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:33 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00016 (mse_score:  0.000)\n",
      "16:33 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:34 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00016 (mse_score:  0.000)\n",
      "16:34 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:35 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00016 (mse_score:  0.000)\n",
      "16:35 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:35 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00016 (mse_score:  0.000)\n",
      "16:35 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:36 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00016 (mse_score:  0.000)\n",
      "16:36 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:37 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00016 (mse_score:  0.000)\n",
      "16:37 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:38 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00016 (mse_score:  0.000)\n",
      "16:38 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:38 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00016 (mse_score:  0.000)\n",
      "16:38 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:39 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00016 (mse_score:  0.000)\n",
      "16:39 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:40 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "16:40 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "16:40 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "16:40 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "16:40 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "16:40 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "16:40 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "16:40 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "16:40 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "16:40 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "16:40 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "16:40 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "16:40 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "16:40 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "16:40 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "16:40 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "16:40 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_mu_smeftsim_full/estimator_0\n",
      "16:40 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_mu_smeftsim_full/estimator_1\n",
      "16:40 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_mu_smeftsim_full/estimator_2\n",
      "16:40 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_mu_smeftsim_full/estimator_3\n",
      "16:40 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_mu_smeftsim_full/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wmh_mu_smeftsim','full',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16:40 madminer.analysis    INFO    Loading data from data/full/signal/wmh_e_smeftsim_lhedata_full.h5\n",
      "16:40 madminer.analysis    INFO    Found 4 parameters\n",
      "16:40 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "16:40 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "16:40 madminer.analysis    INFO    Found 60 observables\n",
      "16:40 madminer.analysis    INFO    Found 1000000 events\n",
      "16:40 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "16:40 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "16:40 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "16:40 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "16:40 madminer.sampling    INFO    Starting sampling serially\n",
      "16:40 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16:40 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "16:40 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "16:40 madminer.sampling    INFO    Starting sampling serially\n",
      "16:40 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "16:40 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "16:40 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "16:40 madminer.sampling    INFO    Starting sampling serially\n",
      "16:40 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "16:40 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "16:40 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "16:40 madminer.sampling    INFO    Starting sampling serially\n",
      "16:40 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "16:41 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "16:41 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "16:41 madminer.sampling    INFO    Starting sampling serially\n",
      "16:41 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "16:41 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "16:41 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "16:41 madminer.sampling    INFO    Starting sampling serially\n",
      "16:41 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16:41 madminer.sampling    INFO    Effective number of samples: mean 199827.145512, with individual thetas ranging from 199827.145512 to 199827.145512\n",
      "16:41 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "16:41 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "16:41 madminer.ml          INFO    Starting training\n",
      "16:41 madminer.ml          INFO      Batch size:             128\n",
      "16:41 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "16:41 madminer.ml          INFO      Epochs:                 50\n",
      "16:41 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "16:41 madminer.ml          INFO      Validation split:       0.25\n",
      "16:41 madminer.ml          INFO      Early stopping:         True\n",
      "16:41 madminer.ml          INFO      Scale inputs:           True\n",
      "16:41 madminer.ml          INFO      Shuffle labels          False\n",
      "16:41 madminer.ml          INFO      Samples:                all\n",
      "16:41 madminer.ml          INFO    Loading training data\n",
      "16:41 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/x_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16:41 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/t_xz_train_score_0.npy into RAM\n",
      "16:41 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "16:41 madminer.ml          INFO    Rescaling inputs\n",
      "16:41 madminer.ml          INFO    Creating model\n",
      "16:41 madminer.ml          INFO    Training model\n",
      "16:41 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "16:42 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "16:42 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "16:42 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "16:42 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:43 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "16:43 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:44 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "16:44 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:45 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "16:45 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:46 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "16:46 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:46 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "16:46 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:47 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "16:47 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:48 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "16:48 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:49 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "16:49 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:49 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "16:49 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:50 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "16:50 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:51 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "16:51 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:52 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "16:52 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:53 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "16:53 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:53 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "16:53 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "16:54 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00016 compared to final loss  0.00016\n",
      "16:54 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "16:54 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "16:54 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "16:54 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "16:54 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "16:54 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "16:54 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "16:54 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "16:54 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "16:54 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "16:54 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "16:54 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "16:54 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "16:54 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "16:54 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "16:54 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "16:54 madminer.ml          INFO    Starting training\n",
      "16:54 madminer.ml          INFO      Batch size:             128\n",
      "16:54 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "16:54 madminer.ml          INFO      Epochs:                 50\n",
      "16:54 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "16:54 madminer.ml          INFO      Validation split:       0.25\n",
      "16:54 madminer.ml          INFO      Early stopping:         True\n",
      "16:54 madminer.ml          INFO      Scale inputs:           True\n",
      "16:54 madminer.ml          INFO      Shuffle labels          False\n",
      "16:54 madminer.ml          INFO      Samples:                all\n",
      "16:54 madminer.ml          INFO    Loading training data\n",
      "16:54 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/x_train_score_1.npy into RAM\n",
      "16:54 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/t_xz_train_score_1.npy into RAM\n",
      "16:54 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "16:54 madminer.ml          INFO    Rescaling inputs\n",
      "16:54 madminer.ml          INFO    Creating model\n",
      "16:54 madminer.ml          INFO    Training model\n",
      "16:54 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "16:55 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00018 (mse_score:  0.000)\n",
      "16:55 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:55 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "16:55 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "16:56 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00017 (mse_score:  0.000)\n",
      "16:56 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:57 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00017 (mse_score:  0.000)\n",
      "16:57 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:58 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "16:58 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:59 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "16:59 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "16:59 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00016 (mse_score:  0.000)\n",
      "16:59 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:00 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00016 (mse_score:  0.000)\n",
      "17:00 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:01 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00016 (mse_score:  0.000)\n",
      "17:01 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "17:02 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00016 (mse_score:  0.000)\n",
      "17:02 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:03 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00016 (mse_score:  0.000)\n",
      "17:03 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:03 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00016 (mse_score:  0.000)\n",
      "17:03 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:04 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00016 (mse_score:  0.000)\n",
      "17:04 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:05 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00016 (mse_score:  0.000)\n",
      "17:05 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:06 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00016 (mse_score:  0.000)\n",
      "17:06 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:06 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00016 (mse_score:  0.000)\n",
      "17:06 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:07 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "17:07 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "17:07 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "17:07 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "17:07 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "17:07 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "17:07 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "17:07 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "17:07 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "17:07 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "17:07 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "17:07 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "17:07 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "17:07 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "17:07 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "17:07 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "17:07 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "17:07 madminer.ml          INFO    Starting training\n",
      "17:07 madminer.ml          INFO      Batch size:             128\n",
      "17:07 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "17:07 madminer.ml          INFO      Epochs:                 50\n",
      "17:07 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "17:07 madminer.ml          INFO      Validation split:       0.25\n",
      "17:07 madminer.ml          INFO      Early stopping:         True\n",
      "17:07 madminer.ml          INFO      Scale inputs:           True\n",
      "17:07 madminer.ml          INFO      Shuffle labels          False\n",
      "17:07 madminer.ml          INFO      Samples:                all\n",
      "17:07 madminer.ml          INFO    Loading training data\n",
      "17:07 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/x_train_score_2.npy into RAM\n",
      "17:07 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/t_xz_train_score_2.npy into RAM\n",
      "17:07 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "17:07 madminer.ml          INFO    Rescaling inputs\n",
      "17:07 madminer.ml          INFO    Creating model\n",
      "17:07 madminer.ml          INFO    Training model\n",
      "17:07 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "17:08 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "17:08 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "17:08 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "17:08 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:09 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00017 (mse_score:  0.000)\n",
      "17:09 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:10 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00017 (mse_score:  0.000)\n",
      "17:10 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:11 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "17:11 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:12 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "17:12 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:12 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "17:12 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:13 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "17:13 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:14 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "17:14 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:15 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "17:15 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:16 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "17:16 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:16 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "17:16 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:17 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "17:17 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:18 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "17:18 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:19 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "17:19 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:19 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00016 (mse_score:  0.000)\n",
      "17:19 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:20 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00016 compared to final loss  0.00016\n",
      "17:20 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "17:20 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "17:20 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                                set lr:   0.00h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "17:20 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "17:20 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "17:20 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "17:20 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "17:20 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "17:20 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "17:20 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "17:20 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "17:20 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "17:20 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "17:20 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "17:20 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "17:20 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "17:20 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "17:20 madminer.ml          INFO    Starting training\n",
      "17:20 madminer.ml          INFO      Batch size:             128\n",
      "17:20 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "17:20 madminer.ml          INFO      Epochs:                 50\n",
      "17:20 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "17:20 madminer.ml          INFO      Validation split:       0.25\n",
      "17:20 madminer.ml          INFO      Early stopping:         True\n",
      "17:20 madminer.ml          INFO      Scale inputs:           True\n",
      "17:20 madminer.ml          INFO      Shuffle labels          False\n",
      "17:20 madminer.ml          INFO      Samples:                all\n",
      "17:20 madminer.ml          INFO    Loading training data\n",
      "17:20 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/x_train_score_3.npy into RAM\n",
      "17:20 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/t_xz_train_score_3.npy into RAM\n",
      "17:20 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "17:20 madminer.ml          INFO    Rescaling inputs\n",
      "17:20 madminer.ml          INFO    Creating model\n",
      "17:20 madminer.ml          INFO    Training model\n",
      "17:20 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "17:21 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00018 (mse_score:  0.000)\n",
      "17:21 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "17:21 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "17:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "17:22 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00017 (mse_score:  0.000)\n",
      "17:22 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "17:23 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00017 (mse_score:  0.000)\n",
      "17:23 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "17:24 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "17:24 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:25 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "17:25 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:25 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00016 (mse_score:  0.000)\n",
      "17:25 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:26 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00016 (mse_score:  0.000)\n",
      "17:26 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:27 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00016 (mse_score:  0.000)\n",
      "17:27 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:28 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00016 (mse_score:  0.000)\n",
      "17:28 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:28 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00016 (mse_score:  0.000)\n",
      "17:28 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:29 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00016 (mse_score:  0.000)\n",
      "17:29 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:30 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00016 (mse_score:  0.000)\n",
      "17:30 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:31 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00016 (mse_score:  0.000)\n",
      "17:31 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:32 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00016 (mse_score:  0.000)\n",
      "17:32 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:32 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00016 (mse_score:  0.000)\n",
      "17:32 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:33 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00016 compared to final loss  0.00016\n",
      "17:33 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "17:33 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "17:33 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "17:33 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "17:33 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "17:33 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "17:33 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "17:33 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "17:33 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "17:33 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "17:33 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "17:33 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "17:33 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "17:33 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "17:33 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "17:33 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "17:33 madminer.ml          INFO    Starting training\n",
      "17:33 madminer.ml          INFO      Batch size:             128\n",
      "17:33 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "17:33 madminer.ml          INFO      Epochs:                 50\n",
      "17:33 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "17:33 madminer.ml          INFO      Validation split:       0.25\n",
      "17:33 madminer.ml          INFO      Early stopping:         True\n",
      "17:33 madminer.ml          INFO      Scale inputs:           True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "17:33 madminer.ml          INFO      Shuffle labels          False\n",
      "17:33 madminer.ml          INFO      Samples:                all\n",
      "17:33 madminer.ml          INFO    Loading training data\n",
      "17:33 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/x_train_score_4.npy into RAM\n",
      "17:33 madminer.utils.vario INFO      Loading samples/full/samples_wmh_e_smeftsim_full/t_xz_train_score_4.npy into RAM\n",
      "17:33 madminer.ml          INFO    Found 500000 samples with 37 parameters and 60 observables\n",
      "17:33 madminer.ml          INFO    Rescaling inputs\n",
      "17:33 madminer.ml          INFO    Creating model\n",
      "17:33 madminer.ml          INFO    Training model\n",
      "17:33 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "17:34 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00018 (mse_score:  0.000)\n",
      "17:34 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "17:35 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "17:35 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "17:35 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "17:35 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:36 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00017 (mse_score:  0.000)\n",
      "17:36 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:37 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00017 (mse_score:  0.000)\n",
      "17:37 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:38 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "17:38 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:38 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "17:38 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:39 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "17:39 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "17:40 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "17:40 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:41 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00016 (mse_score:  0.000)\n",
      "17:41 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:42 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00016 (mse_score:  0.000)\n",
      "17:42 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:42 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00016 (mse_score:  0.000)\n",
      "17:42 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:43 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00016 (mse_score:  0.000)\n",
      "17:43 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:44 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00016 (mse_score:  0.000)\n",
      "17:44 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:45 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00016 (mse_score:  0.000)\n",
      "17:45 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:45 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00016 (mse_score:  0.000)\n",
      "17:45 madminer.utils.ml.tr INFO               val. loss   0.00016 (mse_score:  0.000)\n",
      "17:46 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00016 compared to final loss  0.00016\n",
      "17:46 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "17:46 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "17:46 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "17:46 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "17:46 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "17:46 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "17:46 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "17:46 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "17:46 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "17:46 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "17:46 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "17:46 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "17:46 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "17:46 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "17:46 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "17:46 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_e_smeftsim_full/estimator_0\n",
      "17:46 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_e_smeftsim_full/estimator_1\n",
      "17:46 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_e_smeftsim_full/estimator_2\n",
      "17:46 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_e_smeftsim_full/estimator_3\n",
      "17:46 madminer.ml          INFO    Saving model to models/full/sally_ensemble_wmh_e_smeftsim_full/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wmh_e_smeftsim','full',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### MET Observables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "08:48 madminer.analysis    INFO    Loading data from data/met/signal/wph_mu_smeftsim_lhedata_met.h5\n",
      "08:48 madminer.analysis    INFO    Found 4 parameters\n",
      "08:48 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "08:48 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "08:48 madminer.analysis    INFO    Found 48 observables\n",
      "08:48 madminer.analysis    INFO    Found 1000000 events\n",
      "08:48 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "08:48 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "08:48 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "08:48 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "08:48 madminer.sampling    INFO    Starting sampling serially\n",
      "08:48 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "08:48 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:48 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "08:49 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "08:49 madminer.sampling    INFO    Starting sampling serially\n",
      "08:49 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "08:49 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "08:49 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "08:49 madminer.sampling    INFO    Starting sampling serially\n",
      "08:49 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "08:49 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "08:49 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "08:49 madminer.sampling    INFO    Starting sampling serially\n",
      "08:49 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "08:49 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "08:49 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "08:49 madminer.sampling    INFO    Starting sampling serially\n",
      "08:49 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "08:49 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "08:49 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "08:49 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "08:49 madminer.sampling    INFO    Starting sampling serially\n",
      "08:49 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "08:50 madminer.sampling    INFO    Effective number of samples: mean 199716.36363, with individual thetas ranging from 199716.36363 to 199716.36363\n",
      "08:50 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "08:50 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "08:50 madminer.ml          INFO    Starting training\n",
      "08:50 madminer.ml          INFO      Batch size:             128\n",
      "08:50 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "08:50 madminer.ml          INFO      Epochs:                 50\n",
      "08:50 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "08:50 madminer.ml          INFO      Validation split:       0.25\n",
      "08:50 madminer.ml          INFO      Early stopping:         True\n",
      "08:50 madminer.ml          INFO      Scale inputs:           True\n",
      "08:50 madminer.ml          INFO      Shuffle labels          False\n",
      "08:50 madminer.ml          INFO      Samples:                all\n",
      "08:50 madminer.ml          INFO    Loading training data\n",
      "08:50 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/x_train_score_0.npy into RAM\n",
      "08:50 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "08:50 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "08:50 madminer.ml          INFO    Rescaling inputs\n",
      "08:50 madminer.ml          INFO    Creating model\n",
      "08:50 madminer.ml          INFO    Training model\n",
      "08:50 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "08:50 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "08:50 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "08:51 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "08:51 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "08:52 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "08:52 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "08:53 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "08:53 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "08:53 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "08:53 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "08:54 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "08:54 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "08:55 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "08:55 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "08:56 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "08:56 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "08:57 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "08:57 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "08:57 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "08:57 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "08:58 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "08:58 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "08:59 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "08:59 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:00 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "09:00 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:00 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "09:00 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:01 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "09:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:02 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "09:02 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:03 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "09:03 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "09:03 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "09:03 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "09:03 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "09:03 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "09:03 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "09:03 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "09:03 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "09:03 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "09:03 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "09:03 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "09:03 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "09:03 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "09:03 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "09:03 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "09:03 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "09:03 madminer.ml          INFO    Starting training\n",
      "09:03 madminer.ml          INFO      Batch size:             128\n",
      "09:03 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "09:03 madminer.ml          INFO      Epochs:                 50\n",
      "09:03 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "09:03 madminer.ml          INFO      Validation split:       0.25\n",
      "09:03 madminer.ml          INFO      Early stopping:         True\n",
      "09:03 madminer.ml          INFO      Scale inputs:           True\n",
      "09:03 madminer.ml          INFO      Shuffle labels          False\n",
      "09:03 madminer.ml          INFO      Samples:                all\n",
      "09:03 madminer.ml          INFO    Loading training data\n",
      "09:03 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/x_train_score_1.npy into RAM\n",
      "09:03 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/t_xz_train_score_1.npy into RAM\n",
      "09:03 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "09:03 madminer.ml          INFO    Rescaling inputs\n",
      "09:03 madminer.ml          INFO    Creating model\n",
      "09:03 madminer.ml          INFO    Training model\n",
      "09:03 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "09:03 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "09:03 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:04 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "09:04 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:05 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "09:05 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:06 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "09:06 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:06 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "09:06 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:07 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "09:07 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:08 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "09:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:09 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "09:09 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:10 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "09:10 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:10 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "09:10 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "09:11 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "09:11 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:12 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "09:12 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:13 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "09:13 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:14 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "09:14 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:14 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "09:14 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:15 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "09:15 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:16 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "09:16 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "09:16 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "09:16 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "09:16 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "09:16 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "09:16 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "09:16 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "09:16 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "09:16 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "09:16 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "09:16 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "09:16 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "09:16 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "09:16 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "09:16 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "09:16 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "09:16 madminer.ml          INFO    Starting training\n",
      "09:16 madminer.ml          INFO      Batch size:             128\n",
      "09:16 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "09:16 madminer.ml          INFO      Epochs:                 50\n",
      "09:16 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "09:16 madminer.ml          INFO      Validation split:       0.25\n",
      "09:16 madminer.ml          INFO      Early stopping:         True\n",
      "09:16 madminer.ml          INFO      Scale inputs:           True\n",
      "09:16 madminer.ml          INFO      Shuffle labels          False\n",
      "09:16 madminer.ml          INFO      Samples:                all\n",
      "09:16 madminer.ml          INFO    Loading training data\n",
      "09:16 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/x_train_score_2.npy into RAM\n",
      "09:16 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/t_xz_train_score_2.npy into RAM\n",
      "09:16 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "09:16 madminer.ml          INFO    Rescaling inputs\n",
      "09:16 madminer.ml          INFO    Creating model\n",
      "09:16 madminer.ml          INFO    Training model\n",
      "09:16 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "09:17 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "09:17 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:17 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "09:17 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:18 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "09:18 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:19 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "09:19 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:20 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "09:20 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:21 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "09:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:21 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "09:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:22 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "09:22 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:23 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "09:23 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:24 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "09:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:25 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "09:25 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:25 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "09:25 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:26 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "09:26 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:27 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "09:27 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:28 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "09:28 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:29 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "09:29 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:29 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00018 compared to final loss  0.00018\n",
      "09:29 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "09:29 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "09:29 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "09:29 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "09:29 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "09:29 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "09:29 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "09:29 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "09:29 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "09:29 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "09:29 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "09:29 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "09:29 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "09:29 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "09:29 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "09:29 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "09:29 madminer.ml          INFO    Starting training\n",
      "09:29 madminer.ml          INFO      Batch size:             128\n",
      "09:29 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "09:29 madminer.ml          INFO      Epochs:                 50\n",
      "09:29 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "09:29 madminer.ml          INFO      Validation split:       0.25\n",
      "09:29 madminer.ml          INFO      Early stopping:         True\n",
      "09:29 madminer.ml          INFO      Scale inputs:           True\n",
      "09:29 madminer.ml          INFO      Shuffle labels          False\n",
      "09:29 madminer.ml          INFO      Samples:                all\n",
      "09:29 madminer.ml          INFO    Loading training data\n",
      "09:29 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/x_train_score_3.npy into RAM\n",
      "09:29 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/t_xz_train_score_3.npy into RAM\n",
      "09:29 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "09:29 madminer.ml          INFO    Rescaling inputs\n",
      "09:29 madminer.ml          INFO    Creating model\n",
      "09:29 madminer.ml          INFO    Training model\n",
      "09:29 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "09:30 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "09:30 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:31 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "09:31 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:31 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "09:31 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:32 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "09:32 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:33 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "09:33 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:34 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "09:34 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:35 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "09:35 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:35 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "09:35 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:36 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "09:36 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:37 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "09:37 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:38 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "09:38 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:38 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "09:38 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:39 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "09:39 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:40 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "09:40 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:41 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "09:41 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:42 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "09:42 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:42 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00017 compared to final loss  0.00017\n",
      "09:42 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "09:42 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "09:42 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "09:42 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "09:42 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "09:42 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "09:42 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "09:42 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "09:42 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "09:42 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "09:42 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "09:42 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "09:42 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "09:42 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "09:42 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "09:42 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "09:42 madminer.ml          INFO    Starting training\n",
      "09:42 madminer.ml          INFO      Batch size:             128\n",
      "09:42 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "09:42 madminer.ml          INFO      Epochs:                 50\n",
      "09:42 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "09:42 madminer.ml          INFO      Validation split:       0.25\n",
      "09:42 madminer.ml          INFO      Early stopping:         True\n",
      "09:42 madminer.ml          INFO      Scale inputs:           True\n",
      "09:42 madminer.ml          INFO      Shuffle labels          False\n",
      "09:42 madminer.ml          INFO      Samples:                all\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "09:42 madminer.ml          INFO    Loading training data\n",
      "09:42 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/x_train_score_4.npy into RAM\n",
      "09:42 madminer.utils.vario INFO      Loading samples/met/samples_wph_mu_smeftsim_met/t_xz_train_score_4.npy into RAM\n",
      "09:42 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "09:42 madminer.ml          INFO    Rescaling inputs\n",
      "09:42 madminer.ml          INFO    Creating model\n",
      "09:42 madminer.ml          INFO    Training model\n",
      "09:42 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "09:43 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "09:43 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:44 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "09:44 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:45 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "09:45 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:45 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "09:45 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:46 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "09:46 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:47 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "09:47 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "09:48 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "09:48 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:48 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "09:48 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:49 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "09:49 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:50 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "09:50 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:51 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "09:51 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:52 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "09:52 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:52 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "09:52 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:53 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "09:53 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:54 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "09:54 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:55 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "09:55 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "09:55 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00017 compared to final loss  0.00017\n",
      "09:55 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "09:55 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "09:55 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "09:55 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "09:55 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "09:55 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "09:55 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "09:55 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "09:55 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "09:55 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "09:55 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "09:55 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "09:55 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "09:55 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "09:55 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "09:55 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_mu_smeftsim_met/estimator_0\n",
      "09:55 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_mu_smeftsim_met/estimator_1\n",
      "09:55 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_mu_smeftsim_met/estimator_2\n",
      "09:55 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_mu_smeftsim_met/estimator_3\n",
      "09:55 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_mu_smeftsim_met/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wph_mu_smeftsim','met',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "09:55 madminer.analysis    INFO    Loading data from data/met/signal/wph_e_smeftsim_lhedata_met.h5\n",
      "09:55 madminer.analysis    INFO    Found 4 parameters\n",
      "09:55 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "09:55 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "09:55 madminer.analysis    INFO    Found 48 observables\n",
      "09:55 madminer.analysis    INFO    Found 1000000 events\n",
      "09:55 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "09:55 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "09:55 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "09:55 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "09:55 madminer.sampling    INFO    Starting sampling serially\n",
      "09:55 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "09:55 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "09:55 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "09:55 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "09:55 madminer.sampling    INFO    Starting sampling serially\n",
      "09:55 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "09:56 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "09:56 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "09:56 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "09:56 madminer.sampling    INFO    Starting sampling serially\n",
      "09:56 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "09:56 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "09:56 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "09:56 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "09:56 madminer.sampling    INFO    Starting sampling serially\n",
      "09:56 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "09:56 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "09:56 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "09:56 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "09:56 madminer.sampling    INFO    Starting sampling serially\n",
      "09:56 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "09:56 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "09:56 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "09:56 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "09:56 madminer.sampling    INFO    Starting sampling serially\n",
      "09:56 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "09:56 madminer.sampling    INFO    Effective number of samples: mean 199716.36363, with individual thetas ranging from 199716.36363 to 199716.36363\n",
      "09:56 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "09:56 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "09:56 madminer.ml          INFO    Starting training\n",
      "09:56 madminer.ml          INFO      Batch size:             128\n",
      "09:56 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "09:56 madminer.ml          INFO      Epochs:                 50\n",
      "09:56 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "09:56 madminer.ml          INFO      Validation split:       0.25\n",
      "09:56 madminer.ml          INFO      Early stopping:         True\n",
      "09:56 madminer.ml          INFO      Scale inputs:           True\n",
      "09:56 madminer.ml          INFO      Shuffle labels          False\n",
      "09:56 madminer.ml          INFO      Samples:                all\n",
      "09:56 madminer.ml          INFO    Loading training data\n",
      "09:56 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/x_train_score_0.npy into RAM\n",
      "09:56 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "09:56 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "09:56 madminer.ml          INFO    Rescaling inputs\n",
      "09:56 madminer.ml          INFO    Creating model\n",
      "09:56 madminer.ml          INFO    Training model\n",
      "09:56 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "09:57 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "09:57 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:58 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "09:58 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:59 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "09:59 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "09:59 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "09:59 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:00 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "10:00 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:01 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "10:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:02 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "10:02 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:03 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "10:03 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:03 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "10:03 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:04 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "10:04 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:05 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "10:05 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:06 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "10:06 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:07 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "10:07 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:07 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "10:07 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:08 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "10:08 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:09 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "10:09 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:09 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "10:09 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "10:09 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "10:09 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "10:09 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "10:09 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "10:09 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "10:09 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "10:09 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "10:09 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "10:09 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "10:09 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "10:09 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "10:09 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "10:09 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "10:09 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "10:09 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "10:09 madminer.ml          INFO    Starting training\n",
      "10:09 madminer.ml          INFO      Batch size:             128\n",
      "10:09 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "10:09 madminer.ml          INFO      Epochs:                 50\n",
      "10:09 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "10:09 madminer.ml          INFO      Validation split:       0.25\n",
      "10:09 madminer.ml          INFO      Early stopping:         True\n",
      "10:09 madminer.ml          INFO      Scale inputs:           True\n",
      "10:09 madminer.ml          INFO      Shuffle labels          False\n",
      "10:09 madminer.ml          INFO      Samples:                all\n",
      "10:09 madminer.ml          INFO    Loading training data\n",
      "10:09 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/x_train_score_1.npy into RAM\n",
      "10:09 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/t_xz_train_score_1.npy into RAM\n",
      "10:09 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "10:09 madminer.ml          INFO    Rescaling inputs\n",
      "10:09 madminer.ml          INFO    Creating model\n",
      "10:09 madminer.ml          INFO    Training model\n",
      "10:09 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "10:10 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "10:10 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:11 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "10:11 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:12 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "10:12 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:13 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "10:13 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:13 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "10:13 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:14 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "10:14 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:15 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "10:15 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:16 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "10:16 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:17 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "10:17 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:17 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "10:17 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10:18 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "10:18 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:19 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "10:19 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:20 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "10:20 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:20 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "10:20 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:21 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "10:21 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:22 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "10:22 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:23 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00017 compared to final loss  0.00017\n",
      "10:23 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "10:23 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "10:23 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "10:23 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "10:23 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "10:23 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "10:23 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "10:23 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "10:23 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "10:23 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "10:23 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "10:23 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "10:23 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "10:23 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "10:23 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "10:23 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "10:23 madminer.ml          INFO    Starting training\n",
      "10:23 madminer.ml          INFO      Batch size:             128\n",
      "10:23 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "10:23 madminer.ml          INFO      Epochs:                 50\n",
      "10:23 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "10:23 madminer.ml          INFO      Validation split:       0.25\n",
      "10:23 madminer.ml          INFO      Early stopping:         True\n",
      "10:23 madminer.ml          INFO      Scale inputs:           True\n",
      "10:23 madminer.ml          INFO      Shuffle labels          False\n",
      "10:23 madminer.ml          INFO      Samples:                all\n",
      "10:23 madminer.ml          INFO    Loading training data\n",
      "10:23 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/x_train_score_2.npy into RAM\n",
      "10:23 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/t_xz_train_score_2.npy into RAM\n",
      "10:23 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "10:23 madminer.ml          INFO    Rescaling inputs\n",
      "10:23 madminer.ml          INFO    Creating model\n",
      "10:23 madminer.ml          INFO    Training model\n",
      "10:23 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "10:23 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "10:23 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:24 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "10:24 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:25 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "10:25 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:26 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "10:26 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:27 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "10:27 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:27 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "10:27 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:28 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "10:28 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:29 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "10:29 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:30 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "10:30 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:31 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "10:31 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:31 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "10:31 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:32 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "10:32 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:33 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "10:33 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:34 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "10:34 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:34 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "10:34 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:35 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "10:35 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:36 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00018 compared to final loss  0.00018\n",
      "10:36 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "10:36 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "10:36 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "10:36 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10:36 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "10:36 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "10:36 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "10:36 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "10:36 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "10:36 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "10:36 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "10:36 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "10:36 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "10:36 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "10:36 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "10:36 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "10:36 madminer.ml          INFO    Starting training\n",
      "10:36 madminer.ml          INFO      Batch size:             128\n",
      "10:36 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "10:36 madminer.ml          INFO      Epochs:                 50\n",
      "10:36 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "10:36 madminer.ml          INFO      Validation split:       0.25\n",
      "10:36 madminer.ml          INFO      Early stopping:         True\n",
      "10:36 madminer.ml          INFO      Scale inputs:           True\n",
      "10:36 madminer.ml          INFO      Shuffle labels          False\n",
      "10:36 madminer.ml          INFO      Samples:                all\n",
      "10:36 madminer.ml          INFO    Loading training data\n",
      "10:36 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/x_train_score_3.npy into RAM\n",
      "10:36 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/t_xz_train_score_3.npy into RAM\n",
      "10:36 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "10:36 madminer.ml          INFO    Rescaling inputs\n",
      "10:36 madminer.ml          INFO    Creating model\n",
      "10:36 madminer.ml          INFO    Training model\n",
      "10:36 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "10:37 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "10:37 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:37 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "10:37 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:38 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "10:38 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:39 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "10:39 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:40 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "10:40 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:41 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "10:41 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:41 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "10:41 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "10:42 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "10:42 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:43 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "10:43 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:44 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "10:44 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:44 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "10:44 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:45 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "10:45 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:46 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "10:46 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:47 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "10:47 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:48 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "10:48 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:48 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "10:48 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "10:49 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "10:49 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "10:49 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "10:49 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "10:49 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "10:49 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "10:49 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "10:49 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "10:49 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "10:49 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "10:49 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "10:49 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "10:49 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "10:49 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "10:49 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "10:49 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "10:49 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "10:49 madminer.ml          INFO    Starting training\n",
      "10:49 madminer.ml          INFO      Batch size:             128\n",
      "10:49 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "10:49 madminer.ml          INFO      Epochs:                 50\n",
      "10:49 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "10:49 madminer.ml          INFO      Validation split:       0.25\n",
      "10:49 madminer.ml          INFO      Early stopping:         True\n",
      "10:49 madminer.ml          INFO      Scale inputs:           True\n",
      "10:49 madminer.ml          INFO      Shuffle labels          False\n",
      "10:49 madminer.ml          INFO      Samples:                all\n",
      "10:49 madminer.ml          INFO    Loading training data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10:49 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/x_train_score_4.npy into RAM\n",
      "10:49 madminer.utils.vario INFO      Loading samples/met/samples_wph_e_smeftsim_met/t_xz_train_score_4.npy into RAM\n",
      "10:49 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "10:49 madminer.ml          INFO    Rescaling inputs\n",
      "10:49 madminer.ml          INFO    Creating model\n",
      "10:49 madminer.ml          INFO    Training model\n",
      "10:49 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "10:50 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "10:50 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "10:50 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "10:50 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:51 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "10:51 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:52 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "10:52 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:53 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "10:53 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:53 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "10:53 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:54 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "10:54 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:55 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "10:55 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:56 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "10:56 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:57 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "10:57 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:57 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "10:57 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:58 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "10:58 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "10:59 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "10:59 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:00 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "11:00 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:00 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "11:00 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:01 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "11:01 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:02 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "11:02 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "11:02 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "11:02 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "11:02 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "11:02 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "11:02 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "11:02 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "11:02 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "11:02 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "11:02 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "11:02 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "11:02 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "11:02 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "11:02 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "11:02 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "11:02 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_e_smeftsim_met/estimator_0\n",
      "11:02 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_e_smeftsim_met/estimator_1\n",
      "11:02 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_e_smeftsim_met/estimator_2\n",
      "11:02 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_e_smeftsim_met/estimator_3\n",
      "11:02 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wph_e_smeftsim_met/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wph_e_smeftsim','met',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11:02 madminer.analysis    INFO    Loading data from data/met/signal/wmh_mu_smeftsim_lhedata_met.h5\n",
      "11:02 madminer.analysis    INFO    Found 4 parameters\n",
      "11:02 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "11:02 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "11:02 madminer.analysis    INFO    Found 48 observables\n",
      "11:02 madminer.analysis    INFO    Found 1000000 events\n",
      "11:02 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "11:02 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "11:02 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "11:02 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "11:02 madminer.sampling    INFO    Starting sampling serially\n",
      "11:02 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11:02 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "11:02 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "11:02 madminer.sampling    INFO    Starting sampling serially\n",
      "11:02 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "11:02 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "11:02 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "11:02 madminer.sampling    INFO    Starting sampling serially\n",
      "11:02 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "11:02 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "11:02 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "11:02 madminer.sampling    INFO    Starting sampling serially\n",
      "11:02 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "11:03 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "11:03 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "11:03 madminer.sampling    INFO    Starting sampling serially\n",
      "11:03 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "11:03 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "11:03 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "11:03 madminer.sampling    INFO    Starting sampling serially\n",
      "11:03 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11:03 madminer.sampling    INFO    Effective number of samples: mean 199827.145512, with individual thetas ranging from 199827.145512 to 199827.145512\n",
      "11:03 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "11:03 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "11:03 madminer.ml          INFO    Starting training\n",
      "11:03 madminer.ml          INFO      Batch size:             128\n",
      "11:03 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "11:03 madminer.ml          INFO      Epochs:                 50\n",
      "11:03 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "11:03 madminer.ml          INFO      Validation split:       0.25\n",
      "11:03 madminer.ml          INFO      Early stopping:         True\n",
      "11:03 madminer.ml          INFO      Scale inputs:           True\n",
      "11:03 madminer.ml          INFO      Shuffle labels          False\n",
      "11:03 madminer.ml          INFO      Samples:                all\n",
      "11:03 madminer.ml          INFO    Loading training data\n",
      "11:03 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/x_train_score_0.npy into RAM\n",
      "11:03 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11:03 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "11:03 madminer.ml          INFO    Rescaling inputs\n",
      "11:03 madminer.ml          INFO    Creating model\n",
      "11:03 madminer.ml          INFO    Training model\n",
      "11:03 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "11:04 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "11:04 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:04 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "11:04 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:05 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "11:05 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:06 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "11:06 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:07 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "11:07 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:08 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "11:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:08 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "11:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:09 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "11:09 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:10 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "11:10 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:11 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "11:11 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:11 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "11:11 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:12 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "11:12 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:13 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "11:13 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:14 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "11:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:15 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "11:15 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:15 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "11:15 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:16 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00019 compared to final loss  0.00019\n",
      "11:16 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "11:16 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "11:16 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "11:16 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "11:16 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "11:16 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "11:16 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "11:16 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "11:16 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "11:16 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "11:16 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "11:16 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "11:16 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "11:16 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "11:16 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "11:16 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "11:16 madminer.ml          INFO    Starting training\n",
      "11:16 madminer.ml          INFO      Batch size:             128\n",
      "11:16 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "11:16 madminer.ml          INFO      Epochs:                 50\n",
      "11:16 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "11:16 madminer.ml          INFO      Validation split:       0.25\n",
      "11:16 madminer.ml          INFO      Early stopping:         True\n",
      "11:16 madminer.ml          INFO      Scale inputs:           True\n",
      "11:16 madminer.ml          INFO      Shuffle labels          False\n",
      "11:16 madminer.ml          INFO      Samples:                all\n",
      "11:16 madminer.ml          INFO    Loading training data\n",
      "11:16 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/x_train_score_1.npy into RAM\n",
      "11:16 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/t_xz_train_score_1.npy into RAM\n",
      "11:16 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "11:16 madminer.ml          INFO    Rescaling inputs\n",
      "11:16 madminer.ml          INFO    Creating model\n",
      "11:16 madminer.ml          INFO    Training model\n",
      "11:16 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "11:17 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "11:17 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:17 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "11:17 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:18 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "11:18 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:19 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "11:19 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:20 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "11:20 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:21 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "11:21 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:21 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "11:21 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:22 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "11:22 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:23 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "11:23 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:24 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11:24 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:24 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "11:24 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:25 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "11:25 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:26 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "11:26 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:27 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "11:27 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:28 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "11:28 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:28 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "11:28 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:29 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00020 compared to final loss  0.00020\n",
      "11:29 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "11:29 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "11:29 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "11:29 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "11:29 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "11:29 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "11:29 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "11:29 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "11:29 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "11:29 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "11:29 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "11:29 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "11:29 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "11:29 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "11:29 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "11:29 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "11:29 madminer.ml          INFO    Starting training\n",
      "11:29 madminer.ml          INFO      Batch size:             128\n",
      "11:29 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "11:29 madminer.ml          INFO      Epochs:                 50\n",
      "11:29 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "11:29 madminer.ml          INFO      Validation split:       0.25\n",
      "11:29 madminer.ml          INFO      Early stopping:         True\n",
      "11:29 madminer.ml          INFO      Scale inputs:           True\n",
      "11:29 madminer.ml          INFO      Shuffle labels          False\n",
      "11:29 madminer.ml          INFO      Samples:                all\n",
      "11:29 madminer.ml          INFO    Loading training data\n",
      "11:29 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/x_train_score_2.npy into RAM\n",
      "11:29 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/t_xz_train_score_2.npy into RAM\n",
      "11:29 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "11:29 madminer.ml          INFO    Rescaling inputs\n",
      "11:29 madminer.ml          INFO    Creating model\n",
      "11:29 madminer.ml          INFO    Training model\n",
      "11:29 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "11:30 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "11:30 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:30 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "11:30 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:31 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "11:31 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:32 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "11:32 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:33 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00020 (mse_score:  0.000)\n",
      "11:33 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:34 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00020 (mse_score:  0.000)\n",
      "11:34 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:34 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "11:34 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:35 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "11:35 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:36 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "11:36 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:37 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "11:37 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:38 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "11:38 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:38 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "11:38 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:39 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "11:39 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:40 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "11:40 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:41 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "11:41 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:41 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "11:41 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:42 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00019 compared to final loss  0.00019\n",
      "11:42 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "11:42 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "11:42 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11:42 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "11:42 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "11:42 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "11:42 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "11:42 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "11:42 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "11:42 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "11:42 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "11:42 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "11:42 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "11:42 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "11:42 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "11:42 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "11:42 madminer.ml          INFO    Starting training\n",
      "11:42 madminer.ml          INFO      Batch size:             128\n",
      "11:42 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "11:42 madminer.ml          INFO      Epochs:                 50\n",
      "11:42 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "11:42 madminer.ml          INFO      Validation split:       0.25\n",
      "11:42 madminer.ml          INFO      Early stopping:         True\n",
      "11:42 madminer.ml          INFO      Scale inputs:           True\n",
      "11:42 madminer.ml          INFO      Shuffle labels          False\n",
      "11:42 madminer.ml          INFO      Samples:                all\n",
      "11:42 madminer.ml          INFO    Loading training data\n",
      "11:42 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/x_train_score_3.npy into RAM\n",
      "11:42 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/t_xz_train_score_3.npy into RAM\n",
      "11:42 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "11:42 madminer.ml          INFO    Rescaling inputs\n",
      "11:42 madminer.ml          INFO    Creating model\n",
      "11:42 madminer.ml          INFO    Training model\n",
      "11:42 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "11:43 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "11:43 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:44 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "11:44 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:44 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "11:44 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:45 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "11:45 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:46 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "11:46 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:47 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "11:47 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:48 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "11:48 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:48 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "11:48 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:49 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "11:49 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:50 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "11:50 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:51 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "11:51 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:51 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "11:51 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:52 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "11:52 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:53 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "11:53 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:54 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "11:54 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:55 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "11:55 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "11:55 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00019 compared to final loss  0.00019\n",
      "11:55 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "11:55 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "11:55 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "11:55 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "11:55 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "11:55 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "11:55 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "11:55 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "11:55 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "11:55 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "11:55 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "11:55 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "11:55 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "11:55 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "11:55 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "11:55 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "11:55 madminer.ml          INFO    Starting training\n",
      "11:55 madminer.ml          INFO      Batch size:             128\n",
      "11:55 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "11:55 madminer.ml          INFO      Epochs:                 50\n",
      "11:55 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "11:55 madminer.ml          INFO      Validation split:       0.25\n",
      "11:55 madminer.ml          INFO      Early stopping:         True\n",
      "11:55 madminer.ml          INFO      Scale inputs:           True\n",
      "11:55 madminer.ml          INFO      Shuffle labels          False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11:55 madminer.ml          INFO      Samples:                all\n",
      "11:55 madminer.ml          INFO    Loading training data\n",
      "11:55 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/x_train_score_4.npy into RAM\n",
      "11:55 madminer.utils.vario INFO      Loading samples/met/samples_wmh_mu_smeftsim_met/t_xz_train_score_4.npy into RAM\n",
      "11:55 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "11:55 madminer.ml          INFO    Rescaling inputs\n",
      "11:55 madminer.ml          INFO    Creating model\n",
      "11:55 madminer.ml          INFO    Training model\n",
      "11:55 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "11:56 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "11:56 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:57 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "11:57 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:58 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "11:58 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:58 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "11:58 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "11:59 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "11:59 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:00 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "12:00 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:01 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "12:01 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:02 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "12:02 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:02 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "12:02 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:03 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "12:03 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:04 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "12:04 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:05 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "12:05 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:05 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "12:05 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:06 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "12:06 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:07 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "12:07 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:08 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "12:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:08 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00019 compared to final loss  0.00019\n",
      "12:08 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "12:08 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "12:08 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "12:08 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "12:08 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "12:08 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "12:08 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "12:08 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "12:08 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "12:08 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "12:08 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "12:08 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "12:08 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "12:08 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "12:08 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "12:08 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_mu_smeftsim_met/estimator_0\n",
      "12:08 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_mu_smeftsim_met/estimator_1\n",
      "12:08 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_mu_smeftsim_met/estimator_2\n",
      "12:08 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_mu_smeftsim_met/estimator_3\n",
      "12:08 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_mu_smeftsim_met/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wmh_mu_smeftsim','met',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:08 madminer.analysis    INFO    Loading data from data/met/signal/wmh_e_smeftsim_lhedata_met.h5\n",
      "12:08 madminer.analysis    INFO    Found 4 parameters\n",
      "12:08 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "12:08 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "12:08 madminer.analysis    INFO    Found 48 observables\n",
      "12:08 madminer.analysis    INFO    Found 1000000 events\n",
      "12:08 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "12:08 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "12:08 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "12:08 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "12:08 madminer.sampling    INFO    Starting sampling serially\n",
      "12:08 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:09 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "12:09 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "12:09 madminer.sampling    INFO    Starting sampling serially\n",
      "12:09 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "12:09 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "12:09 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "12:09 madminer.sampling    INFO    Starting sampling serially\n",
      "12:09 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "12:09 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "12:09 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "12:09 madminer.sampling    INFO    Starting sampling serially\n",
      "12:09 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "12:09 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "12:09 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "12:09 madminer.sampling    INFO    Starting sampling serially\n",
      "12:09 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "12:10 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "12:10 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "12:10 madminer.sampling    INFO    Starting sampling serially\n",
      "12:10 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:10 madminer.sampling    INFO    Effective number of samples: mean 199827.145512, with individual thetas ranging from 199827.145512 to 199827.145512\n",
      "12:10 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "12:10 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "12:10 madminer.ml          INFO    Starting training\n",
      "12:10 madminer.ml          INFO      Batch size:             128\n",
      "12:10 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "12:10 madminer.ml          INFO      Epochs:                 50\n",
      "12:10 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "12:10 madminer.ml          INFO      Validation split:       0.25\n",
      "12:10 madminer.ml          INFO      Early stopping:         True\n",
      "12:10 madminer.ml          INFO      Scale inputs:           True\n",
      "12:10 madminer.ml          INFO      Shuffle labels          False\n",
      "12:10 madminer.ml          INFO      Samples:                all\n",
      "12:10 madminer.ml          INFO    Loading training data\n",
      "12:10 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/x_train_score_0.npy into RAM\n",
      "12:10 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:10 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "12:10 madminer.ml          INFO    Rescaling inputs\n",
      "12:10 madminer.ml          INFO    Creating model\n",
      "12:10 madminer.ml          INFO    Training model\n",
      "12:10 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "12:11 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "12:11 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:11 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "12:11 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:12 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "12:12 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:13 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "12:13 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:14 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "12:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:14 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "12:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:15 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "12:15 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:16 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "12:16 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:17 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "12:17 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:18 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "12:18 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:18 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "12:18 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:19 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "12:19 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:20 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "12:20 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:21 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "12:21 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:22 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "12:22 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:22 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "12:22 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:23 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "12:23 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "12:23 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "12:23 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "12:23 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "12:23 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "12:23 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "12:23 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "12:23 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "12:23 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "12:23 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "12:23 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "12:23 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "12:23 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "12:23 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "12:23 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "12:23 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "12:23 madminer.ml          INFO    Starting training\n",
      "12:23 madminer.ml          INFO      Batch size:             128\n",
      "12:23 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "12:23 madminer.ml          INFO      Epochs:                 50\n",
      "12:23 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "12:23 madminer.ml          INFO      Validation split:       0.25\n",
      "12:23 madminer.ml          INFO      Early stopping:         True\n",
      "12:23 madminer.ml          INFO      Scale inputs:           True\n",
      "12:23 madminer.ml          INFO      Shuffle labels          False\n",
      "12:23 madminer.ml          INFO      Samples:                all\n",
      "12:23 madminer.ml          INFO    Loading training data\n",
      "12:23 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/x_train_score_1.npy into RAM\n",
      "12:23 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/t_xz_train_score_1.npy into RAM\n",
      "12:23 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "12:23 madminer.ml          INFO    Rescaling inputs\n",
      "12:23 madminer.ml          INFO    Creating model\n",
      "12:23 madminer.ml          INFO    Training model\n",
      "12:23 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "12:24 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "12:24 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:24 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "12:24 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:25 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "12:25 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:26 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "12:26 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:27 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "12:27 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:28 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "12:28 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:28 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "12:28 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:29 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "12:29 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:30 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "12:30 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:31 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "12:31 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:31 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "12:31 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:32 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "12:32 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:33 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "12:33 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:34 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "12:34 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:35 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "12:35 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:35 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "12:35 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "12:36 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "12:36 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "12:36 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "12:36 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "12:36 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "12:36 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "12:36 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "12:36 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "12:36 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "12:36 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "12:36 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "12:36 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "12:36 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "12:36 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "12:36 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "12:36 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "12:36 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "12:36 madminer.ml          INFO    Starting training\n",
      "12:36 madminer.ml          INFO      Batch size:             128\n",
      "12:36 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "12:36 madminer.ml          INFO      Epochs:                 50\n",
      "12:36 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "12:36 madminer.ml          INFO      Validation split:       0.25\n",
      "12:36 madminer.ml          INFO      Early stopping:         True\n",
      "12:36 madminer.ml          INFO      Scale inputs:           True\n",
      "12:36 madminer.ml          INFO      Shuffle labels          False\n",
      "12:36 madminer.ml          INFO      Samples:                all\n",
      "12:36 madminer.ml          INFO    Loading training data\n",
      "12:36 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/x_train_score_2.npy into RAM\n",
      "12:36 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/t_xz_train_score_2.npy into RAM\n",
      "12:36 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "12:36 madminer.ml          INFO    Rescaling inputs\n",
      "12:36 madminer.ml          INFO    Creating model\n",
      "12:36 madminer.ml          INFO    Training model\n",
      "12:36 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "12:37 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "12:37 madminer.utils.ml.tr INFO               val. loss   0.00021 (mse_score:  0.000)\n",
      "12:37 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "12:37 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:38 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "12:38 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:39 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "12:39 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:40 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "12:40 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:41 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "12:41 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:41 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "12:41 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:42 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "12:42 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:43 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "12:43 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:44 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "12:44 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:45 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "12:45 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:45 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "12:45 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:46 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "12:46 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:47 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "12:47 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:48 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "12:48 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:48 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "12:48 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:49 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "12:49 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "12:49 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "12:49 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "12:49 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "12:49 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:49 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "12:49 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "12:49 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "12:49 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "12:49 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "12:49 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "12:49 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "12:49 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "12:49 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "12:49 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "12:49 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "12:49 madminer.ml          INFO    Starting training\n",
      "12:49 madminer.ml          INFO      Batch size:             128\n",
      "12:49 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "12:49 madminer.ml          INFO      Epochs:                 50\n",
      "12:49 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "12:49 madminer.ml          INFO      Validation split:       0.25\n",
      "12:49 madminer.ml          INFO      Early stopping:         True\n",
      "12:49 madminer.ml          INFO      Scale inputs:           True\n",
      "12:49 madminer.ml          INFO      Shuffle labels          False\n",
      "12:49 madminer.ml          INFO      Samples:                all\n",
      "12:49 madminer.ml          INFO    Loading training data\n",
      "12:49 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/x_train_score_3.npy into RAM\n",
      "12:49 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/t_xz_train_score_3.npy into RAM\n",
      "12:49 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "12:49 madminer.ml          INFO    Rescaling inputs\n",
      "12:49 madminer.ml          INFO    Creating model\n",
      "12:49 madminer.ml          INFO    Training model\n",
      "12:49 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "12:50 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "12:50 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:51 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "12:51 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:51 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "12:51 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:52 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "12:52 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:53 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00020 (mse_score:  0.000)\n",
      "12:53 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:54 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00020 (mse_score:  0.000)\n",
      "12:54 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:54 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "12:54 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:55 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "12:55 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:56 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "12:56 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:57 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "12:57 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:58 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "12:58 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:58 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "12:58 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "12:59 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "12:59 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "13:00 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "13:00 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "13:01 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "13:01 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "13:01 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "13:01 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "13:02 madminer.utils.ml.tr INFO    Early stopping after epoch 46, with loss  0.00020 compared to final loss  0.00020\n",
      "13:02 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "13:02 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "13:02 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "13:02 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "13:02 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "13:02 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "13:02 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "13:02 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "13:02 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "13:02 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "13:02 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "13:02 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "13:02 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "13:02 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "13:02 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "13:02 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "13:02 madminer.ml          INFO    Starting training\n",
      "13:02 madminer.ml          INFO      Batch size:             128\n",
      "13:02 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "13:02 madminer.ml          INFO      Epochs:                 50\n",
      "13:02 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "13:02 madminer.ml          INFO      Validation split:       0.25\n",
      "13:02 madminer.ml          INFO      Early stopping:         True\n",
      "13:02 madminer.ml          INFO      Scale inputs:           True\n",
      "13:02 madminer.ml          INFO      Shuffle labels          False\n",
      "13:02 madminer.ml          INFO      Samples:                all\n",
      "13:02 madminer.ml          INFO    Loading training data\n",
      "13:02 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/x_train_score_4.npy into RAM\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13:02 madminer.utils.vario INFO      Loading samples/met/samples_wmh_e_smeftsim_met/t_xz_train_score_4.npy into RAM\n",
      "13:02 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "13:02 madminer.ml          INFO    Rescaling inputs\n",
      "13:02 madminer.ml          INFO    Creating model\n",
      "13:02 madminer.ml          INFO    Training model\n",
      "13:02 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "13:03 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "13:03 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "13:03 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "13:03 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "13:04 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "13:04 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:05 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "13:05 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:06 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00020 (mse_score:  0.000)\n",
      "13:06 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:07 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00020 (mse_score:  0.000)\n",
      "13:07 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:07 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00020 (mse_score:  0.000)\n",
      "13:07 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:08 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "13:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:09 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "13:09 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:10 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "13:10 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:11 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "13:11 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:11 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "13:11 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:12 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "13:12 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:13 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "13:13 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:14 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "13:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:14 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "13:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "13:15 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00019 compared to final loss  0.00019\n",
      "13:15 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "13:15 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "13:15 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "13:15 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "13:15 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "13:15 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "13:15 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "13:15 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "13:15 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "13:15 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "13:15 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "13:15 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "13:15 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "13:15 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "13:15 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "13:15 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_e_smeftsim_met/estimator_0\n",
      "13:15 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_e_smeftsim_met/estimator_1\n",
      "13:15 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_e_smeftsim_met/estimator_2\n",
      "13:15 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_e_smeftsim_met/estimator_3\n",
      "13:15 madminer.ml          INFO    Saving model to models/met/sally_ensemble_wmh_e_smeftsim_met/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wmh_e_smeftsim','met',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### $p_{T,W}$ Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18:11 madminer.analysis    INFO    Loading data from data/met/signal/wph_mu_smeftsim_lhedata_met.h5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18:11 madminer.analysis    INFO    Found 4 parameters\n",
      "18:11 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "18:11 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "18:11 madminer.analysis    INFO    Found 48 observables\n",
      "18:11 madminer.analysis    INFO    Found 1000000 events\n",
      "18:11 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "18:11 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "18:11 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "18:11 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "18:11 madminer.sampling    INFO    Starting sampling serially\n",
      "18:11 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "18:12 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:12 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:12 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "18:12 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "18:12 madminer.sampling    INFO    Starting sampling serially\n",
      "18:12 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "18:12 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:12 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:12 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "18:12 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "18:12 madminer.sampling    INFO    Starting sampling serially\n",
      "18:12 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "18:12 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:12 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:13 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "18:13 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "18:13 madminer.sampling    INFO    Starting sampling serially\n",
      "18:13 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "18:13 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:13 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:13 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "18:13 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "18:13 madminer.sampling    INFO    Starting sampling serially\n",
      "18:13 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "18:13 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:13 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "18:13 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "18:13 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "18:13 madminer.sampling    INFO    Starting sampling serially\n",
      "18:13 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18:13 madminer.sampling    INFO    Effective number of samples: mean 199716.36363, with individual thetas ranging from 199716.36363 to 199716.36363\n",
      "18:13 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "18:13 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "18:13 madminer.ml          INFO    Starting training\n",
      "18:13 madminer.ml          INFO      Batch size:             128\n",
      "18:13 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "18:13 madminer.ml          INFO      Epochs:                 50\n",
      "18:13 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "18:13 madminer.ml          INFO      Validation split:       0.25\n",
      "18:13 madminer.ml          INFO      Early stopping:         True\n",
      "18:13 madminer.ml          INFO      Scale inputs:           True\n",
      "18:13 madminer.ml          INFO      Shuffle labels          False\n",
      "18:13 madminer.ml          INFO      Samples:                all\n",
      "18:13 madminer.ml          INFO    Loading training data\n",
      "18:13 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/x_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18:13 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/t_xz_train_score_0.npy into RAM\n",
      "18:13 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "18:13 madminer.ml          INFO    Rescaling inputs\n",
      "18:13 madminer.ml          INFO    Creating model\n",
      "18:13 madminer.ml          INFO    Training model\n",
      "18:13 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "18:14 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "18:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:15 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "18:15 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:16 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "18:16 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:17 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "18:17 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:17 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "18:17 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:18 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "18:18 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:19 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "18:19 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:20 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "18:20 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:20 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "18:20 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:21 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "18:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:22 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "18:22 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:23 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "18:23 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:24 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "18:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:24 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "18:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:25 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "18:25 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:26 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "18:26 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:26 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00018 compared to final loss  0.00018\n",
      "18:26 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "18:26 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "18:26 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "18:26 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "18:26 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "18:26 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "18:26 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "18:26 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "18:26 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "18:26 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "18:26 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "18:26 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "18:26 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "18:26 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "18:26 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "18:26 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "18:26 madminer.ml          INFO    Starting training\n",
      "18:26 madminer.ml          INFO      Batch size:             128\n",
      "18:26 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "18:26 madminer.ml          INFO      Epochs:                 50\n",
      "18:26 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "18:26 madminer.ml          INFO      Validation split:       0.25\n",
      "18:26 madminer.ml          INFO      Early stopping:         True\n",
      "18:26 madminer.ml          INFO      Scale inputs:           True\n",
      "18:26 madminer.ml          INFO      Shuffle labels          False\n",
      "18:26 madminer.ml          INFO      Samples:                all\n",
      "18:26 madminer.ml          INFO    Loading training data\n",
      "18:26 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/x_train_score_1.npy into RAM\n",
      "18:26 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/t_xz_train_score_1.npy into RAM\n",
      "18:26 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "18:26 madminer.ml          INFO    Rescaling inputs\n",
      "18:27 madminer.ml          INFO    Creating model\n",
      "18:27 madminer.ml          INFO    Training model\n",
      "18:27 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "18:27 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "18:27 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:28 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "18:28 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:29 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "18:29 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:30 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "18:30 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:30 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "18:30 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:31 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "18:31 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:32 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "18:32 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:33 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "18:33 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:34 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "18:34 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18:34 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "18:34 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:35 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "18:35 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:36 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "18:36 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:37 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "18:37 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:38 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "18:38 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:38 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "18:38 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:39 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "18:39 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:40 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "18:40 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "18:40 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "18:40 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "18:40 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "18:40 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "18:40 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "18:40 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "18:40 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "18:40 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "18:40 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "18:40 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "18:40 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "18:40 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "18:40 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "18:40 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "18:40 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "18:40 madminer.ml          INFO    Starting training\n",
      "18:40 madminer.ml          INFO      Batch size:             128\n",
      "18:40 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "18:40 madminer.ml          INFO      Epochs:                 50\n",
      "18:40 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "18:40 madminer.ml          INFO      Validation split:       0.25\n",
      "18:40 madminer.ml          INFO      Early stopping:         True\n",
      "18:40 madminer.ml          INFO      Scale inputs:           True\n",
      "18:40 madminer.ml          INFO      Shuffle labels          False\n",
      "18:40 madminer.ml          INFO      Samples:                all\n",
      "18:40 madminer.ml          INFO    Loading training data\n",
      "18:40 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/x_train_score_2.npy into RAM\n",
      "18:40 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/t_xz_train_score_2.npy into RAM\n",
      "18:40 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "18:40 madminer.ml          INFO    Rescaling inputs\n",
      "18:40 madminer.ml          INFO    Creating model\n",
      "18:40 madminer.ml          INFO    Training model\n",
      "18:40 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "18:41 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "18:41 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:41 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "18:41 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:42 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "18:42 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:43 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "18:43 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:44 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "18:44 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:45 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "18:45 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:45 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "18:45 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:46 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "18:46 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:47 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "18:47 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:48 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "18:48 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:49 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "18:49 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:49 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "18:49 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:50 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "18:50 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:51 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "18:51 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:52 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "18:52 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:52 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "18:52 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "18:53 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "18:53 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "18:53 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "18:53 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                                set lr:   0.00h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18:53 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "18:53 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "18:53 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "18:53 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "18:53 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "18:53 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "18:53 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "18:53 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "18:53 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "18:53 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "18:53 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "18:53 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "18:53 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "18:53 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "18:53 madminer.ml          INFO    Starting training\n",
      "18:53 madminer.ml          INFO      Batch size:             128\n",
      "18:53 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "18:53 madminer.ml          INFO      Epochs:                 50\n",
      "18:53 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "18:53 madminer.ml          INFO      Validation split:       0.25\n",
      "18:53 madminer.ml          INFO      Early stopping:         True\n",
      "18:53 madminer.ml          INFO      Scale inputs:           True\n",
      "18:53 madminer.ml          INFO      Shuffle labels          False\n",
      "18:53 madminer.ml          INFO      Samples:                all\n",
      "18:53 madminer.ml          INFO    Loading training data\n",
      "18:53 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/x_train_score_3.npy into RAM\n",
      "18:53 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/t_xz_train_score_3.npy into RAM\n",
      "18:53 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "18:53 madminer.ml          INFO    Rescaling inputs\n",
      "18:53 madminer.ml          INFO    Creating model\n",
      "18:53 madminer.ml          INFO    Training model\n",
      "18:53 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "18:54 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "18:54 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:55 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "18:55 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:55 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "18:55 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:56 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "18:56 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:57 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "18:57 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:58 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "18:58 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "18:59 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "18:59 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:00 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "19:00 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:01 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "19:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:01 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "19:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:02 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "19:02 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:03 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "19:03 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:04 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "19:04 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:05 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "19:05 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:05 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "19:05 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:06 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "19:06 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:07 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00018 compared to final loss  0.00018\n",
      "19:07 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "19:07 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                                   ALL:   0.23h\n",
      "19:07 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "19:07 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "19:07 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "19:07 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "19:07 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "19:07 madminer.utils.ml.tr INFO                 training forward pass:   0.05h\n",
      "19:07 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "19:07 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "19:07 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "19:07 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "19:07 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "19:07 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "19:07 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "19:07 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "19:07 madminer.ml          INFO    Starting training\n",
      "19:07 madminer.ml          INFO      Batch size:             128\n",
      "19:07 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "19:07 madminer.ml          INFO      Epochs:                 50\n",
      "19:07 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "19:07 madminer.ml          INFO      Validation split:       0.25\n",
      "19:07 madminer.ml          INFO      Early stopping:         True\n",
      "19:07 madminer.ml          INFO      Scale inputs:           True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19:07 madminer.ml          INFO      Shuffle labels          False\n",
      "19:07 madminer.ml          INFO      Samples:                all\n",
      "19:07 madminer.ml          INFO    Loading training data\n",
      "19:07 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/x_train_score_4.npy into RAM\n",
      "19:07 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_mu_smeftsim_ptw/t_xz_train_score_4.npy into RAM\n",
      "19:07 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "19:07 madminer.ml          INFO    Rescaling inputs\n",
      "19:07 madminer.ml          INFO    Creating model\n",
      "19:07 madminer.ml          INFO    Training model\n",
      "19:07 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "19:08 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "19:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:08 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "19:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:09 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "19:09 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:10 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "19:10 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:11 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "19:11 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:11 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "19:11 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:12 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "19:12 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:13 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "19:13 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:14 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "19:14 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:15 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "19:15 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:15 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "19:15 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:16 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "19:16 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:17 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "19:17 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:18 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "19:18 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:18 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "19:18 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:19 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "19:19 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:20 madminer.utils.ml.tr INFO    Early stopping after epoch 45, with loss  0.00018 compared to final loss  0.00018\n",
      "19:20 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "19:20 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "19:20 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "19:20 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "19:20 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "19:20 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "19:20 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "19:20 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "19:20 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "19:20 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "19:20 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "19:20 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "19:20 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "19:20 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "19:20 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "19:20 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_mu_smeftsim_ptw/estimator_0\n",
      "19:20 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_mu_smeftsim_ptw/estimator_1\n",
      "19:20 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_mu_smeftsim_ptw/estimator_2\n",
      "19:20 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_mu_smeftsim_ptw/estimator_3\n",
      "19:20 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_mu_smeftsim_ptw/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wph_mu_smeftsim','ptw',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19:20 madminer.analysis    INFO    Loading data from data/met/signal/wph_e_smeftsim_lhedata_met.h5\n",
      "19:20 madminer.analysis    INFO    Found 4 parameters\n",
      "19:20 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "19:20 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "19:20 madminer.analysis    INFO    Found 48 observables\n",
      "19:20 madminer.analysis    INFO    Found 1000000 events\n",
      "19:20 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "19:20 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "19:20 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "19:20 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19:20 madminer.sampling    INFO    Starting sampling serially\n",
      "19:20 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "19:20 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "19:20 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "19:20 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "19:20 madminer.sampling    INFO    Starting sampling serially\n",
      "19:20 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "19:20 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "19:20 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "19:20 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "19:20 madminer.sampling    INFO    Starting sampling serially\n",
      "19:20 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "19:21 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "19:21 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "19:21 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "19:21 madminer.sampling    INFO    Starting sampling serially\n",
      "19:21 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "19:21 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "19:21 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "19:21 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "19:21 madminer.sampling    INFO    Starting sampling serially\n",
      "19:21 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "19:21 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "19:21 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "19:21 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19:21 madminer.sampling    INFO    Starting sampling serially\n",
      "19:21 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "19:21 madminer.sampling    INFO    Effective number of samples: mean 199716.36363, with individual thetas ranging from 199716.36363 to 199716.36363\n",
      "19:21 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "19:21 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "19:21 madminer.ml          INFO    Starting training\n",
      "19:21 madminer.ml          INFO      Batch size:             128\n",
      "19:21 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "19:21 madminer.ml          INFO      Epochs:                 50\n",
      "19:21 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "19:21 madminer.ml          INFO      Validation split:       0.25\n",
      "19:21 madminer.ml          INFO      Early stopping:         True\n",
      "19:21 madminer.ml          INFO      Scale inputs:           True\n",
      "19:21 madminer.ml          INFO      Shuffle labels          False\n",
      "19:21 madminer.ml          INFO      Samples:                all\n",
      "19:21 madminer.ml          INFO    Loading training data\n",
      "19:21 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/x_train_score_0.npy into RAM\n",
      "19:21 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19:21 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "19:21 madminer.ml          INFO    Rescaling inputs\n",
      "19:21 madminer.ml          INFO    Creating model\n",
      "19:21 madminer.ml          INFO    Training model\n",
      "19:21 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "19:22 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "19:22 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:23 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "19:23 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:24 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "19:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:24 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "19:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:25 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "19:25 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:26 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "19:26 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:27 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "19:27 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:27 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "19:27 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:28 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "19:28 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:29 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "19:29 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:30 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "19:30 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:31 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "19:31 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:31 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "19:31 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:32 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "19:32 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:33 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "19:33 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:34 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "19:34 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:34 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00017 compared to final loss  0.00017\n",
      "19:34 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "19:34 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "19:34 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "19:34 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "19:34 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "19:34 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "19:34 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "19:34 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "19:34 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "19:34 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "19:34 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "19:34 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "19:34 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "19:34 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "19:34 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "19:34 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "19:34 madminer.ml          INFO    Starting training\n",
      "19:34 madminer.ml          INFO      Batch size:             128\n",
      "19:34 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "19:34 madminer.ml          INFO      Epochs:                 50\n",
      "19:34 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "19:34 madminer.ml          INFO      Validation split:       0.25\n",
      "19:34 madminer.ml          INFO      Early stopping:         True\n",
      "19:34 madminer.ml          INFO      Scale inputs:           True\n",
      "19:34 madminer.ml          INFO      Shuffle labels          False\n",
      "19:34 madminer.ml          INFO      Samples:                all\n",
      "19:34 madminer.ml          INFO    Loading training data\n",
      "19:34 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/x_train_score_1.npy into RAM\n",
      "19:34 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/t_xz_train_score_1.npy into RAM\n",
      "19:34 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "19:34 madminer.ml          INFO    Rescaling inputs\n",
      "19:34 madminer.ml          INFO    Creating model\n",
      "19:34 madminer.ml          INFO    Training model\n",
      "19:34 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "19:35 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "19:35 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:36 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "19:36 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:36 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "19:36 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:37 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "19:37 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:38 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "19:38 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:39 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "19:39 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:40 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "19:40 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:40 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "19:40 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:41 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "19:41 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:42 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19:42 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:43 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "19:43 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:43 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "19:43 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:44 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "19:44 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:45 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "19:45 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:46 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "19:46 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:47 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "19:47 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:47 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "19:47 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "19:47 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "19:47 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "19:47 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "19:47 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "19:47 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "19:47 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "19:47 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "19:47 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "19:47 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "19:47 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "19:47 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "19:47 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "19:47 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "19:47 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "19:47 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "19:47 madminer.ml          INFO    Starting training\n",
      "19:47 madminer.ml          INFO      Batch size:             128\n",
      "19:47 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "19:47 madminer.ml          INFO      Epochs:                 50\n",
      "19:47 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "19:47 madminer.ml          INFO      Validation split:       0.25\n",
      "19:47 madminer.ml          INFO      Early stopping:         True\n",
      "19:47 madminer.ml          INFO      Scale inputs:           True\n",
      "19:47 madminer.ml          INFO      Shuffle labels          False\n",
      "19:47 madminer.ml          INFO      Samples:                all\n",
      "19:47 madminer.ml          INFO    Loading training data\n",
      "19:47 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/x_train_score_2.npy into RAM\n",
      "19:47 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/t_xz_train_score_2.npy into RAM\n",
      "19:47 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "19:47 madminer.ml          INFO    Rescaling inputs\n",
      "19:47 madminer.ml          INFO    Creating model\n",
      "19:47 madminer.ml          INFO    Training model\n",
      "19:47 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "19:48 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "19:48 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "19:49 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "19:49 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:49 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "19:49 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:50 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "19:50 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:51 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "19:51 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:52 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "19:52 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:52 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "19:52 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:53 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "19:53 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "19:54 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "19:54 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:55 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "19:55 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:56 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "19:56 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:56 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "19:56 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:57 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "19:57 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:58 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "19:58 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:59 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "19:59 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "19:59 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "19:59 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "20:00 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00017 compared to final loss  0.00017\n",
      "20:00 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "20:00 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "20:00 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20:00 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "20:00 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "20:00 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "20:00 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "20:00 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "20:00 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "20:00 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "20:00 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "20:00 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "20:00 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "20:00 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "20:00 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "20:00 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "20:00 madminer.ml          INFO    Starting training\n",
      "20:00 madminer.ml          INFO      Batch size:             128\n",
      "20:00 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "20:00 madminer.ml          INFO      Epochs:                 50\n",
      "20:00 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "20:00 madminer.ml          INFO      Validation split:       0.25\n",
      "20:00 madminer.ml          INFO      Early stopping:         True\n",
      "20:00 madminer.ml          INFO      Scale inputs:           True\n",
      "20:00 madminer.ml          INFO      Shuffle labels          False\n",
      "20:00 madminer.ml          INFO      Samples:                all\n",
      "20:00 madminer.ml          INFO    Loading training data\n",
      "20:00 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/x_train_score_3.npy into RAM\n",
      "20:00 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/t_xz_train_score_3.npy into RAM\n",
      "20:00 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "20:00 madminer.ml          INFO    Rescaling inputs\n",
      "20:00 madminer.ml          INFO    Creating model\n",
      "20:00 madminer.ml          INFO    Training model\n",
      "20:00 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "20:01 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "20:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:01 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "20:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:02 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "20:02 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:03 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "20:03 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:04 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "20:04 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:05 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "20:05 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:05 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "20:05 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:06 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "20:06 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:07 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "20:07 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:08 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "20:08 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:08 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "20:08 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:09 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "20:09 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "20:10 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "20:10 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "20:11 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "20:11 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "20:12 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "20:12 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "20:12 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "20:12 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "20:13 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00017 compared to final loss  0.00017\n",
      "20:13 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "20:13 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "20:13 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "20:13 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "20:13 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "20:13 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "20:13 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "20:13 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "20:13 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "20:13 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "20:13 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "20:13 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "20:13 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "20:13 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "20:13 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "20:13 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "20:13 madminer.ml          INFO    Starting training\n",
      "20:13 madminer.ml          INFO      Batch size:             128\n",
      "20:13 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "20:13 madminer.ml          INFO      Epochs:                 50\n",
      "20:13 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "20:13 madminer.ml          INFO      Validation split:       0.25\n",
      "20:13 madminer.ml          INFO      Early stopping:         True\n",
      "20:13 madminer.ml          INFO      Scale inputs:           True\n",
      "20:13 madminer.ml          INFO      Shuffle labels          False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20:13 madminer.ml          INFO      Samples:                all\n",
      "20:13 madminer.ml          INFO    Loading training data\n",
      "20:13 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/x_train_score_4.npy into RAM\n",
      "20:13 madminer.utils.vario INFO      Loading samples/ptw/samples_wph_e_smeftsim_ptw/t_xz_train_score_4.npy into RAM\n",
      "20:13 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "20:13 madminer.ml          INFO    Rescaling inputs\n",
      "20:13 madminer.ml          INFO    Creating model\n",
      "20:13 madminer.ml          INFO    Training model\n",
      "20:13 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "20:14 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "20:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "20:14 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "20:14 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:15 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "20:15 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:16 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "20:16 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:17 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "20:17 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:17 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "20:17 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:18 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "20:18 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:19 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "20:19 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:20 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "20:20 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:21 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "20:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:21 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "20:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:22 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "20:22 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:23 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "20:23 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:24 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "20:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:24 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "20:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:25 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "20:25 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "20:26 madminer.utils.ml.tr INFO    Early stopping after epoch 47, with loss  0.00018 compared to final loss  0.00018\n",
      "20:26 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "20:26 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "20:26 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "20:26 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "20:26 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "20:26 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "20:26 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "20:26 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "20:26 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "20:26 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "20:26 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "20:26 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "20:26 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "20:26 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "20:26 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "20:26 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_e_smeftsim_ptw/estimator_0\n",
      "20:26 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_e_smeftsim_ptw/estimator_1\n",
      "20:26 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_e_smeftsim_ptw/estimator_2\n",
      "20:26 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_e_smeftsim_ptw/estimator_3\n",
      "20:26 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wph_e_smeftsim_ptw/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wph_e_smeftsim','ptw',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20:26 madminer.analysis    INFO    Loading data from data/met/signal/wmh_mu_smeftsim_lhedata_met.h5\n",
      "20:26 madminer.analysis    INFO    Found 4 parameters\n",
      "20:26 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "20:26 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "20:26 madminer.analysis    INFO    Found 48 observables\n",
      "20:26 madminer.analysis    INFO    Found 1000000 events\n",
      "20:26 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "20:26 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "20:26 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "20:26 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "20:26 madminer.sampling    INFO    Starting sampling serially\n",
      "20:26 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20:26 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "20:26 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "20:26 madminer.sampling    INFO    Starting sampling serially\n",
      "20:26 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "20:26 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "20:26 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "20:26 madminer.sampling    INFO    Starting sampling serially\n",
      "20:26 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "20:26 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "20:26 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "20:26 madminer.sampling    INFO    Starting sampling serially\n",
      "20:26 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "20:27 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "20:27 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "20:27 madminer.sampling    INFO    Starting sampling serially\n",
      "20:27 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "20:27 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "20:27 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "20:27 madminer.sampling    INFO    Starting sampling serially\n",
      "20:27 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20:27 madminer.sampling    INFO    Effective number of samples: mean 199827.145512, with individual thetas ranging from 199827.145512 to 199827.145512\n",
      "20:27 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "20:27 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "20:27 madminer.ml          INFO    Starting training\n",
      "20:27 madminer.ml          INFO      Batch size:             128\n",
      "20:27 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "20:27 madminer.ml          INFO      Epochs:                 50\n",
      "20:27 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "20:27 madminer.ml          INFO      Validation split:       0.25\n",
      "20:27 madminer.ml          INFO      Early stopping:         True\n",
      "20:27 madminer.ml          INFO      Scale inputs:           True\n",
      "20:27 madminer.ml          INFO      Shuffle labels          False\n",
      "20:27 madminer.ml          INFO      Samples:                all\n",
      "20:27 madminer.ml          INFO    Loading training data\n",
      "20:27 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/x_train_score_0.npy into RAM\n",
      "20:27 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20:27 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "20:27 madminer.ml          INFO    Rescaling inputs\n",
      "20:27 madminer.ml          INFO    Creating model\n",
      "20:27 madminer.ml          INFO    Training model\n",
      "20:27 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "20:28 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "20:28 madminer.utils.ml.tr INFO               val. loss   0.00021 (mse_score:  0.000)\n",
      "20:29 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "20:29 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:29 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "20:29 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:30 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "20:30 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:31 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "20:31 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:32 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "20:32 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:32 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "20:32 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:33 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "20:33 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:34 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "20:34 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:35 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "20:35 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:35 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "20:35 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:36 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "20:36 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:37 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "20:37 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:38 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "20:38 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:39 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "20:39 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:39 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "20:39 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:40 madminer.utils.ml.tr INFO    Early stopping after epoch 47, with loss  0.00020 compared to final loss  0.00020\n",
      "20:40 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "20:40 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "20:40 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "20:40 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "20:40 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "20:40 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "20:40 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "20:40 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "20:40 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "20:40 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "20:40 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "20:40 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "20:40 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "20:40 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "20:40 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "20:40 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "20:40 madminer.ml          INFO    Starting training\n",
      "20:40 madminer.ml          INFO      Batch size:             128\n",
      "20:40 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "20:40 madminer.ml          INFO      Epochs:                 50\n",
      "20:40 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "20:40 madminer.ml          INFO      Validation split:       0.25\n",
      "20:40 madminer.ml          INFO      Early stopping:         True\n",
      "20:40 madminer.ml          INFO      Scale inputs:           True\n",
      "20:40 madminer.ml          INFO      Shuffle labels          False\n",
      "20:40 madminer.ml          INFO      Samples:                all\n",
      "20:40 madminer.ml          INFO    Loading training data\n",
      "20:40 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/x_train_score_1.npy into RAM\n",
      "20:40 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/t_xz_train_score_1.npy into RAM\n",
      "20:40 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "20:40 madminer.ml          INFO    Rescaling inputs\n",
      "20:40 madminer.ml          INFO    Creating model\n",
      "20:40 madminer.ml          INFO    Training model\n",
      "20:40 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "20:41 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "20:41 madminer.utils.ml.tr INFO               val. loss   0.00021 (mse_score:  0.000)\n",
      "20:41 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "20:41 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:42 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "20:42 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:43 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "20:43 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:44 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "20:44 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:44 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "20:44 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:45 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "20:45 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:46 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "20:46 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:47 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "20:47 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:47 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20:47 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:48 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "20:48 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:49 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "20:49 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:50 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "20:50 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:51 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "20:51 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:51 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "20:51 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:52 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "20:52 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:53 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00020 compared to final loss  0.00020\n",
      "20:53 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "20:53 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "20:53 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "20:53 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "20:53 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "20:53 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "20:53 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "20:53 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "20:53 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "20:53 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "20:53 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "20:53 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "20:53 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "20:53 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "20:53 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "20:53 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "20:53 madminer.ml          INFO    Starting training\n",
      "20:53 madminer.ml          INFO      Batch size:             128\n",
      "20:53 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "20:53 madminer.ml          INFO      Epochs:                 50\n",
      "20:53 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "20:53 madminer.ml          INFO      Validation split:       0.25\n",
      "20:53 madminer.ml          INFO      Early stopping:         True\n",
      "20:53 madminer.ml          INFO      Scale inputs:           True\n",
      "20:53 madminer.ml          INFO      Shuffle labels          False\n",
      "20:53 madminer.ml          INFO      Samples:                all\n",
      "20:53 madminer.ml          INFO    Loading training data\n",
      "20:53 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/x_train_score_2.npy into RAM\n",
      "20:53 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/t_xz_train_score_2.npy into RAM\n",
      "20:53 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "20:53 madminer.ml          INFO    Rescaling inputs\n",
      "20:53 madminer.ml          INFO    Creating model\n",
      "20:53 madminer.ml          INFO    Training model\n",
      "20:53 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "20:53 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "20:53 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "20:54 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "20:54 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "20:55 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "20:55 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "20:56 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "20:56 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "20:56 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "20:56 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "20:57 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "20:57 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "20:58 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "20:58 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "20:59 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "20:59 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:00 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "21:00 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:00 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "21:00 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:01 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "21:01 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:02 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "21:02 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:03 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "21:03 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:03 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "21:03 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:04 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "21:04 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:05 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "21:05 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:05 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "21:05 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "21:05 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "21:05 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21:05 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "21:05 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "21:05 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "21:05 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "21:05 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "21:05 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "21:05 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "21:05 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "21:05 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "21:05 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "21:05 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "21:05 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "21:05 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "21:05 madminer.ml          INFO    Starting training\n",
      "21:05 madminer.ml          INFO      Batch size:             128\n",
      "21:05 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "21:05 madminer.ml          INFO      Epochs:                 50\n",
      "21:05 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "21:05 madminer.ml          INFO      Validation split:       0.25\n",
      "21:05 madminer.ml          INFO      Early stopping:         True\n",
      "21:05 madminer.ml          INFO      Scale inputs:           True\n",
      "21:05 madminer.ml          INFO      Shuffle labels          False\n",
      "21:05 madminer.ml          INFO      Samples:                all\n",
      "21:05 madminer.ml          INFO    Loading training data\n",
      "21:05 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/x_train_score_3.npy into RAM\n",
      "21:05 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/t_xz_train_score_3.npy into RAM\n",
      "21:05 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "21:05 madminer.ml          INFO    Rescaling inputs\n",
      "21:05 madminer.ml          INFO    Creating model\n",
      "21:05 madminer.ml          INFO    Training model\n",
      "21:05 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "21:06 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "21:06 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:07 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "21:07 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:08 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "21:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:09 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "21:09 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:09 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "21:09 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:10 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "21:10 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:11 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "21:11 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:12 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "21:12 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:12 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "21:12 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:13 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "21:13 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:14 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "21:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:15 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "21:15 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:16 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "21:16 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:16 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "21:16 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:17 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "21:17 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:18 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "21:18 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:18 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "21:18 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "21:18 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "21:18 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "21:18 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "21:18 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "21:18 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "21:18 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "21:18 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "21:18 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "21:18 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "21:18 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "21:18 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "21:18 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "21:18 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "21:18 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "21:18 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "21:18 madminer.ml          INFO    Starting training\n",
      "21:18 madminer.ml          INFO      Batch size:             128\n",
      "21:18 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "21:18 madminer.ml          INFO      Epochs:                 50\n",
      "21:18 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "21:18 madminer.ml          INFO      Validation split:       0.25\n",
      "21:18 madminer.ml          INFO      Early stopping:         True\n",
      "21:18 madminer.ml          INFO      Scale inputs:           True\n",
      "21:18 madminer.ml          INFO      Shuffle labels          False\n",
      "21:18 madminer.ml          INFO      Samples:                all\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21:18 madminer.ml          INFO    Loading training data\n",
      "21:18 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/x_train_score_4.npy into RAM\n",
      "21:18 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_mu_smeftsim_ptw/t_xz_train_score_4.npy into RAM\n",
      "21:18 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "21:18 madminer.ml          INFO    Rescaling inputs\n",
      "21:18 madminer.ml          INFO    Creating model\n",
      "21:18 madminer.ml          INFO    Training model\n",
      "21:18 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "21:19 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "21:19 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:20 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "21:20 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:21 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "21:21 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:21 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "21:21 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:22 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00020 (mse_score:  0.000)\n",
      "21:22 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:23 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00020 (mse_score:  0.000)\n",
      "21:23 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:24 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "21:24 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:25 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "21:25 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:25 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "21:25 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:26 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "21:26 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:27 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "21:27 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:28 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "21:28 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:28 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "21:28 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:29 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "21:29 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:30 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "21:30 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:31 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "21:31 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "21:31 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00018 compared to final loss  0.00018\n",
      "21:31 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "21:31 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "21:31 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "21:31 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "21:31 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "21:31 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "21:31 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "21:31 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "21:31 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "21:31 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "21:31 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "21:31 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "21:31 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "21:31 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "21:31 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "21:31 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_mu_smeftsim_ptw/estimator_0\n",
      "21:31 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_mu_smeftsim_ptw/estimator_1\n",
      "21:31 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_mu_smeftsim_ptw/estimator_2\n",
      "21:31 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_mu_smeftsim_ptw/estimator_3\n",
      "21:31 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_mu_smeftsim_ptw/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wmh_mu_smeftsim','ptw',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21:31 madminer.analysis    INFO    Loading data from data/met/signal/wmh_e_smeftsim_lhedata_met.h5\n",
      "21:31 madminer.analysis    INFO    Found 4 parameters\n",
      "21:31 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "21:31 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "21:31 madminer.analysis    INFO    Found 48 observables\n",
      "21:31 madminer.analysis    INFO    Found 1000000 events\n",
      "21:31 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "21:31 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "21:31 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "21:31 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21:31 madminer.sampling    INFO    Starting sampling serially\n",
      "21:31 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "21:32 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "21:32 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "21:32 madminer.sampling    INFO    Starting sampling serially\n",
      "21:32 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "21:32 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "21:32 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "21:32 madminer.sampling    INFO    Starting sampling serially\n",
      "21:32 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "21:32 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "21:32 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "21:32 madminer.sampling    INFO    Starting sampling serially\n",
      "21:32 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "21:32 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "21:32 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "21:32 madminer.sampling    INFO    Starting sampling serially\n",
      "21:32 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "21:32 madminer.sampling    INFO    Effective number of samples: mean 798420.881202, with individual thetas ranging from 798420.881202 to 798420.881202\n",
      "21:32 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21:32 madminer.sampling    INFO    Starting sampling serially\n",
      "21:32 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "21:33 madminer.sampling    INFO    Effective number of samples: mean 199827.145512, with individual thetas ranging from 199827.145512 to 199827.145512\n",
      "21:33 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "21:33 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "21:33 madminer.ml          INFO    Starting training\n",
      "21:33 madminer.ml          INFO      Batch size:             128\n",
      "21:33 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "21:33 madminer.ml          INFO      Epochs:                 50\n",
      "21:33 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "21:33 madminer.ml          INFO      Validation split:       0.25\n",
      "21:33 madminer.ml          INFO      Early stopping:         True\n",
      "21:33 madminer.ml          INFO      Scale inputs:           True\n",
      "21:33 madminer.ml          INFO      Shuffle labels          False\n",
      "21:33 madminer.ml          INFO      Samples:                all\n",
      "21:33 madminer.ml          INFO    Loading training data\n",
      "21:33 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/x_train_score_0.npy into RAM\n",
      "21:33 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21:33 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "21:33 madminer.ml          INFO    Rescaling inputs\n",
      "21:33 madminer.ml          INFO    Creating model\n",
      "21:33 madminer.ml          INFO    Training model\n",
      "21:33 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "21:33 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "21:33 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:34 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "21:34 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:35 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "21:35 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:36 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "21:36 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:36 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "21:36 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:37 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "21:37 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:38 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "21:38 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:39 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "21:39 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:39 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "21:39 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:40 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "21:40 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:41 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "21:41 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:42 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "21:42 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:43 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "21:43 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:43 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "21:43 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:44 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "21:44 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:45 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "21:45 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "21:45 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "21:45 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "21:45 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "21:45 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "21:45 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "21:45 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "21:45 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "21:45 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "21:45 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "21:45 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "21:45 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "21:45 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "21:45 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "21:45 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "21:45 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "21:45 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "21:45 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "21:45 madminer.ml          INFO    Starting training\n",
      "21:45 madminer.ml          INFO      Batch size:             128\n",
      "21:45 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "21:45 madminer.ml          INFO      Epochs:                 50\n",
      "21:45 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "21:45 madminer.ml          INFO      Validation split:       0.25\n",
      "21:45 madminer.ml          INFO      Early stopping:         True\n",
      "21:45 madminer.ml          INFO      Scale inputs:           True\n",
      "21:45 madminer.ml          INFO      Shuffle labels          False\n",
      "21:45 madminer.ml          INFO      Samples:                all\n",
      "21:45 madminer.ml          INFO    Loading training data\n",
      "21:45 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/x_train_score_1.npy into RAM\n",
      "21:45 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/t_xz_train_score_1.npy into RAM\n",
      "21:45 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "21:45 madminer.ml          INFO    Rescaling inputs\n",
      "21:45 madminer.ml          INFO    Creating model\n",
      "21:45 madminer.ml          INFO    Training model\n",
      "21:45 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "21:46 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "21:46 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:47 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "21:47 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:48 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "21:48 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:49 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "21:49 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:49 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "21:49 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:50 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "21:50 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:51 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "21:51 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:52 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "21:52 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:52 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "21:52 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:53 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "21:53 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21:54 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "21:54 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:55 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "21:55 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:55 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "21:55 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:56 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "21:56 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:57 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "21:57 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:58 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "21:58 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "21:58 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "21:58 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "21:58 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "21:58 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "21:58 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "21:58 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "21:58 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "21:58 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "21:58 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "21:58 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "21:58 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "21:58 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "21:58 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "21:58 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "21:58 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "21:58 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "21:58 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "21:58 madminer.ml          INFO    Starting training\n",
      "21:58 madminer.ml          INFO      Batch size:             128\n",
      "21:58 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "21:58 madminer.ml          INFO      Epochs:                 50\n",
      "21:58 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "21:58 madminer.ml          INFO      Validation split:       0.25\n",
      "21:58 madminer.ml          INFO      Early stopping:         True\n",
      "21:58 madminer.ml          INFO      Scale inputs:           True\n",
      "21:58 madminer.ml          INFO      Shuffle labels          False\n",
      "21:58 madminer.ml          INFO      Samples:                all\n",
      "21:58 madminer.ml          INFO    Loading training data\n",
      "21:58 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/x_train_score_2.npy into RAM\n",
      "21:58 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/t_xz_train_score_2.npy into RAM\n",
      "21:58 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "21:58 madminer.ml          INFO    Rescaling inputs\n",
      "21:58 madminer.ml          INFO    Creating model\n",
      "21:58 madminer.ml          INFO    Training model\n",
      "21:58 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "21:59 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "21:59 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:00 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "22:00 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:01 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "22:01 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:01 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "22:01 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:02 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00020 (mse_score:  0.000)\n",
      "22:02 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:03 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00020 (mse_score:  0.000)\n",
      "22:03 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:04 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00020 (mse_score:  0.000)\n",
      "22:04 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:05 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00020 (mse_score:  0.000)\n",
      "22:05 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:05 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00020 (mse_score:  0.000)\n",
      "22:05 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:06 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00020 (mse_score:  0.000)\n",
      "22:06 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:07 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00020 (mse_score:  0.000)\n",
      "22:07 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:08 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "22:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:08 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "22:08 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:09 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "22:09 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:10 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "22:10 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:11 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "22:11 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:11 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00019 compared to final loss  0.00019\n",
      "22:11 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "22:11 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "22:11 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "22:11 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22:11 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "22:11 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "22:11 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "22:11 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "22:11 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "22:11 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "22:11 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "22:11 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "22:11 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "22:11 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "22:11 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "22:11 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "22:11 madminer.ml          INFO    Starting training\n",
      "22:11 madminer.ml          INFO      Batch size:             128\n",
      "22:11 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "22:11 madminer.ml          INFO      Epochs:                 50\n",
      "22:11 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "22:11 madminer.ml          INFO      Validation split:       0.25\n",
      "22:11 madminer.ml          INFO      Early stopping:         True\n",
      "22:11 madminer.ml          INFO      Scale inputs:           True\n",
      "22:11 madminer.ml          INFO      Shuffle labels          False\n",
      "22:11 madminer.ml          INFO      Samples:                all\n",
      "22:11 madminer.ml          INFO    Loading training data\n",
      "22:11 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/x_train_score_3.npy into RAM\n",
      "22:11 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/t_xz_train_score_3.npy into RAM\n",
      "22:11 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "22:11 madminer.ml          INFO    Rescaling inputs\n",
      "22:11 madminer.ml          INFO    Creating model\n",
      "22:11 madminer.ml          INFO    Training model\n",
      "22:11 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "22:12 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "22:12 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:13 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "22:13 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:14 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "22:14 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:14 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "22:14 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:15 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "22:15 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:16 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00019 (mse_score:  0.000)\n",
      "22:16 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:17 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00019 (mse_score:  0.000)\n",
      "22:17 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:17 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00019 (mse_score:  0.000)\n",
      "22:17 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:18 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00019 (mse_score:  0.000)\n",
      "22:18 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:19 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00019 (mse_score:  0.000)\n",
      "22:19 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:20 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00019 (mse_score:  0.000)\n",
      "22:20 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:21 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00019 (mse_score:  0.000)\n",
      "22:21 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:21 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00019 (mse_score:  0.000)\n",
      "22:21 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:22 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00019 (mse_score:  0.000)\n",
      "22:22 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:23 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "22:23 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:24 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "22:24 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:24 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "22:24 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "22:24 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "22:24 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "22:24 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "22:24 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "22:24 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "22:24 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "22:24 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "22:24 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "22:24 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "22:24 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "22:24 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "22:24 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "22:24 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "22:24 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "22:24 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "22:24 madminer.ml          INFO    Starting training\n",
      "22:24 madminer.ml          INFO      Batch size:             128\n",
      "22:24 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "22:24 madminer.ml          INFO      Epochs:                 50\n",
      "22:24 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "22:24 madminer.ml          INFO      Validation split:       0.25\n",
      "22:24 madminer.ml          INFO      Early stopping:         True\n",
      "22:24 madminer.ml          INFO      Scale inputs:           True\n",
      "22:24 madminer.ml          INFO      Shuffle labels          False\n",
      "22:24 madminer.ml          INFO      Samples:                all\n",
      "22:24 madminer.ml          INFO    Loading training data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22:24 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/x_train_score_4.npy into RAM\n",
      "22:24 madminer.utils.vario INFO      Loading samples/ptw/samples_wmh_e_smeftsim_ptw/t_xz_train_score_4.npy into RAM\n",
      "22:24 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "22:24 madminer.ml          INFO    Rescaling inputs\n",
      "22:24 madminer.ml          INFO    Creating model\n",
      "22:24 madminer.ml          INFO    Training model\n",
      "22:24 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "22:25 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00020 (mse_score:  0.000)\n",
      "22:25 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:26 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00020 (mse_score:  0.000)\n",
      "22:26 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:26 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00020 (mse_score:  0.000)\n",
      "22:26 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:27 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00020 (mse_score:  0.000)\n",
      "22:27 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "22:28 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00020 (mse_score:  0.000)\n",
      "22:28 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:29 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00020 (mse_score:  0.000)\n",
      "22:29 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:30 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00020 (mse_score:  0.000)\n",
      "22:30 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:30 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00020 (mse_score:  0.000)\n",
      "22:30 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:31 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00020 (mse_score:  0.000)\n",
      "22:31 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:32 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00020 (mse_score:  0.000)\n",
      "22:32 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:33 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00020 (mse_score:  0.000)\n",
      "22:33 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:33 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00020 (mse_score:  0.000)\n",
      "22:33 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:34 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00020 (mse_score:  0.000)\n",
      "22:34 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:35 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00020 (mse_score:  0.000)\n",
      "22:35 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:36 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00019 (mse_score:  0.000)\n",
      "22:36 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:37 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00019 (mse_score:  0.000)\n",
      "22:37 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:37 madminer.utils.ml.tr INFO    Early stopping after epoch 49, with loss  0.00019 compared to final loss  0.00019\n",
      "22:37 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "22:37 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "22:37 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "22:37 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "22:37 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "22:37 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "22:37 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "22:37 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "22:37 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "22:37 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "22:37 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "22:37 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "22:37 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "22:37 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "22:37 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "22:37 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_e_smeftsim_ptw/estimator_0\n",
      "22:37 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_e_smeftsim_ptw/estimator_1\n",
      "22:37 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_e_smeftsim_ptw/estimator_2\n",
      "22:37 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_e_smeftsim_ptw/estimator_3\n",
      "22:37 madminer.ml          INFO    Saving model to models/ptw/sally_ensemble_wmh_e_smeftsim_ptw/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wmh_e_smeftsim','ptw',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### $p_{T,W}$ and $m_{T,\\mathrm{tot}}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22:37 madminer.analysis    INFO    Loading data from data/met/signal/wph_mu_smeftsim_lhedata_met.h5\n",
      "22:37 madminer.analysis    INFO    Found 4 parameters\n",
      "22:37 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "22:37 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "22:37 madminer.analysis    INFO    Found 48 observables\n",
      "22:37 madminer.analysis    INFO    Found 1000000 events\n",
      "22:37 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "22:37 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "22:37 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "22:37 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "22:37 madminer.sampling    INFO    Starting sampling serially\n",
      "22:37 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22:37 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:37 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "22:38 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "22:38 madminer.sampling    INFO    Starting sampling serially\n",
      "22:38 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "22:38 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "22:38 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "22:38 madminer.sampling    INFO    Starting sampling serially\n",
      "22:38 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "22:38 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "22:38 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "22:38 madminer.sampling    INFO    Starting sampling serially\n",
      "22:38 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "22:38 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "22:38 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "22:38 madminer.sampling    INFO    Starting sampling serially\n",
      "22:38 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "22:38 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "22:38 madminer.sampling    INFO    Effective number of samples: mean 797577.291162, with individual thetas ranging from 797577.291162 to 797577.291162\n",
      "22:38 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "22:38 madminer.sampling    INFO    Starting sampling serially\n",
      "22:38 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22:39 madminer.sampling    INFO    Effective number of samples: mean 199716.36363, with individual thetas ranging from 199716.36363 to 199716.36363\n",
      "22:39 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "22:39 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "22:39 madminer.ml          INFO    Starting training\n",
      "22:39 madminer.ml          INFO      Batch size:             128\n",
      "22:39 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "22:39 madminer.ml          INFO      Epochs:                 50\n",
      "22:39 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "22:39 madminer.ml          INFO      Validation split:       0.25\n",
      "22:39 madminer.ml          INFO      Early stopping:         True\n",
      "22:39 madminer.ml          INFO      Scale inputs:           True\n",
      "22:39 madminer.ml          INFO      Shuffle labels          False\n",
      "22:39 madminer.ml          INFO      Samples:                all\n",
      "22:39 madminer.ml          INFO    Loading training data\n",
      "22:39 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/x_train_score_0.npy into RAM\n",
      "22:39 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22:39 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "22:39 madminer.ml          INFO    Rescaling inputs\n",
      "22:39 madminer.ml          INFO    Creating model\n",
      "22:39 madminer.ml          INFO    Training model\n",
      "22:39 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "22:39 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "22:39 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:40 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "22:40 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:41 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "22:41 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:42 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "22:42 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:42 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "22:42 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:43 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "22:43 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:44 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "22:44 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:45 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "22:45 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:46 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "22:46 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:46 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "22:46 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:47 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "22:47 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:48 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "22:48 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:49 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "22:49 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:49 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "22:49 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:50 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "22:50 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:51 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "22:51 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "22:51 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "22:51 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "22:51 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "22:51 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "22:51 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "22:51 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "22:51 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "22:51 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "22:51 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "22:51 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "22:51 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "22:51 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "22:51 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "22:51 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "22:51 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "22:51 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "22:51 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "22:51 madminer.ml          INFO    Starting training\n",
      "22:51 madminer.ml          INFO      Batch size:             128\n",
      "22:51 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "22:51 madminer.ml          INFO      Epochs:                 50\n",
      "22:51 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "22:51 madminer.ml          INFO      Validation split:       0.25\n",
      "22:51 madminer.ml          INFO      Early stopping:         True\n",
      "22:51 madminer.ml          INFO      Scale inputs:           True\n",
      "22:51 madminer.ml          INFO      Shuffle labels          False\n",
      "22:51 madminer.ml          INFO      Samples:                all\n",
      "22:51 madminer.ml          INFO    Loading training data\n",
      "22:51 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/x_train_score_1.npy into RAM\n",
      "22:51 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/t_xz_train_score_1.npy into RAM\n",
      "22:51 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "22:51 madminer.ml          INFO    Rescaling inputs\n",
      "22:51 madminer.ml          INFO    Creating model\n",
      "22:51 madminer.ml          INFO    Training model\n",
      "22:51 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "22:52 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "22:52 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:53 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "22:53 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "22:54 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "22:54 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:55 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "22:55 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:55 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00019 (mse_score:  0.000)\n",
      "22:55 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:56 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "22:56 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:57 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "22:57 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:58 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "22:58 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:58 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "22:58 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "22:59 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "22:59 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23:00 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "23:00 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:01 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "23:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:02 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "23:02 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:02 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "23:02 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:03 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "23:03 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:04 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "23:04 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:04 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "23:04 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "23:04 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "23:04 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "23:04 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "23:04 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "23:04 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "23:04 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "23:04 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "23:04 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "23:04 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "23:04 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "23:04 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "23:04 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "23:04 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "23:04 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "23:04 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "23:04 madminer.ml          INFO    Starting training\n",
      "23:04 madminer.ml          INFO      Batch size:             128\n",
      "23:04 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "23:04 madminer.ml          INFO      Epochs:                 50\n",
      "23:04 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "23:04 madminer.ml          INFO      Validation split:       0.25\n",
      "23:04 madminer.ml          INFO      Early stopping:         True\n",
      "23:04 madminer.ml          INFO      Scale inputs:           True\n",
      "23:04 madminer.ml          INFO      Shuffle labels          False\n",
      "23:04 madminer.ml          INFO      Samples:                all\n",
      "23:04 madminer.ml          INFO    Loading training data\n",
      "23:04 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/x_train_score_2.npy into RAM\n",
      "23:04 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/t_xz_train_score_2.npy into RAM\n",
      "23:04 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "23:04 madminer.ml          INFO    Rescaling inputs\n",
      "23:04 madminer.ml          INFO    Creating model\n",
      "23:04 madminer.ml          INFO    Training model\n",
      "23:04 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "23:05 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "23:05 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:06 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "23:06 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:07 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00019 (mse_score:  0.000)\n",
      "23:07 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:07 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00019 (mse_score:  0.000)\n",
      "23:07 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:08 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "23:08 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:09 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "23:09 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:10 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "23:10 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:11 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "23:11 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:11 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "23:11 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:12 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "23:12 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:13 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "23:13 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:14 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "23:14 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:14 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "23:14 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:15 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "23:15 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:16 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "23:16 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:17 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "23:17 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "23:17 madminer.utils.ml.tr INFO    Early stopping after epoch 47, with loss  0.00017 compared to final loss  0.00017\n",
      "23:17 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "23:17 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "23:17 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "23:17 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23:17 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "23:17 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "23:17 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "23:17 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "23:17 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "23:17 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "23:17 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "23:17 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "23:17 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "23:17 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "23:17 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "23:17 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "23:17 madminer.ml          INFO    Starting training\n",
      "23:17 madminer.ml          INFO      Batch size:             128\n",
      "23:17 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "23:17 madminer.ml          INFO      Epochs:                 50\n",
      "23:17 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "23:17 madminer.ml          INFO      Validation split:       0.25\n",
      "23:17 madminer.ml          INFO      Early stopping:         True\n",
      "23:17 madminer.ml          INFO      Scale inputs:           True\n",
      "23:17 madminer.ml          INFO      Shuffle labels          False\n",
      "23:17 madminer.ml          INFO      Samples:                all\n",
      "23:17 madminer.ml          INFO    Loading training data\n",
      "23:17 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/x_train_score_3.npy into RAM\n",
      "23:17 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/t_xz_train_score_3.npy into RAM\n",
      "23:17 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "23:17 madminer.ml          INFO    Rescaling inputs\n",
      "23:17 madminer.ml          INFO    Creating model\n",
      "23:17 madminer.ml          INFO    Training model\n",
      "23:17 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "23:18 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "23:18 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:19 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "23:19 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:20 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "23:20 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:20 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "23:20 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:21 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "23:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:22 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "23:22 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:23 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "23:23 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:23 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "23:23 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:24 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "23:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:25 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "23:25 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:26 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "23:26 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:27 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "23:27 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:27 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "23:27 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:28 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "23:28 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:29 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "23:29 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:30 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "23:30 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:30 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "23:30 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "23:30 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                                   ALL:   0.22h\n",
      "23:30 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "23:30 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "23:30 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "23:30 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "23:30 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "23:30 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "23:30 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "23:30 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "23:30 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "23:30 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "23:30 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "23:30 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "23:30 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "23:30 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "23:30 madminer.ml          INFO    Starting training\n",
      "23:30 madminer.ml          INFO      Batch size:             128\n",
      "23:30 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "23:30 madminer.ml          INFO      Epochs:                 50\n",
      "23:30 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "23:30 madminer.ml          INFO      Validation split:       0.25\n",
      "23:30 madminer.ml          INFO      Early stopping:         True\n",
      "23:30 madminer.ml          INFO      Scale inputs:           True\n",
      "23:30 madminer.ml          INFO      Shuffle labels          False\n",
      "23:30 madminer.ml          INFO      Samples:                all\n",
      "23:30 madminer.ml          INFO    Loading training data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23:30 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/x_train_score_4.npy into RAM\n",
      "23:30 madminer.utils.vario INFO      Loading samples/2d/samples_wph_mu_smeftsim_2d/t_xz_train_score_4.npy into RAM\n",
      "23:30 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "23:30 madminer.ml          INFO    Rescaling inputs\n",
      "23:30 madminer.ml          INFO    Creating model\n",
      "23:30 madminer.ml          INFO    Training model\n",
      "23:30 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "23:31 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "23:31 madminer.utils.ml.tr INFO               val. loss   0.00020 (mse_score:  0.000)\n",
      "23:32 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "23:32 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:33 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "23:33 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:33 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "23:33 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:34 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "23:34 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:35 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "23:35 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:36 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "23:36 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:36 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "23:36 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:37 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "23:37 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:38 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "23:38 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:39 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "23:39 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:40 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "23:40 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:40 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "23:40 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:41 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "23:41 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:42 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "23:42 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:43 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "23:43 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:43 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "23:43 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "23:43 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "23:43 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "23:43 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "23:43 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "23:43 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "23:43 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "23:43 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "23:43 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "23:43 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "23:43 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "23:43 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "23:43 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "23:43 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "23:43 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "23:43 madminer.ml          INFO    Saving model to models/2d/sally_ensemble_wph_mu_smeftsim_2d/estimator_0\n",
      "23:43 madminer.ml          INFO    Saving model to models/2d/sally_ensemble_wph_mu_smeftsim_2d/estimator_1\n",
      "23:43 madminer.ml          INFO    Saving model to models/2d/sally_ensemble_wph_mu_smeftsim_2d/estimator_2\n",
      "23:43 madminer.ml          INFO    Saving model to models/2d/sally_ensemble_wph_mu_smeftsim_2d/estimator_3\n",
      "23:43 madminer.ml          INFO    Saving model to models/2d/sally_ensemble_wph_mu_smeftsim_2d/estimator_4\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wph_mu_smeftsim','2d',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23:43 madminer.analysis    INFO    Loading data from data/met/signal/wph_e_smeftsim_lhedata_met.h5\n",
      "23:43 madminer.analysis    INFO    Found 4 parameters\n",
      "23:43 madminer.analysis    INFO    Found 33 nuisance parameters\n",
      "23:43 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical\n",
      "23:43 madminer.analysis    INFO    Found 48 observables\n",
      "23:43 madminer.analysis    INFO    Found 1000000 events\n",
      "23:43 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm\n",
      "23:43 madminer.analysis    INFO    Found morphing setup with 15 components\n",
      "23:43 madminer.analysis    INFO    Found nuisance morphing setup\n",
      "23:43 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Training Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23:43 madminer.sampling    INFO    Starting sampling serially\n",
      "23:43 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "23:44 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "23:44 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "23:44 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "23:44 madminer.sampling    INFO    Starting sampling serially\n",
      "23:44 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "23:44 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "23:44 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "23:44 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "23:44 madminer.sampling    INFO    Starting sampling serially\n",
      "23:44 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "23:44 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "23:44 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "23:44 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "23:44 madminer.sampling    INFO    Starting sampling serially\n",
      "23:44 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "23:44 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "23:44 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "23:44 madminer.sampling    INFO    Extracting training sample for local score regression. Sampling and score evaluation according to sm\n",
      "23:44 madminer.sampling    INFO    Starting sampling serially\n",
      "23:44 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n",
      "23:44 madminer.sampling    WARNING For this value of theta, 1 / 100000 events have negative weight and will be ignored\n",
      "23:44 madminer.sampling    INFO    Effective number of samples: mean 797579.2814, with individual thetas ranging from 797579.2814 to 797579.2814\n",
      "23:44 madminer.sampling    INFO    Extracting evaluation sample. Sampling according to sm\n",
      "23:44 madminer.sampling    INFO    Starting sampling serially\n",
      "23:44 madminer.sampling    INFO    Sampling from parameter point 1 / 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Testing Samples...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23:45 madminer.sampling    INFO    Effective number of samples: mean 199716.36363, with individual thetas ranging from 199716.36363 to 199716.36363\n",
      "23:45 madminer.ml          INFO    Training 5 estimators in ensemble\n",
      "23:45 madminer.ml          INFO    Training estimator 1 / 5 in ensemble\n",
      "23:45 madminer.ml          INFO    Starting training\n",
      "23:45 madminer.ml          INFO      Batch size:             128\n",
      "23:45 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "23:45 madminer.ml          INFO      Epochs:                 50\n",
      "23:45 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "23:45 madminer.ml          INFO      Validation split:       0.25\n",
      "23:45 madminer.ml          INFO      Early stopping:         True\n",
      "23:45 madminer.ml          INFO      Scale inputs:           True\n",
      "23:45 madminer.ml          INFO      Shuffle labels          False\n",
      "23:45 madminer.ml          INFO      Samples:                all\n",
      "23:45 madminer.ml          INFO    Loading training data\n",
      "23:45 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/x_train_score_0.npy into RAM\n",
      "23:45 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/t_xz_train_score_0.npy into RAM\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Ensemble...\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23:45 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "23:45 madminer.ml          INFO    Rescaling inputs\n",
      "23:45 madminer.ml          INFO    Creating model\n",
      "23:45 madminer.ml          INFO    Training model\n",
      "23:45 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "23:45 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "23:45 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:46 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "23:46 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:47 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "23:47 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:48 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "23:48 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:49 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "23:49 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:49 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "23:49 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:50 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "23:50 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:51 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "23:51 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:52 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "23:52 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:52 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "23:52 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "23:53 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "23:53 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:54 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "23:54 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:55 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "23:55 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:56 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "23:56 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:56 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "23:56 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:57 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "23:57 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:58 madminer.utils.ml.tr INFO    Early stopping after epoch 47, with loss  0.00018 compared to final loss  0.00018\n",
      "23:58 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "23:58 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "23:58 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "23:58 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "23:58 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "23:58 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "23:58 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "23:58 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "23:58 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "23:58 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "23:58 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "23:58 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "23:58 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "23:58 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "23:58 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "23:58 madminer.ml          INFO    Training estimator 2 / 5 in ensemble\n",
      "23:58 madminer.ml          INFO    Starting training\n",
      "23:58 madminer.ml          INFO      Batch size:             128\n",
      "23:58 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "23:58 madminer.ml          INFO      Epochs:                 50\n",
      "23:58 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "23:58 madminer.ml          INFO      Validation split:       0.25\n",
      "23:58 madminer.ml          INFO      Early stopping:         True\n",
      "23:58 madminer.ml          INFO      Scale inputs:           True\n",
      "23:58 madminer.ml          INFO      Shuffle labels          False\n",
      "23:58 madminer.ml          INFO      Samples:                all\n",
      "23:58 madminer.ml          INFO    Loading training data\n",
      "23:58 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/x_train_score_1.npy into RAM\n",
      "23:58 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/t_xz_train_score_1.npy into RAM\n",
      "23:58 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "23:58 madminer.ml          INFO    Rescaling inputs\n",
      "23:58 madminer.ml          INFO    Creating model\n",
      "23:58 madminer.ml          INFO    Training model\n",
      "23:58 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "23:58 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "23:58 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "23:59 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "23:59 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:00 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "00:00 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:01 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "00:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:01 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "00:01 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:02 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "00:02 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:03 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "00:03 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:04 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "00:04 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:05 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "00:05 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:05 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "00:05 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:06 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "00:06 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:07 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "00:07 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:08 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "00:08 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:08 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "00:08 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:09 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "00:09 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:10 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "00:10 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:11 madminer.utils.ml.tr INFO    Early stopping did not improve performance\n",
      "00:11 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "00:11 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "00:11 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "00:11 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "00:11 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "00:11 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "00:11 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "00:11 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "00:11 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "00:11 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "00:11 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "00:11 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "00:11 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "00:11 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "00:11 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "00:11 madminer.ml          INFO    Training estimator 3 / 5 in ensemble\n",
      "00:11 madminer.ml          INFO    Starting training\n",
      "00:11 madminer.ml          INFO      Batch size:             128\n",
      "00:11 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "00:11 madminer.ml          INFO      Epochs:                 50\n",
      "00:11 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "00:11 madminer.ml          INFO      Validation split:       0.25\n",
      "00:11 madminer.ml          INFO      Early stopping:         True\n",
      "00:11 madminer.ml          INFO      Scale inputs:           True\n",
      "00:11 madminer.ml          INFO      Shuffle labels          False\n",
      "00:11 madminer.ml          INFO      Samples:                all\n",
      "00:11 madminer.ml          INFO    Loading training data\n",
      "00:11 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/x_train_score_2.npy into RAM\n",
      "00:11 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/t_xz_train_score_2.npy into RAM\n",
      "00:11 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "00:11 madminer.ml          INFO    Rescaling inputs\n",
      "00:11 madminer.ml          INFO    Creating model\n",
      "00:11 madminer.ml          INFO    Training model\n",
      "00:11 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "00:11 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "00:11 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "00:12 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00019 (mse_score:  0.000)\n",
      "00:12 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "00:13 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "00:13 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "00:14 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "00:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "00:14 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "00:14 madminer.utils.ml.tr INFO               val. loss   0.00019 (mse_score:  0.000)\n",
      "00:15 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00018 (mse_score:  0.000)\n",
      "00:15 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:16 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00018 (mse_score:  0.000)\n",
      "00:16 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:17 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00018 (mse_score:  0.000)\n",
      "00:17 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:17 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00018 (mse_score:  0.000)\n",
      "00:17 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:18 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00018 (mse_score:  0.000)\n",
      "00:18 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:19 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00018 (mse_score:  0.000)\n",
      "00:19 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:20 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00018 (mse_score:  0.000)\n",
      "00:20 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:21 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00018 (mse_score:  0.000)\n",
      "00:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:21 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00018 (mse_score:  0.000)\n",
      "00:21 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:22 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00018 (mse_score:  0.000)\n",
      "00:22 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:23 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00018 (mse_score:  0.000)\n",
      "00:23 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:23 madminer.utils.ml.tr INFO    Early stopping after epoch 48, with loss  0.00018 compared to final loss  0.00018\n",
      "00:23 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "00:23 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "00:23 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "00:23 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "00:23 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "00:23 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "00:23 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "00:23 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "00:23 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "00:23 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "00:23 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "00:23 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "00:23 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "00:23 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "00:23 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "00:23 madminer.ml          INFO    Training estimator 4 / 5 in ensemble\n",
      "00:23 madminer.ml          INFO    Starting training\n",
      "00:23 madminer.ml          INFO      Batch size:             128\n",
      "00:23 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "00:23 madminer.ml          INFO      Epochs:                 50\n",
      "00:23 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "00:23 madminer.ml          INFO      Validation split:       0.25\n",
      "00:23 madminer.ml          INFO      Early stopping:         True\n",
      "00:23 madminer.ml          INFO      Scale inputs:           True\n",
      "00:23 madminer.ml          INFO      Shuffle labels          False\n",
      "00:23 madminer.ml          INFO      Samples:                all\n",
      "00:23 madminer.ml          INFO    Loading training data\n",
      "00:23 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/x_train_score_3.npy into RAM\n",
      "00:23 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/t_xz_train_score_3.npy into RAM\n",
      "00:23 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "00:23 madminer.ml          INFO    Rescaling inputs\n",
      "00:23 madminer.ml          INFO    Creating model\n",
      "00:23 madminer.ml          INFO    Training model\n",
      "00:23 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "00:24 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00018 (mse_score:  0.000)\n",
      "00:24 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:25 madminer.utils.ml.tr INFO    Epoch   6: train loss  0.00018 (mse_score:  0.000)\n",
      "00:25 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:26 madminer.utils.ml.tr INFO    Epoch   9: train loss  0.00018 (mse_score:  0.000)\n",
      "00:26 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:27 madminer.utils.ml.tr INFO    Epoch  12: train loss  0.00018 (mse_score:  0.000)\n",
      "00:27 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:27 madminer.utils.ml.tr INFO    Epoch  15: train loss  0.00018 (mse_score:  0.000)\n",
      "00:27 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n",
      "00:28 madminer.utils.ml.tr INFO    Epoch  18: train loss  0.00017 (mse_score:  0.000)\n",
      "00:28 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:29 madminer.utils.ml.tr INFO    Epoch  21: train loss  0.00017 (mse_score:  0.000)\n",
      "00:29 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:30 madminer.utils.ml.tr INFO    Epoch  24: train loss  0.00017 (mse_score:  0.000)\n",
      "00:30 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:30 madminer.utils.ml.tr INFO    Epoch  27: train loss  0.00017 (mse_score:  0.000)\n",
      "00:30 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:31 madminer.utils.ml.tr INFO    Epoch  30: train loss  0.00017 (mse_score:  0.000)\n",
      "00:31 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:32 madminer.utils.ml.tr INFO    Epoch  33: train loss  0.00017 (mse_score:  0.000)\n",
      "00:32 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:33 madminer.utils.ml.tr INFO    Epoch  36: train loss  0.00017 (mse_score:  0.000)\n",
      "00:33 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:33 madminer.utils.ml.tr INFO    Epoch  39: train loss  0.00017 (mse_score:  0.000)\n",
      "00:33 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:34 madminer.utils.ml.tr INFO    Epoch  42: train loss  0.00017 (mse_score:  0.000)\n",
      "00:34 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:35 madminer.utils.ml.tr INFO    Epoch  45: train loss  0.00017 (mse_score:  0.000)\n",
      "00:35 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:36 madminer.utils.ml.tr INFO    Epoch  48: train loss  0.00017 (mse_score:  0.000)\n",
      "00:36 madminer.utils.ml.tr INFO               val. loss   0.00017 (mse_score:  0.000)\n",
      "00:36 madminer.utils.ml.tr INFO    Early stopping after epoch 47, with loss  0.00017 compared to final loss  0.00017\n",
      "00:36 madminer.utils.ml.tr INFO    Training time spend on:\n",
      "00:36 madminer.utils.ml.tr INFO                      initialize model:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                                   ALL:   0.21h\n",
      "00:36 madminer.utils.ml.tr INFO                            check data:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                          make dataset:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                       make dataloader:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                       setup optimizer:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                   initialize training:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                                set lr:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                   load training batch:   0.05h\n",
      "00:36 madminer.utils.ml.tr INFO                        fwd: move data:   0.01h\n",
      "00:36 madminer.utils.ml.tr INFO                   fwd: check for nans:   0.02h\n",
      "00:36 madminer.utils.ml.tr INFO                    fwd: model.forward:   0.02h\n",
      "00:36 madminer.utils.ml.tr INFO                 fwd: calculate losses:   0.01h\n",
      "00:36 madminer.utils.ml.tr INFO                 training forward pass:   0.04h\n",
      "00:36 madminer.utils.ml.tr INFO                   training sum losses:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                        opt: zero grad:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                         opt: backward:   0.08h\n",
      "00:36 madminer.utils.ml.tr INFO                   opt: clip grad norm:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                             opt: step:   0.01h\n",
      "00:36 madminer.utils.ml.tr INFO                        optimizer step:   0.09h\n",
      "00:36 madminer.utils.ml.tr INFO                 load validation batch:   0.02h\n",
      "00:36 madminer.utils.ml.tr INFO               validation forward pass:   0.01h\n",
      "00:36 madminer.utils.ml.tr INFO                 validation sum losses:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                        early stopping:   0.00h\n",
      "00:36 madminer.utils.ml.tr INFO                          report epoch:   0.00h\n",
      "00:36 madminer.ml          INFO    Training estimator 5 / 5 in ensemble\n",
      "00:36 madminer.ml          INFO    Starting training\n",
      "00:36 madminer.ml          INFO      Batch size:             128\n",
      "00:36 madminer.ml          INFO      Optimizer:              amsgrad\n",
      "00:36 madminer.ml          INFO      Epochs:                 50\n",
      "00:36 madminer.ml          INFO      Learning rate:          0.001 initially, decaying to 0.0001\n",
      "00:36 madminer.ml          INFO      Validation split:       0.25\n",
      "00:36 madminer.ml          INFO      Early stopping:         True\n",
      "00:36 madminer.ml          INFO      Scale inputs:           True\n",
      "00:36 madminer.ml          INFO      Shuffle labels          False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "00:36 madminer.ml          INFO      Samples:                all\n",
      "00:36 madminer.ml          INFO    Loading training data\n",
      "00:36 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/x_train_score_4.npy into RAM\n",
      "00:36 madminer.utils.vario INFO      Loading samples/2d/samples_wph_e_smeftsim_2d/t_xz_train_score_4.npy into RAM\n",
      "00:36 madminer.ml          INFO    Found 500000 samples with 37 parameters and 48 observables\n",
      "00:36 madminer.ml          INFO    Rescaling inputs\n",
      "00:36 madminer.ml          INFO    Creating model\n",
      "00:36 madminer.ml          INFO    Training model\n",
      "00:36 madminer.utils.ml.tr INFO    Training on CPU with single precision\n",
      "00:37 madminer.utils.ml.tr INFO    Epoch   3: train loss  0.00019 (mse_score:  0.000)\n",
      "00:37 madminer.utils.ml.tr INFO               val. loss   0.00018 (mse_score:  0.000)\n"
     ]
    }
   ],
   "source": [
    "augment_and_train('wph_e_smeftsim','2d',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_smeftsim','2d',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_smeftsim','2d',20*50000,is_signal_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Runs with Systematics on Signal Only"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Full Observables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_mu_wbkgs_sigsystonly','full',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_e_wbkgs_sigsystonly','full',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_wbkgs_sigsystonly','full',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_wbkgs_sigsystonly','full',100*50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### MET Observables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_mu_wbkgs_sigsystonly','met',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_e_wbkgs_sigsystonly','met',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_wbkgs_sigsystonly','met',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_wbkgs_sigsystonly','met',100*50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### $p_{T,W}$ Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_mu_wbkgs_sigsystonly','ptw',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_e_wbkgs_sigsystonly','ptw',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_wbkgs_sigsystonly','ptw',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_wbkgs_sigsystonly','ptw',100*50000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### $p_{T,W}$ and $m_{T,\\mathrm{tot}}$ Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_mu_wbkgs_sigsystonly','2d',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wph_e_wbkgs_sigsystonly','2d',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_mu_wbkgs_sigsystonly','2d',100*50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "augment_and_train('wmh_e_wbkgs_sigsystonly','2d',100*50000)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}