release

Author: ghliu

.gitignore

@@ -127,3 +127,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# for this repo
+data/*
+result/*

assets/result.png

@@ -0,0 +1 @@
+from .loader import get_uea_loader, get_tabular_loader, get_img_loader

datasets/__init__.py

@@ -0,0 +1,143 @@
+import numpy as np
+import torch
+
+from torchvision import transforms
+from prefetch_generator import BackgroundGenerator
+import util
+
+import torchvision.datasets as torch_data
+from .time_series import uea as uea_data
+from .tabular import maf as maf_data
+
+
+def _gen_mini_dataset(dataset, dataset_ratio):
+    n_dataset = dataset.shape[0]
+    n_mini_dataset = int(dataset_ratio*n_dataset)
+    s = torch.from_numpy(np.random.choice(
+        np.arange(n_dataset, dtype=np.int64), n_mini_dataset, replace=False)
+    )
+    return dataset[s]
+
+
+class DataLoaderX(torch.utils.data.DataLoader):
+    def __iter__(self):
+        return BackgroundGenerator(super().__iter__())
+
+
+class TabularLoader:
+    def __init__(self,opt, data, batch_size=None, shuffle=True):
+
+        self.data_size = data.shape[0]
+        self.opt = opt
+        self.device = opt.device
+
+        self.data = data.to(opt.device)
+        self.batch_size = opt.batch_size if batch_size is None else batch_size
+        self.shuffle = shuffle
+
+        self.input_dim = data.shape[-1]
+        self.output_dim= [data.shape[-1]]
+
+        loc = torch.zeros(data.shape[-1]).to(opt.device)
+        covariance_matrix = torch.eye(data.shape[-1]).to(opt.device) # TODO(Guan) scale down the cov ?
+        self.p_z0 = torch.distributions.MultivariateNormal(loc=loc, covariance_matrix=covariance_matrix)
+        self._reset_idxs()
+        self.data_size = len(self.idxs_by_batch_size)
+
+    def _reset_idxs(self):
+        idxs = torch.randperm(self.data.shape[0]) if self.shuffle else torch.arange(self.data.shape[0])
+        self.idxs_by_batch_size = idxs.split(self.batch_size)
+        self.batch_idx = 0
+
+    def __len__(self):
+        return self.data_size
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self.batch_idx >= len(self.idxs_by_batch_size):
+            self._reset_idxs()
+            raise StopIteration
+
+        s = self.idxs_by_batch_size[self.batch_idx]
+        self.batch_idx += 1
+        x = self.data[s]
+        logp_diff_t1 = torch.zeros(x.shape[0], 1, device=x.device)
+        return (x, logp_diff_t1), self.p_z0
+
+
+def get_uea_loader(opt):
+
+    print(util.magenta("loading uea data..."))
+
+    dataset_name = {
+        'CharT' :'CharacterTrajectories',
+        'ArtWR' : 'ArticularyWordRecognition',
+        'SpoAD' :     'SpokenArabicDigits',
+    }.get(opt.problem)
+
+    missing_rate = 0.0
+    device = opt.device
+    intensity_data = True
+
+    (times, train_dataloader, val_dataloader,
+     test_dataloader, num_classes, input_channels) = uea_data.get_data(dataset_name, missing_rate, device,
+                                                                           intensity=intensity_data,
+                                                                           batch_size=opt.batch_size)
+
+    # we'll return dataloader and store the rest in opt
+    opt.times = times
+    opt.output_dim = num_classes
+    opt.input_dim = input_channels
+    return train_dataloader, test_dataloader
+
+
+def get_tabular_loader(opt, test_batch_size=1000):
+    assert opt.problem in ['gas', 'miniboone']
+    print(util.magenta("loading tabular data..."))
+
+    data = maf_data.get_data(opt.problem)
+    data.trn.x = torch.from_numpy(data.trn.x)
+    data.val.x = torch.from_numpy(data.val.x)
+    data.tst.x = torch.from_numpy(data.tst.x)
+
+    if opt.dataset_ratio < 1.0:
+        data.trn.x = _gen_mini_dataset(data.trn.x, opt.dataset_ratio)
+        data.val.x = _gen_mini_dataset(data.val.x, opt.dataset_ratio)
+        data.tst.x = _gen_mini_dataset(data.tst.x, opt.dataset_ratio)
+
+    train_loader = TabularLoader(opt, data.trn.x, shuffle=True)
+    val_loader   = TabularLoader(opt, data.val.x, batch_size=test_batch_size, shuffle=False)
+    test_loader  = TabularLoader(opt, data.tst.x, batch_size=test_batch_size, shuffle=False)
+
+    opt.input_dim = train_loader.input_dim
+    opt.output_dim = train_loader.output_dim
+
+    return train_loader, test_loader
+
+
+def get_img_loader(opt, test_batch_size=1000):
+    print(util.magenta("loading image data..."))
+
+    dataset_builder, root, input_dim, output_dim = {
+        'mnist':   [torch_data.MNIST,  'data/img/mnist',  [1,28,28], 10],
+        'SVHN':    [torch_data.SVHN,   'data/img/svhn',   [3,32,32], 10],
+        'cifar10': [torch_data.CIFAR10,'data/img/cifar10',[3,32,32], 10],
+    }.get(opt.problem)
+    opt.input_dim = input_dim
+    opt.output_dim = output_dim
+
+    transform = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.1307,), (0.3081,)),
+    ])
+    feed_dict = dict(download=True, root=root, transform=transform)
+    train_dataset = dataset_builder(**feed_dict) if opt.problem=='SVHN' else dataset_builder(train=True, **feed_dict)
+    test_dataset  = dataset_builder(**feed_dict) if opt.problem=='SVHN' else dataset_builder(train=False, **feed_dict)
+
+    feed_dict = dict(num_workers=2, drop_last=True)
+    train_loader = DataLoaderX(train_dataset, batch_size=opt.batch_size, shuffle=True, **feed_dict)
+    test_loader  = DataLoaderX(test_dataset, batch_size=test_batch_size, shuffle=False, **feed_dict)
+
+    return train_loader, test_loader

datasets/loader.py

@@ -0,0 +1,70 @@
+import pandas as pd
+import numpy as np
+
+
+class GAS:
+
+    class Data:
+
+        def __init__(self, data):
+
+            self.x = data.astype(np.float32)
+            self.N = self.x.shape[0]
+
+    def __init__(self, path):
+
+        file = path / 'ethylene_CO.pickle'
+        trn, val, tst = load_data_and_clean_and_split(file)
+
+        self.trn = self.Data(trn)
+        self.val = self.Data(val)
+        self.tst = self.Data(tst)
+
+        self.n_dims = self.trn.x.shape[1]
+
+
+def load_data(file):
+
+    data = pd.read_pickle(file)
+    # data = pd.read_pickle(file).sample(frac=0.25)
+    # data.to_pickle(file)
+    data.drop("Meth", axis=1, inplace=True)
+    data.drop("Eth", axis=1, inplace=True)
+    data.drop("Time", axis=1, inplace=True)
+    return data
+
+
+def get_correlation_numbers(data):
+    C = data.corr()
+    A = C > 0.98
+    B = A.to_numpy().sum(axis=1)
+    return B
+
+
+def load_data_and_clean(file):
+
+    data = load_data(file)
+    B = get_correlation_numbers(data)
+
+    while np.any(B > 1):
+        col_to_remove = np.where(B > 1)[0][0]
+        col_name = data.columns[col_to_remove]
+        data.drop(col_name, axis=1, inplace=True)
+        B = get_correlation_numbers(data)
+    # print(data.corr())
+    data = (data - data.mean()) / data.std()
+
+    return data
+
+
+def load_data_and_clean_and_split(file):
+
+    data = load_data_and_clean(file).to_numpy()
+    N_test = int(0.1 * data.shape[0])
+    data_test = data[-N_test:]
+    data_train = data[0:-N_test]
+    N_validate = int(0.1 * data_train.shape[0])
+    data_validate = data_train[-N_validate:]
+    data_train = data_train[0:-N_validate]
+
+    return data_train, data_validate, data_test

datasets/tabular/__init__.py

@@ -0,0 +1,63 @@
+import collections as co
+import numpy as np
+import os
+import pathlib
+import sktime.utils.load_data
+import torch
+import urllib.request
+import tarfile
+
+from .gas import GAS
+from .miniboone import MINIBOONE
+
+
+here = pathlib.Path(__file__).resolve().parent.parent.parent
+
+def download():
+    base_base_loc = here / 'data'
+    base_loc = base_base_loc / 'maf'
+    loc = base_loc / 'maf.tar.gz'
+    if os.path.exists(loc):
+        return
+    if not os.path.exists(base_base_loc):
+        os.mkdir(base_base_loc)
+    if not os.path.exists(base_loc):
+        os.mkdir(base_loc)
+
+    print('download from https://zenodo.org/record/1161203/files/data.tar.gz .....')
+    urllib.request.urlretrieve('https://zenodo.org/record/1161203/files/data.tar.gz',
+                               str(loc))
+
+    def gas(tar):
+        l = len("data/")
+        for member in tar.getmembers():
+            if member.path.startswith("data/gas"):
+                member.path = member.path[l:]
+                yield member
+
+    def miniboone(tar):
+        l = len("data/")
+        for member in tar.getmembers():
+            if member.path.startswith("data/miniboone"):
+                member.path = member.path[l:]
+                yield member
+
+    with tarfile.open(loc, "r:gz") as tar:
+        # tar.extractall(path=base_loc) # <---- TODO(Guan) use this if you wish to extract all datasets.
+        tar.extractall(path=base_loc, members=gas(tar))
+        tar.extractall(path=base_loc, members=miniboone(tar))
+
+def get_data(dataset_name):
+
+    base_base_loc = here / 'data'
+    base_loc = base_base_loc / 'maf'
+    loc = base_loc / dataset_name
+
+    if not os.path.exists(loc):
+        download()
+
+    return {
+        'gas': GAS,
+        'miniboone': MINIBOONE
+    }.get(dataset_name)(loc)
+

datasets/tabular/gas.py

@@ -0,0 +1,67 @@
+import numpy as np
+
+
+class MINIBOONE:
+
+    class Data:
+
+        def __init__(self, data):
+
+            self.x = data.astype(np.float32)
+            self.N = self.x.shape[0]
+
+    def __init__(self, path):
+
+        file = path / 'data.npy'
+        trn, val, tst = load_data_normalised(file)
+
+        self.trn = self.Data(trn)
+        self.val = self.Data(val)
+        self.tst = self.Data(tst)
+
+        self.n_dims = self.trn.x.shape[1]
+
+
+def load_data(root_path):
+    # NOTE: To remember how the pre-processing was done.
+    # data = pd.read_csv(root_path, names=[str(x) for x in range(50)], delim_whitespace=True)
+    # print data.head()
+    # data = data.to_numpy()
+    # # Remove some random outliers
+    # indices = (data[:, 0] < -100)
+    # data = data[~indices]
+    #
+    # i = 0
+    # # Remove any features that have too many re-occuring real values.
+    # features_to_remove = []
+    # for feature in data.T:
+    #     c = Counter(feature)
+    #     max_count = np.array([v for k, v in sorted(c.iteritems())])[0]
+    #     if max_count > 5:
+    #         features_to_remove.append(i)
+    #     i += 1
+    # data = data[:, np.array([i for i in range(data.shape[1]) if i not in features_to_remove])]
+    # np.save("~/data/miniboone/data.npy", data)
+
+    data = np.load(root_path)
+    N_test = int(0.1 * data.shape[0])
+    data_test = data[-N_test:]
+    data = data[0:-N_test]
+    N_validate = int(0.1 * data.shape[0])
+    data_validate = data[-N_validate:]
+    data_train = data[0:-N_validate]
+
+    return data_train, data_validate, data_test
+
+
+def load_data_normalised(root_path):
+
+    data_train, data_validate, data_test = load_data(root_path)
+    data = np.vstack((data_train, data_validate))
+    mu = data.mean(axis=0)
+    s = data.std(axis=0)
+    data_train = (data_train - mu) / s
+    data_validate = (data_validate - mu) / s
+    data_test = (data_test - mu) / s
+
+    return data_train, data_validate, data_test