torchstudio 0.9.8

see 0.9.8 release changelog for more details

Author: divideconcept

torchstudio/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Robin Lobel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

torchstudio/metrics/accuracy.py

@@ -27,12 +27,12 @@ def update(self, preds, target):
             raise ValueError("prediction and target have different shapes or aren't compatible with multiclass prediction")
 
         self.num_correct += torch.sum(correct)
-        self.num_samples += correct.shape[0]
+        self.num_samples += torch.tensor(correct.shape[0])
 
     def compute(self):
         if self.num_samples == 0:
             raise ValueError("Accuracy must have at least one sample before it can be computed.")
-        return self.num_correct / self.num_samples
+        return self.num_correct.float() / self.num_samples.float()
 
     def reset(self):
         self.num_correct = 0

torchstudio/metrics/fscore.py

@@ -25,8 +25,8 @@ def update(self, preds, target):
             if self.normalize:
                 preds=F.softmax(preds, dim=1)
             tp = torch.sum(torch.eq(torch.argmax(preds, dim=1), target).view(-1))
-            tpfp = tp.shape[0]
-            tpfn = tp.shape[0]
+            tpfp = torch.tensor(tp.shape[0])
+            tpfn = torch.tensor(tp.shape[0])
         elif preds.shape==target.shape:
             if self.normalize:
                 preds=F.sigmoid(preds)
@@ -40,8 +40,8 @@ def update(self, preds, target):
         self.tpfp += tpfp
         self.tp += tp
     def compute(self):
-        precision = self.tp / self.tpfp
-        recall = self.tp / self.tpfn
+        precision = self.tp.float() / self.tpfp.float()
+        recall = self.tp.float() / self.tpfn.float()
         fscore = (1.0+self.beta_square)*(precision*recall)/(self.beta_square*precision+recall)
         return fscore
 

torchstudio/metrics/precision.py

@@ -22,7 +22,7 @@ def update(self, preds, target):
             if self.normalize:
                 preds=F.softmax(preds, dim=1)
             tp = torch.sum(torch.eq(torch.argmax(preds, dim=1), target).view(-1))
-            tpfp = tp.shape[0]
+            tpfp = torch.tensor(tp.shape[0])
         elif preds.shape==target.shape:
             if self.normalize:
                 preds=F.sigmoid(preds)
@@ -34,7 +34,7 @@ def update(self, preds, target):
         self.tpfp += tpfp
         self.tp += tp
     def compute(self):
-        return self.tp / self.tpfp
+        return self.tp.float() / self.tpfp.float()
 
     def reset(self):
         self.tpfp = 0

torchstudio/metrics/recall.py

@@ -22,7 +22,7 @@ def update(self, preds, target):
             if self.normalize:
                 preds=F.softmax(preds, dim=1)
             tp = torch.sum(torch.eq(torch.argmax(preds, dim=1), target).view(-1))
-            tpfn = tp.shape[0]
+            tpfn = torch.tensor(tp.shape[0])
         elif preds.shape==target.shape:
             if self.normalize:
                 preds=F.sigmoid(preds)
@@ -34,7 +34,7 @@ def update(self, preds, target):
         self.tpfn += tpfn
         self.tp += tp
     def compute(self):
-        return self.tp / self.tpfn
+        return self.tp.float() / self.tpfn.float()
 
     def reset(self):
         self.tpfn = 0

torchstudio/modelbuild.py

@@ -234,7 +234,7 @@ def level_trace(root):
                 for tensor in output_tensors:
                     metric.append("Accuracy")
 
-                tc.send_msg(app_socket, 'SetHyperParametersValues', tc.encode_ints([128,0,100,0]))
+                tc.send_msg(app_socket, 'SetHyperParametersValues', tc.encode_ints([128,0,100,1,1,1]))
                 tc.send_msg(app_socket, 'SetHyperParametersNames', tc.encode_strings(loss+metric+['Adam','Step']))
 
     if msg_type == 'Exit':

torchstudio/modeltrain.py

@@ -11,6 +11,7 @@
 import io
 import tempfile
 from tqdm.auto import tqdm
+from collections.abc import Iterable
 
 
 class CachedDataset(Dataset):
@@ -44,6 +45,16 @@ def __getitem__(self, id):
             sample=self.index[id]
         return sample
 
+def deepcopy_cpu(value):
+    if isinstance(value, torch.Tensor):
+        value = value.to("cpu")
+        return value
+    elif isinstance(value, dict):
+        return {k: deepcopy_cpu(v) for k, v in value.items()}
+    elif isinstance(value, Iterable):
+        return type(value)(deepcopy_cpu(v) for v in value)
+    else:
+        return value
 
 modules_valid=True
 
@@ -65,19 +76,23 @@ def __getitem__(self, id):
         pin_memory = True if 'cuda' in device_id else False
 
     if msg_type == 'SetTorchScriptModel' and modules_valid:
-        print("Setting torchscript model...\n", file=sys.stderr)
-        buffer=io.BytesIO(msg_data)
-        model = torch.jit.load(buffer, map_location=device)
+        if msg_data:
+            print("Setting torchscript model...\n", file=sys.stderr)
+            buffer=io.BytesIO(msg_data)
+            model = torch.jit.load(buffer)
 
     if msg_type == 'SetPackageModel' and modules_valid:
-        print("Setting package model...\n", file=sys.stderr)
-        buffer=io.BytesIO(msg_data)
-        model = torch.package.PackageImporter(buffer).load_pickle('model', 'model.pkl', map_location=device)
+        if msg_data:
+            print("Setting package model...\n", file=sys.stderr)
+            buffer=io.BytesIO(msg_data)
+            model = torch.package.PackageImporter(buffer).load_pickle('model', 'model.pkl')
 
     if msg_type == 'SetModelState' and modules_valid:
         if model is not None:
-            buffer=io.BytesIO(msg_data)
-            model.load_state_dict(torch.load(buffer,map_location=device))
+            if msg_data:
+                buffer=io.BytesIO(msg_data)
+                model.load_state_dict(torch.load(buffer))
+            model.to(device)
 
     if msg_type == 'SetLossCodes' and modules_valid:
         print("Setting loss code...\n", file=sys.stderr)
@@ -116,9 +131,11 @@ def __getitem__(self, id):
             tc.send_msg(app_socket, 'TrainingError')
         else:
             optimizer = optimizer_env['optimizer']
+
     if msg_type == 'SetOptimizerState' and modules_valid:
-        buffer=io.BytesIO(msg_data)
-        optimizer.load_state_dict(torch.load(buffer,map_location=device))
+        if msg_data:
+            buffer=io.BytesIO(msg_data)
+            optimizer.load_state_dict(torch.load(buffer))
 
     if msg_type == 'SetSchedulerCode' and modules_valid:
         print("Setting scheduler code...\n", file=sys.stderr)
@@ -131,9 +148,11 @@ def __getitem__(self, id):
             scheduler = scheduler_env['scheduler']
 
     if msg_type == 'SetHyperParametersValues' and modules_valid: #set other hyperparameters values
-        batch_size, shuffle, epochs, early_stop = tc.decode_ints(msg_data)
-        early_stop=True if early_stop==1 else False
+        batch_size, shuffle, epochs, early_stop, monitor_metric, restore_best = tc.decode_ints(msg_data)
         shuffle=True if shuffle==1 else False
+        early_stop=True if early_stop==1 else False
+        monitor_metric=True if monitor_metric==1 else False
+        restore_best=True if restore_best==1 else False
 
     if msg_type == 'StartTrainingServer' and modules_valid:
         print("Caching...\n", file=sys.stderr)
@@ -267,11 +286,11 @@ def __getitem__(self, id):
         tc.send_msg(app_socket, 'ValidationMetric', tc.encode_floats(valid_metrics))
 
         buffer=io.BytesIO()
-        torch.save(model.state_dict(), buffer)
+        torch.save(deepcopy_cpu(model.state_dict()), buffer)
         tc.send_msg(app_socket, 'ModelState', buffer.getvalue())
 
         buffer=io.BytesIO()
-        torch.save(optimizer.state_dict(), buffer)
+        torch.save(deepcopy_cpu(optimizer.state_dict()), buffer)
         tc.send_msg(app_socket, 'OptimizerState', buffer.getvalue())
 
         tc.send_msg(app_socket, 'Trained')
@@ -280,7 +299,7 @@ def __getitem__(self, id):
         if train_bar is not None:
             train_bar.bar_format='{desc} epoch {n_fmt} | {remaining} left |{rate_fmt}\n\n'
         else:
-            train_bar = tqdm(total=epochs, desc='Training...', bar_format='{desc} epoch '+str(scheduler.last_epoch)+'\n\n')
+            train_bar = tqdm(total=epochs, desc='Training...', bar_format='{desc} epoch '+str(scheduler.last_epoch)+'\n\n', initial=scheduler.last_epoch-1)
         train_bar.update(1)
 
     if msg_type == 'StopTraining' and modules_valid:

torchstudio/pythoncheck.py

@@ -2,6 +2,7 @@
 
 print("Checking Python version...\n", file=sys.stderr)
 
+import platform
 import argparse
 import importlib
 
@@ -23,8 +24,8 @@
 checked_modules = ["torch", "torchvision"]
 required_packages = ["pytorch", "torchvision"]
 if not args.remote:
-    checked_modules += ["torchaudio", "matplotlib", "graphviz"]
-    required_packages += ["torchaudio", "matplotlib-base", "python-graphviz"]
+    checked_modules += ["torchaudio", "torchtext", "matplotlib", "graphviz"]
+    required_packages += ["torchaudio", "torchtext", "matplotlib-base", "python-graphviz"]
 missing_modules = []
 for module_check in checked_modules:
     module = importlib.util.find_spec(module_check)
@@ -46,6 +47,7 @@
     #warn about missing modules
     print("Error: Missing Python modules:", file=sys.stderr)
     print(*missing_modules, sep = " ", file=sys.stderr)
+    print("", file=sys.stderr)
     print("The following packages are required:", file=sys.stderr)
     print(' '.join(required_packages), file=sys.stderr)
     exit(1)
@@ -75,12 +77,15 @@
     devices['cpu'] = {'name': 'CPU', 'pin_memory': False}
     for i in range(torch.cuda.device_count()):
         devices['cuda:'+str(i)] = {'name': torch.cuda.get_device_name(i), 'pin_memory': True}
+    if pytorch_version>=(1,12):
+        if torch.backends.mps.is_available():
+            devices['mps'] = {'name': 'Metal Acceleration', 'pin_memory': False}
     #other possible devices:
     #'hpu' (https://docs.habana.ai/en/latest/PyTorch_User_Guide/PyTorch_User_Guide.html)
     #'dml' (https://docs.microsoft.com/en-us/windows/ai/directml/gpu-pytorch-windows)
     devices_string_list=[]
     for id in devices:
         devices_string_list.append(devices[id]['name']+" ("+id+")")
-    print(("Online and functional " if args.remote else "Functional environment ")+"(Python "+str(python_version[0])+"."+str(python_version[1])+", PyTorch "+str(pytorch_version[0])+"."+str(pytorch_version[1])+", Devices: "+", ".join(devices_string_list)+")");
+    print(("Online and functional " if args.remote else "Functional ")+"("+platform.platform()+", Python "+str(python_version[0])+"."+str(python_version[1])+", PyTorch "+str(pytorch_version[0])+"."+str(pytorch_version[1])+", Devices: "+", ".join(devices_string_list)+")");
 
 

torchstudio/pythoninstall.py

@@ -3,7 +3,9 @@
 import importlib.util
 import argparse
 parser = argparse.ArgumentParser()
+parser.add_argument("--base", help="install base packages", action="store_true", default=False)
 parser.add_argument("--gpu", help="install nvidia gpu support", action="store_true", default=False)
+parser.add_argument("--package", help="install specific package", action='append', nargs='+', default=[])
 args, unknown = parser.parse_known_args()
 
 if importlib.util.find_spec("conda") is None:
@@ -12,37 +14,32 @@
 
 import conda.cli.python_api as Conda
 
-# datasets(+huggingface_hub) required by hugging face hub
-# scipy required by torchvision: Caltech ImageNet SBD SVHN datasets and Inception v3 GoogLeNet models
-# pandas required by the dataset tutorial: https://pytorch.org/tutorials/beginner/data_loading_tutorial.html
-# matplotlib-base required by torchstudio renderers
-# python-graphviz required by torchstudio graph
-# paramiko required for ssh connections
-# pysoundfile required on windows by torchaudio: https://pytorch.org/audio/stable/backend.html#soundfile-backend
-if sys.platform.startswith('win'):
-    if args.gpu:
-        conda_install="pytorch torchvision torchaudio cudatoolkit=11.3 datasets scipy pandas matplotlib-base python-graphviz paramiko pysoundfile"
-    else:
-        conda_install="pytorch torchvision torchaudio cpuonly datasets scipy pandas matplotlib-base python-graphviz paramiko pysoundfile"
-elif sys.platform.startswith('darwin'):
-    # force a pytorch/mkl version, because pytorch 1.10.2+ depends on mkl 2022 which is incompatible with Rosetta 2 in M1 macs, and update cffi 1.15.0-py39hc55c11b_1 to 1.15.0-py39he338e87_0+ to avoid paramiko error
-    conda_install="pytorch==1.10.1 torchvision==0.11.2 torchaudio==0.10.1 mkl==2021.4.0 datasets scipy pandas matplotlib-base python-graphviz paramiko cffi"
-elif sys.platform.startswith('linux'):
-    if args.gpu:
-        conda_install="pytorch torchvision torchaudio cudatoolkit=11.3 datasets scipy pandas matplotlib-base python-graphviz paramiko"
-    else:
-        conda_install="pytorch torchvision torchaudio cpuonly datasets scipy pandas matplotlib-base python-graphviz paramiko"
-else:
-    print("Error: Unsupported platform.", file=sys.stderr)
-    print("Windows, macOS or Linux is required.", file=sys.stderr)
-    exit()
+conda_install=""
+if args.base:
+    # scipy required by torchvision: Caltech ImageNet SBD SVHN datasets and Inception v3 GoogLeNet models
+    # pandas required by the dataset tutorial: https://pytorch.org/tutorials/beginner/data_loading_tutorial.html
+    # matplotlib-base required by torchstudio renderers
+    # python-graphviz required by torchstudio graph
+    # paramiko required for ssh connections
+    # pysoundfile required by torchaudio datasets: https://pytorch.org/audio/stable/backend.html#soundfile-backend
+    # datasets(+huggingface_hub) is required by hugging face hub
+    conda_install="pytorch torchvision torchaudio torchtext scipy pandas matplotlib-base python-graphviz paramiko pysoundfile datasets"
+    if (sys.platform.startswith('win') or sys.platform.startswith('linux')) and not args.gpu:
+        conda_install+=" cpuonly"
+    if sys.platform.startswith('darwin'):
+        conda_install+=" cffi"
+
+if args.package:
+    if args.base:
+        conda_install+=" "
+    conda_install+=" ".join(args.package[0])
 
-print("Downloading and installing PyTorch and additional packages:")
+print("Downloading and installing conda packages:")
 print(conda_install)
 print("")
 
-# channels: pytorch for pytorch torchvision torchaudio, nvidia for cudatoolkit=11.1 on Linux, huggingface for datasets(+huggingface_hub), conda-forge for everything else except anaconda for python-graphviz
-conda_install+=" -c pytorch -c nvidia -c huggingface -c conda-forge -c anaconda"
+# channels: pytorch for pytorch torchvision torchaudio, conda-forge for everything else
+conda_install+=" -c pytorch -c conda-forge"
 
 # https://stackoverflow.com/questions/41767340/using-conda-install-within-a-python-script
 (stdout_str, stderr_str, return_code_int) = Conda.run_command(Conda.Commands.INSTALL,conda_install.split(),stdout=sys.stdout,stderr=sys.stderr)

torchstudio/pythonparse.py

@@ -29,13 +29,17 @@ def gather_parameters(node):
         elif inspect.isclass(param.default) or inspect.isfunction(param.default):
             params.append(param.default.__module__+'.'+param.default.__name__)
         else:
-            params.append(repr(param.default))
+            value=repr(param.default)
+            if "<class '" in value:
+                value=value.replace("<class '","")
+                value=value.replace("'>","")
+            params.append(value)
     return params
 
 def gather_objects(module):
     objects=[]
     for name, obj in inspect.getmembers(module):
-        if (inspect.isclass(obj) or inspect.isfunction(obj)) and not hasattr(obj, '_fields') and obj.__module__.find('.utils')==-1: #filter unwanted torch objects
+        if ((inspect.isclass(obj) and hasattr(obj, '__mro__') and ("torch.nn.modules.module.Module" in str(obj.__mro__) or "torch.utils.data.dataset.Dataset" in str(obj.__mro__))) or inspect.isfunction(obj)): #filter unwanted torch objects
             object={}
             object['type']='class' if inspect.isclass(obj) else 'function'
             object['name']=name
@@ -357,13 +361,17 @@ def scan_folder(path):
                     for i in range(len(objects_batch)):
                         objects_batch[i]['code']=code #set whole source code for each object, as we don't know the dependencies
                     objects.extend(objects_batch)
+                else:
+                    print("Error parsing code:", error_msg, "\n", file=sys.stderr)
         else:
             #parse module
             error_msg, module = safe_exec(importlib.import_module,(path,))
             if error_msg is None and module is not None:
                 objects=gather_objects(module)
                 for i, object in enumerate(objects):
                     objects[i]['code']=generate_code(path,object) #generate inherited source code
+            else:
+                print("Error parsing module:", error_msg, "\n", file=sys.stderr)
 
         tc.send_msg(app_socket, 'ObjectsBegin', tc.encode_strings(path))
         for object in objects: