Conversion to sliced checkpoints now supports PathManager (facebookre…

…search#387)

Summary:
Pull Request resolved: facebookresearch#387

The sliced were not created at the right place because of os.path.abspath

Reviewed By: iseessel

Differential Revision: D30109789

fbshipit-source-id: c332fbf5f5c52241a537bd1188e3268a2f5cb966

vissl/utils/checkpoint.py

@@ -20,7 +20,7 @@
 from fvcore.common.file_io import PathManager
 from vissl.config import AttrDict
 from vissl.utils.env import get_machine_local_and_dist_rank
-from vissl.utils.io import create_file_symlink, makedir
+from vissl.utils.io import abspath, create_file_symlink, makedir
 from vissl.utils.layer_memory_tracking import null_context
 
 
@@ -353,10 +353,10 @@ def save_slice(cls, checkpoint_path: str, param_path: str, param) -> str:
         - return the created file name
         """
         checkpoint_sub_folder = os.path.splitext(checkpoint_path)[0] + "_layers"
-        os.makedirs(checkpoint_sub_folder, exist_ok=True)
+        makedir(checkpoint_sub_folder)
         hash_name = hashlib.sha1(param_path.encode()).hexdigest()
-        file_path = os.path.join(f"{checkpoint_sub_folder}", f"{hash_name}.torch")
-        file_path = os.path.abspath(file_path)
+        file_path = os.path.join(checkpoint_sub_folder, f"{hash_name}.torch")
+        file_path = abspath(file_path)
         checkpoint_slice = {"type": CheckpointItemType.slice.name, "weight": param}
         with PathManager.open(file_path, "wb") as f:
             torch.save(checkpoint_slice, f)

vissl/utils/io.py

@@ -129,6 +129,18 @@ def load_file(filename, mmap_mode=None):
     return data
 
 
+def abspath(resource_path: str):
+    """
+    Make a path absolute, but take into account prefixes like
+    "http://" or "manifold://"
+    """
+    regex = re.compile(r"^\w+://")
+    if regex.match(resource_path) is None:
+        return os.path.abspath(resource_path)
+    else:
+        return resource_path
+
+
 def makedir(dir_path):
     """
     Create the directory if it does not exist.