Create //torchx/specs:lib_core (#1048)

Summary:

IMPORTANT:  If you believe you are broken by this diff due to autodeps/codemod having removed a dep to 
`//torchx/specs:api` in favor of `//torchx/specs:lib_core` then try adding a manual (`# manual`) dep to `//torchx/specs/fb:api_extended` to fix forward.  

**Example:** 
```
python_library|binary(  
  name = "my-lib-or-bin",
  srcs = [...],
  deps = [
    "//torchx/specs:lib_core", # <-- added by autodeps codemod
    "//torchx/specs/fb:api_extended", # manual <-- manually add api_extended
    ...
  ]
)
```

Milestone 1.1 of torchx-lite (see [doc](https://fburl.com/gdoc/4wxcoaoa) for details)

Creates `//torchx/specs:lib_core` with only the source files of `torchx.specs` module and a minimal set of dependencies for a proper closure of this module.

For BC, we keep `//torchx/specs:lib` and `//torchx/specs:api` as is by adding the existing bundled deps as `manual` so that autodeps doesn't remove them on future codemods.

Switches over to `//torchx/specs:lib_core` for TorchX internal targets (e.g. within `//torchx/...`).

Differential Revision: D73195094

Author: kiukchung

torchx/cli/test/cmd_run_test.py

@@ -22,6 +22,7 @@
 
 from torchx.cli.argparse_util import ArgOnceAction, torchxconfig
 from torchx.cli.cmd_run import _parse_component_name_and_args, CmdBuiltins, CmdRun
+from torchx.runner.config import ENV_TORCHXCONFIG
 from torchx.schedulers.local_scheduler import SignalException
 
 from torchx.specs import AppDryRunInfo
@@ -40,11 +41,19 @@ def cwd(path: str) -> Generator[None, None, None]:
 class CmdRunTest(unittest.TestCase):
     def setUp(self) -> None:
         self.tmpdir = Path(tempfile.mkdtemp())
+
+        # create empty .torchxconfig so that user .torchxconfig is not picked up
+        empty_config = self.tmpdir / ".torchxconfig"
+        empty_config.touch()
+        self.mock_env = patch.dict(os.environ, {ENV_TORCHXCONFIG: str(empty_config)})
+        self.mock_env.start()
+
         self.parser = argparse.ArgumentParser()
         self.cmd_run = CmdRun()
         self.cmd_run.add_arguments(self.parser)
 
     def tearDown(self) -> None:
+        self.mock_env.stop()
         shutil.rmtree(self.tmpdir, ignore_errors=True)
         ArgOnceAction.called_args = set()
         torchxconfig.called_args = set()

torchx/specs/__init__.py

@@ -13,7 +13,7 @@
 scheduler or pipeline adapter.
 """
 import difflib
-from typing import Callable, Dict, Optional
+from typing import Callable, Dict, Mapping, Optional
 
 from torchx.specs.api import (
     ALL,
@@ -48,12 +48,17 @@
 )
 from torchx.specs.builders import make_app_handle, materialize_appdef, parse_mounts
 
-from torchx.specs.named_resources_aws import NAMED_RESOURCES as AWS_NAMED_RESOURCES
-from torchx.specs.named_resources_generic import (
-    NAMED_RESOURCES as GENERIC_NAMED_RESOURCES,
-)
 from torchx.util.entrypoints import load_group
 
+from torchx.util.modules import import_attr
+
+AWS_NAMED_RESOURCES: Mapping[str, Callable[[], Resource]] = import_attr(
+    "torchx.specs.named_resources_aws", "NAMED_RESOURCES", default={}
+)
+GENERIC_NAMED_RESOURCES: Mapping[str, Callable[[], Resource]] = import_attr(
+    "torchx.specs.named_resources_generic", "NAMED_RESOURCES", default={}
+)
+
 GiB: int = 1024
 
 

torchx/specs/finder.py

@@ -18,7 +18,6 @@
 from types import ModuleType
 from typing import Any, Callable, Dict, Generator, List, Optional, Union
 
-from torchx.specs import AppDef
 from torchx.specs.file_linter import get_fn_docstring, TorchxFunctionValidator, validate
 from torchx.util import entrypoints
 from torchx.util.io import read_conf_file