Initial vendoring mock-up

azext/batch/_file_utils.py

@@ -320,36 +320,23 @@ def get_container_list(self, source):
 
     def resolve_resource_file(self, resource_file):
         """Convert new resourceFile reference to server-supported reference"""
-        if resource_file.http_url:
-            # Support original resourceFile reference
-            if not resource_file.file_path:
-                raise ValueError('Malformed ResourceFile: \'httpUrl\' must '
-                                 'also have \'file_path\' attribute')
-            return [resource_file]
-
-        if resource_file.storage_container_url or resource_file.auto_storage_container_name:
-            return [resource_file]
-
-        if not hasattr(resource_file, 'source') or not resource_file.source:
-            raise ValueError('Malformed ResourceFile: Must have either '
-                             ' \'source\' or \'httpUrl\'')
-
         storage_client = self.resolve_storage_account()
         container = None
         blobs = []
-
-        if resource_file.source.file_group:
-            # Input data stored in auto-storage
-            container = get_container_name(resource_file.source.file_group)
-            blobs = self.list_container_contents(resource_file.source, container, storage_client)
-            return convert_blobs_to_resource_files(blobs, resource_file)
-        if resource_file.source.container_url:
-            # Input data storage in arbitrary container
-            uri = urlsplit(resource_file.source.container_url)
-            container = uri.pathname.split('/')[1]
-            blobs = self.list_container_contents(resource_file.source, container, storage_client)
-            return convert_blobs_to_resource_files(blobs, resource_file)
-        if resource_file.source.url:
-            # TODO: Input data from an arbitrary HTTP GET source
-            raise ValueError('Not implemented')
-        raise ValueError('Malformed ResourceFile')
+        if hasattr(resource_file, 'source') and  resource_file.source:
+            if resource_file.source.file_group:
+                # Input data stored in auto-storage
+                container = get_container_name(resource_file.source.file_group)
+                blobs = self.list_container_contents(resource_file.source, container, storage_client)
+                return convert_blobs_to_resource_files(blobs, resource_file)
+            if resource_file.source.container_url:
+                # Input data storage in arbitrary container
+                uri = urlsplit(resource_file.source.container_url)
+                container = uri.pathname.split('/')[1]
+                blobs = self.list_container_contents(resource_file.source, container, storage_client)
+                return convert_blobs_to_resource_files(blobs, resource_file)
+            if resource_file.source.url:
+                # TODO: Input data from an arbitrary HTTP GET source
+                raise ValueError('Not implemented')
+
+        return [resource_file]

azext/batch/_template_utils.py

@@ -9,7 +9,7 @@
 import copy
 import itertools
 import json
-from logging import getLogger
+import logging
 import re
 from msrest.serialization import Model
 try:
@@ -21,7 +21,7 @@
 from . import _pool_utils as pool_utils
 from . import models
 
-logger = getLogger(__name__)
+logger = logging.getLogger(__name__)
 try:
     _UNICODE_TYPE = unicode
 except NameError:

azext/batch/batch_extensions_client.py

@@ -8,7 +8,7 @@
 from six.moves.urllib.parse import urlsplit  # pylint: disable=import-error,relative-import
 from msrest import Serializer, Deserializer
 
-from azure.batch import BatchServiceClient
+from ..generated.sdk.batch import BatchServiceClient
 from azure.mgmt.batch import BatchManagementClient
 from azure.mgmt.storage import StorageManagementClient
 from azure.storage.blob import BlockBlobService
@@ -76,12 +76,9 @@ def __init__(self, credentials=None, batch_url=None, subscription_id=None,
         self._serialize = Serializer(client_models)
         self._deserialize = Deserializer(client_models)
 
-        self.pool = ExtendedPoolOperations(
-            self, self._client, self.config, self._serialize, self._deserialize, self._storage_account)
-        self.job = ExtendedJobOperations(
-            self, self._client, self.config, self._serialize, self._deserialize, self._storage_account)
-        self.file = ExtendedFileOperations(
-            self, self._client, self.config, self._serialize, self._deserialize, self._storage_account)
+        self.pool_extensions = ExtendedPoolOperations(self, self._client, self.config, self._serialize, self._deserialize, self._storage_account)
+        self.job_extensions = ExtendedJobOperations(self, self._client, self.config, self._serialize, self._deserialize, self._storage_account)
+        self.file_extensions = ExtendedFileOperations(self, self._client, self.config, self._serialize, self._deserialize, self._storage_account)
 
     def _get_cli_profile(self, subscription_id):  # pylint:disable=no-self-use
         try:

azext/batch/errors.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
-from azure.batch.models import BatchErrorException
+from .models import BatchErrorException
 
 
 class MissingParameterValue(ValueError):

azext/batch/models/__init__.py

@@ -7,8 +7,7 @@
 
 # Not ideal syntax - but saves us having to check and repopulate this
 # list every time the SDK is regenerated.
-from azure.batch.models import *
-
+from ...generated.sdk.batch.v2019_08_01.models import *
 try:
     from ._models_py3 import ExtendedTaskParameter
     from ._models_py3 import ExtendedJobParameter

azext/batch/models/_models.py

@@ -898,6 +898,7 @@ class ExtendedResourceFile(models.ResourceFile):
         'http_url': {'key': 'httpUrl', 'type': 'str'},
         'auto_storage_container_name': {'key': 'autoStorageContainerName', 'type': 'str'},
         'blob_prefix': {'key': 'blobPrefix', 'type': 'str'},
+        'blob_source': {'key': 'blobSource', 'type': 'str'},
         'storage_container_url': {'key': 'storageContainerUrl', 'type': 'str'},
         'file_path': {'key': 'filePath', 'type': 'str'},
         'file_mode': {'key': 'fileMode', 'type': 'str'},

azext/batch/models/_models_py3.py

@@ -5,11 +5,18 @@
 
 # pylint: disable=too-many-lines
 
+import importlib
 import os
-import azure.batch.models as models
 from msrest.serialization import Model
 from .constants import ATTRS_RESERVED_FOR_TEMPLATES
 
+models_base = "azext.generated.sdk.batch.v2019_08_01.models.{}"
+JobAddParameter = importlib.import_module(models_base.format("JobAddParameter"))
+PoolAddParameter = importlib.import_module(models_base.format("PoolAddParameter"))
+TaskAddParameter = importlib.import_module(models_base.format("TaskAddParameter"))
+PoolSpecification = importlib.import_module(models_base.format("PoolSpecification"))
+ResourceFile = importlib.import_module(models_base.format("ResourceFile"))
+
 
 class TaskFactoryBase(Model):
     """A Task Factory for automatically adding a collection of tasks to a job on
@@ -326,7 +333,7 @@ def __init__(self, *, id: str, version: str=None, allow_empty_checksums: bool=No
         self.type = 'chocolateyPackage'
 
 
-class ExtendedJobParameter(models.JobAddParameter):
+class ExtendedJobParameter(Model):
     """An Azure Batch job to add.
 
     :param id: A string that uniquely identifies the job within the account.
@@ -430,6 +437,9 @@ class ExtendedJobParameter(models.JobAddParameter):
     'on_task_failure', 'task_factory', 'job_preparation_task', 'job_release_task'.
     :type application_template_info: :class:`ApplicationTemplateInfo
      <azext.batch.models.ApplicationTemplateInfo>`
+    :param base_model: A reference to the object this class should extend
+    :type base_model: :class:`Model
+     <msrest.serialization.Model>`
     """
 
     _validation = {
@@ -452,15 +462,17 @@ class ExtendedJobParameter(models.JobAddParameter):
         'metadata': {'key': 'metadata', 'type': '[MetadataItem]'},
         'uses_task_dependencies': {'key': 'usesTaskDependencies', 'type': 'bool'},
         'task_factory': {'key': 'taskFactory', 'type': 'TaskFactoryBase'},
-        'application_template_info': {'key': 'applicationTemplateInfo', 'type': 'ApplicationTemplateInfo'}
+        'application_template_info': {'key': 'applicationTemplateInfo', 'type': 'ApplicationTemplateInfo'},
+        'base_model': {'key': 'baseModel', 'type': 'Model'}
     }
 
     def __init__(self, *, id: str, pool_info, display_name: str=None, priority: int=None, constraints=None,
                  job_manager_task=None, job_preparation_task=None, job_release_task=None,
                  common_environment_settings=None, on_all_tasks_complete=None, on_task_failure=None,
                  metadata=None, uses_task_dependencies: bool=None, task_factory=None,
-                 application_template_info=None, **kwargs) -> None:
-        super(ExtendedJobParameter, self).__init__(
+                 application_template_info=None, base_model=JobAddParameter, **kwargs) -> None:
+        base_model.__init__(
+            self,
             id=id,
             display_name=display_name,
             priority=priority,
@@ -499,11 +511,15 @@ class ExtendedOutputFileDestination(Model):
      combined with container.
     :type auto_storage: :class:`OutputFileAutoStorageDestination
      <azext.batch.models.OutputFileAutoStorageDestination>`
+    :param base_model: A reference to the object this class should extend
+    :type base_model: :class:`Model
+     <msrest.serialization.Model>`
     """
 
     _attribute_map = {
         'container': {'key': 'container', 'type': 'OutputFileBlobContainerDestination'},
         'auto_storage': {'key': 'autoStorage', 'type': 'OutputFileAutoStorageDestination'},
+        'base_model': {'key': 'baseModel', 'type': 'Model'}
     }
 
     def __init__(self, *, container=None, auto_storage=None, **kwargs) -> None:
@@ -514,7 +530,7 @@ def __init__(self, *, container=None, auto_storage=None, **kwargs) -> None:
         self.auto_storage = auto_storage
 
 
-class ExtendedPoolParameter(models.PoolAddParameter):
+class ExtendedPoolParameter(Model):
     """A pool in the Azure Batch service to add.
 
     :param id: Required. A string that uniquely identifies the Pool within the
@@ -653,6 +669,9 @@ class ExtendedPoolParameter(models.PoolAddParameter):
      operating system.
     :type package_references: list of :class:`PackageReferenceBase
      <azext.batch.models.PackageReferenceBase>`
+    :param base_model: A reference to the object this class should extend
+    :type base_model: :class:`Model
+     <msrest.serialization.Model>`
     """
 
     _validation = {
@@ -683,7 +702,8 @@ class ExtendedPoolParameter(models.PoolAddParameter):
         'user_accounts': {'key': 'userAccounts', 'type': '[UserAccount]'},
         'metadata': {'key': 'metadata', 'type': '[MetadataItem]'},
         'mount_configuration': {'key': 'mountConfiguration', 'type': '[MountConfiguration]'},
-        'package_references': {'key': 'packageReferences', 'type': '[PackageReferenceBase]'}
+        'package_references': {'key': 'packageReferences', 'type': '[PackageReferenceBase]'},
+        'base_model': {'key': 'baseModel', 'type': 'Model'}
     }
 
     def __init__(self, *, id: str, vm_size: str, display_name: str=None, cloud_service_configuration=None,
@@ -692,9 +712,10 @@ def __init__(self, *, id: str, vm_size: str, display_name: str=None, cloud_servi
                  auto_scale_evaluation_interval=None, enable_inter_node_communication: bool=None, network_configuration=None,
                  start_task=None, certificate_references=None, application_package_references=None, application_licenses=None,
                  max_tasks_per_node: int=None, task_scheduling_policy=None, user_accounts=None, metadata=None,
-                 mount_configuration=None, package_references=None,
+                 mount_configuration=None, package_references=None, base_model=PoolAddParameter,
                  **kwargs) -> None:
-        super(ExtendedPoolParameter, self).__init__(
+        base_model.__init__(
+            self,
             id=id,
             display_name=display_name,
             vm_size=vm_size,
@@ -721,7 +742,7 @@ def __init__(self, *, id: str, vm_size: str, display_name: str=None, cloud_servi
         self.package_references = package_references
 
 
-class ExtendedPoolSpecification(models.PoolSpecification):
+class ExtendedPoolSpecification(Model):
     """Specification for creating a new pool.
 
     :param display_name: The display name for the pool. The display name need
@@ -856,6 +877,9 @@ class ExtendedPoolSpecification(models.PoolSpecification):
      operating system.
     :type package_references: list of :class:`PackageReferenceBase
      <azext.batch.models.PackageReferenceBase>`
+    :param base_model: A reference to the object this class should extend
+    :type base_model: :class:`Model
+     <msrest.serialization.Model>`
     """
 
     _validation = {
@@ -886,7 +910,8 @@ class ExtendedPoolSpecification(models.PoolSpecification):
         'application_licenses': {'key': 'applicationLicenses', 'type': '[str]'},
         'user_accounts': {'key': 'userAccounts', 'type': '[UserAccount]'},
         'metadata': {'key': 'metadata', 'type': '[MetadataItem]'},
-        'package_references': {'key': 'packageReferences', 'type': '[PackageReferenceBase]'}
+        'package_references': {'key': 'packageReferences', 'type': '[PackageReferenceBase]'},
+        'base_model': {'key': 'baseModel', 'type': 'Model'}
     }
 
     def __init__(self, *, vm_size: str, display_name: str=None, cloud_service_configuration=None,
@@ -895,8 +920,10 @@ def __init__(self, *, vm_size: str, display_name: str=None, cloud_service_config
                  enable_auto_scale: bool=None, auto_scale_formula: str=None, auto_scale_evaluation_interval=None,
                  enable_inter_node_communication: bool=None, network_configuration=None, start_task=None,
                  certificate_references=None, application_package_references=None, application_licenses=None,
-                 user_accounts=None, metadata=None, package_references=None, **kwargs) -> None:
-        super(ExtendedPoolSpecification, self).__init__(
+                 user_accounts=None, metadata=None, package_references=None, base_model=PoolSpecification,
+                 **kwargs) -> None:
+        base_model.__init__(
+            self,
             display_name=display_name,
             vm_size=vm_size,
             cloud_service_configuration=cloud_service_configuration,
@@ -921,7 +948,7 @@ def __init__(self, *, vm_size: str, display_name: str=None, cloud_service_config
         self.package_references = package_references
 
 
-class ExtendedResourceFile(models.ResourceFile):
+class ExtendedResourceFile(Model):
     """A file to be downloaded from Azure blob storage to a compute node.
 
     :param http_url: The URL of the file within Azure Blob Storage. This
@@ -969,6 +996,9 @@ class ExtendedResourceFile(models.ResourceFile):
      a Azure Storage container or an auto-storage file group.
     :type source: :class:`FileSource
      <azext.batch.models.FileSource>`
+    :param base_model: A reference to the object this class should extend
+    :type base_model: :class:`Model
+     <msrest.serialization.Model>`
     """
 
     _attribute_map = {
@@ -978,7 +1008,9 @@ class ExtendedResourceFile(models.ResourceFile):
         'storage_container_url': {'key': 'storageContainerUrl', 'type': 'str'},
         'file_path': {'key': 'filePath', 'type': 'str'},
         'file_mode': {'key': 'fileMode', 'type': 'str'},
-        'source': {'key': 'source', 'type': 'FileSource'}
+        'blob_source': {'key': 'blobSource', 'type': 'str'},
+        'source': {'key': 'source', 'type': 'FileSource'},
+        'base_model': {'key': 'baseModel', 'type': 'Model'}
     }
 
     def __init__(self,
@@ -989,8 +1021,11 @@ def __init__(self,
                  blob_prefix: str=None,
                  file_path: str=None,
                  file_mode: str=None,
-                 source=None, **kwargs) -> None:
-        super(ExtendedResourceFile, self).__init__(
+                 source=None,
+                 base_model=ResourceFile,
+                 **kwargs) -> None:
+        base_model.__init__(
+            self,
             http_url=http_url,
             auto_storage_container_name=auto_storage_container_name,
             storage_container_url=storage_container_url,
@@ -1001,7 +1036,7 @@ def __init__(self,
         self.source = source
 
 
-class ExtendedTaskParameter(models.TaskAddParameter):
+class ExtendedTaskParameter(Model):
     """An Azure Batch task to add.
 
     :param id: A string that uniquely identifies the task within the job. The
@@ -1107,6 +1142,9 @@ class ExtendedTaskParameter(models.TaskAddParameter):
      operating system.
     :type package_references: list of :class:`PackageReferenceBase
      <azext.batch.models.PackageReferenceBase>`
+    :param base_model: A reference to the object this class should extend
+    :type base_model: :class:`Model
+     <msrest.serialization.Model>`
     """
 
     _validation = {
@@ -1132,16 +1170,18 @@ class ExtendedTaskParameter(models.TaskAddParameter):
                                            'type': '[ApplicationPackageReference]'},
         'authentication_token_settings': {'key': 'authenticationTokenSettings',
                                           'type': 'AuthenticationTokenSettings'},
-        'package_references': {'key': 'packageReferences', 'type': '[PackageReferenceBase]'}
+        'package_references': {'key': 'packageReferences', 'type': '[PackageReferenceBase]'},
+        'base_model': {'key': 'baseModel', 'type': 'Model'}
     }
 
     def __init__(self, *, id: str, command_line: str, display_name: str=None, container_settings=None,
                  exit_conditions=None, resource_files=None, output_files=None, environment_settings=None,
                  affinity_info=None, constraints=None, user_identity=None,
                  multi_instance_settings=None, depends_on=None,
                  application_package_references=None, authentication_token_settings=None,
-                 package_references=None, **kwargs) -> None:
-        super(ExtendedTaskParameter, self).__init__(
+                 package_references=None, base_model=TaskAddParameter, **kwargs) -> None:
+        base_model.__init__(
+            self,
             id=id,
             display_name=display_name,
             command_line=command_line,

azext/batch/models/constants.py

@@ -65,6 +65,21 @@
     'dependsOn'})
 
 
-# Dates used as cutoffs for different SDK extension versions
-class KnownTemplateVersion(Enum):
-    Dec2018 = "2018-12-01"
+# Ensure the first member of this array is the official REST Version
+class SupportedRestApi(Enum):
+    Aug2018 = ["2018-08-01.7.0", "2018-08-01"]
+    Dec2018 = ["2018-12-01.8.0", "2018-12-01"]
+    Jun2019 = ["2019-06-01.9.0", "2019-06-01"]
+    Aug2019 = ["2019-08-01.10.0", "2019-08-01", "latest"]
+
+
+class SupportedTemplateApi(Enum):
+    Latest = ["latest"]
+
+
+SupportRestApiToSdkVersion = {
+    SupportedRestApi.Aug2018: "2018_08_01",
+    SupportedRestApi.Dec2018: "2018_12_01",
+    SupportedRestApi.Jun2019: "2019_06_01",
+    SupportedRestApi.Aug2019: "2019_08_01",
+}