Skip to content

Commit 57ef170

Browse files
sbaidachnisbaidachni
authored
Remove sys.path from everywhere to enable IntelliSense (#162)
1 parent a24870a commit 57ef170

20 files changed

+77
-94
lines changed

‎.pipelines/azdo-abtest-pipeline.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ stages:
4040
inlineScript: |
4141
set -e
4242
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
43-
python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt
43+
python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt
4444
displayName: 'Create Scoring Image'
4545
name: 'buildscoringimage'
4646

‎.pipelines/azdo-base-pipeline.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ steps:
1010
enabled: 'true'
1111

1212
- script: |
13-
pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit
13+
python -m pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit
1414
displayName: 'Run unit tests'
1515
enabled: 'true'
1616
env:

‎.pipelines/diabetes_regression-ci-build-train.yml‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,12 @@ stages:
3131
inputs:
3232
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
3333
scriptLocation: inlineScript
34+
workingDirectory: $(Build.SourcesDirectory)
3435
inlineScript: |
3536
set -e # fail on error
3637
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
3738
# Invoke the Python building and publishing a training pipeline
38-
python $(Build.SourcesDirectory)/ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }}
39+
python -m ml_service.pipelines.diabetes_regression_build_train_pipeline
3940
displayName: 'Publish Azure Machine Learning Pipeline'
4041

4142
- stage: 'Trigger_AML_Pipeline'
@@ -53,10 +54,11 @@ stages:
5354
inputs:
5455
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
5556
scriptLocation: inlineScript
57+
workingDirectory: $(Build.SourcesDirectory)
5658
inlineScript: |
5759
set -e # fail on error
5860
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
59-
python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution
61+
python -m ml_service.pipelines.run_train_pipeline --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution
6062
# Set AMLPIPELINEID variable for next AML Pipeline task in next job
6163
AMLPIPELINEID="$(cat pipeline_id.txt)"
6264
echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID"
@@ -95,7 +97,7 @@ stages:
9597
container: mcr.microsoft.com/mlops/python:latest
9698
timeoutInMinutes: 0
9799
steps:
98-
- template: azdo-template-get-model-version.yml
100+
- template: diabetes_regression-template-get-model-version.yml
99101
- stage: 'Deploy_ACI'
100102
displayName: 'Deploy to ACI'
101103
dependsOn: Trigger_AML_Pipeline
@@ -108,7 +110,7 @@ stages:
108110
container: mcr.microsoft.com/mlops/python:latest
109111
timeoutInMinutes: 0
110112
steps:
111-
- template: azdo-template-get-model-version.yml
113+
- template: diabetes_regression-template-get-model-version.yml
112114
- task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
113115
displayName: 'Azure ML Model Deploy'
114116
inputs:
@@ -129,7 +131,7 @@ stages:
129131
inlineScript: |
130132
set -e # fail on error
131133
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
132-
python ml_service/util/smoke_test_scoring_service.py --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
134+
python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
133135
134136
- stage: 'Deploy_AKS'
135137
displayName: 'Deploy to AKS'
@@ -143,7 +145,7 @@ stages:
143145
container: mcr.microsoft.com/mlops/python:latest
144146
timeoutInMinutes: 0
145147
steps:
146-
- template: azdo-template-get-model-version.yml
148+
- template: diabetes_regression-template-get-model-version.yml
147149
- task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
148150
displayName: 'Azure ML Model Deploy'
149151
inputs:
@@ -165,7 +167,7 @@ stages:
165167
inlineScript: |
166168
set -e # fail on error
167169
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
168-
python ml_service/util/smoke_test_scoring_service.py --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
170+
python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
169171
170172
- stage: 'Deploy_Webapp'
171173
displayName: 'Deploy to Webapp'
@@ -179,7 +181,7 @@ stages:
179181
container: mcr.microsoft.com/mlops/python:latest
180182
timeoutInMinutes: 0
181183
steps:
182-
- template: azdo-template-get-model-version.yml
184+
- template: diabetes_regression-template-get-model-version.yml
183185
- task: AzureCLI@1
184186
displayName: 'Create scoring image and set IMAGE_LOCATION variable'
185187
inputs:
@@ -188,7 +190,7 @@ stages:
188190
inlineScript: |
189191
set -e # fail on error
190192
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
191-
python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt
193+
python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt
192194
# Output image location to Azure DevOps job
193195
IMAGE_LOCATION="$(cat image_location.txt)"
194196
echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
@@ -207,4 +209,4 @@ stages:
207209
inlineScript: |
208210
set -e # fail on error
209211
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
210-
python ml_service/util/smoke_test_scoring_service.py --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"
212+
python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"

‎.pipelines/diabetes_regression-ci-image.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,10 @@ steps:
2828
inputs:
2929
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
3030
scriptLocation: inlineScript
31+
workingDirectory: $(Build.SourcesDirectory)
3132
inlineScript: |
3233
set -e
3334
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
34-
python3 $(Build.SourcesDirectory)/ml_service/util/create_scoring_image.py
35+
python3 -m ml_service.util.create_scoring_image
3536
displayName: 'Create Scoring Image'
3637

‎.pipelines/azdo-template-get-model-version.yml‎ renamed to ‎.pipelines/diabetes_regression-template-get-model-version.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ steps:
66
inlineScript: |
77
set -e # fail on error
88
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
9-
python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --output_model_version_file "model_version.txt"
9+
python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline --build_id $(Build.BuildId) --output_model_version_file "model_version.txt"
1010
# Output model version to Azure DevOps job
1111
MODEL_VERSION="$(cat model_version.txt)"
1212
echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION"

‎diabetes_regression/evaluate/evaluate_model.py‎

Lines changed: 43 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -24,47 +24,54 @@
2424
POSSIBILITY OF SUCH DAMAGE.
2525
"""
2626
import os
27-
import sys
28-
from azureml.core import Run, Workspace, Experiment
27+
from azureml.core import Run
2928
import argparse
3029
import traceback
30+
from util.model_helper import get_model_by_tag
3131

3232
run = Run.get_context()
33-
if (run.id.startswith('OfflineRun')):
34-
from dotenv import load_dotenv
35-
# For local development, set values in this section
36-
load_dotenv()
37-
sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
38-
if (sources_dir is None):
39-
sources_dir = 'diabetes_regression'
40-
path_to_util = os.path.join(".", sources_dir, "util")
41-
sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402
42-
from model_helper import get_model_by_tag
43-
workspace_name = os.environ.get("WORKSPACE_NAME")
44-
experiment_name = os.environ.get("EXPERIMENT_NAME")
45-
resource_group = os.environ.get("RESOURCE_GROUP")
46-
subscription_id = os.environ.get("SUBSCRIPTION_ID")
47-
tenant_id = os.environ.get("TENANT_ID")
48-
model_name = os.environ.get("MODEL_NAME")
49-
app_id = os.environ.get('SP_APP_ID')
50-
app_secret = os.environ.get('SP_APP_SECRET')
51-
build_id = os.environ.get('BUILD_BUILDID')
52-
# run_id useful to query previous runs
53-
run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70"
5433

55-
aml_workspace = Workspace.get(
56-
name=workspace_name,
57-
subscription_id=subscription_id,
58-
resource_group=resource_group
59-
)
60-
ws = aml_workspace
61-
exp = Experiment(ws, experiment_name)
62-
else:
63-
sys.path.append(os.path.abspath("./util")) # NOQA: E402
64-
from model_helper import get_model_by_tag
65-
exp = run.experiment
66-
ws = run.experiment.workspace
67-
run_id = 'amlcompute'
34+
# if you would like to run this script on a local computer
35+
# the following code is a good starting point for you
36+
# use
37+
# python -m evaluate.evaluate_model
38+
# in diabetes_regression folder context
39+
40+
# if (run.id.startswith('OfflineRun')):
41+
# from dotenv import load_dotenv
42+
# # For local development, set values in this section
43+
# load_dotenv()
44+
# sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
45+
# if (sources_dir is None):
46+
# sources_dir = 'diabetes_regression'
47+
# path_to_util = os.path.join(".", sources_dir, "util")
48+
# sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402
49+
# from model_helper import get_model_by_tag
50+
# workspace_name = os.environ.get("WORKSPACE_NAME")
51+
# experiment_name = os.environ.get("EXPERIMENT_NAME")
52+
# resource_group = os.environ.get("RESOURCE_GROUP")
53+
# subscription_id = os.environ.get("SUBSCRIPTION_ID")
54+
# tenant_id = os.environ.get("TENANT_ID")
55+
# model_name = os.environ.get("MODEL_NAME")
56+
# app_id = os.environ.get('SP_APP_ID')
57+
# app_secret = os.environ.get('SP_APP_SECRET')
58+
# build_id = os.environ.get('BUILD_BUILDID')
59+
# # run_id useful to query previous runs
60+
# run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70"
61+
62+
# aml_workspace = Workspace.get(
63+
# name=workspace_name,
64+
# subscription_id=subscription_id,
65+
# resource_group=resource_group
66+
# )
67+
# ws = aml_workspace
68+
# exp = Experiment(ws, experiment_name)
69+
70+
# comment the following three lines
71+
# if you would like to use Offline mode
72+
exp = run.experiment
73+
ws = run.experiment.workspace
74+
run_id = 'amlcompute'
6875

6976
parser = argparse.ArgumentParser("evaluate")
7077
parser.add_argument(

‎diabetes_regression/util/__init__.py‎

Whitespace-only changes.

‎docs/code_description.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
- `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py` : builds and publishes an ML training pipeline. It uses R on ML Compute.
2525
- `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py` : builds and publishes an ML training pipeline. It uses R on Databricks Compute.
2626
- `ml_service/pipelines/run_train_pipeline.py` : invokes a published ML training pipeline (Python on ML Compute) via REST API.
27-
- `ml_service/pipelines/verify_train_pipeline.py` : determines whether the evaluate_model.py step of the training pipeline registered a new model.
27+
- `ml_service/pipelines/diabetes_regression_verify_train_pipeline.py` : determines whether the evaluate_model.py step of the training pipeline registered a new model.
2828
- `ml_service/util` : contains common utility functions used to build and publish an ML training pipeline.
2929

3030
### Code

‎ml_service/pipelines/__init__.py‎

Whitespace-only changes.

‎ml_service/pipelines/diabetes_regression_build_train_pipeline.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,8 @@
44
from azureml.core import Workspace
55
from azureml.core.runconfig import RunConfiguration, CondaDependencies
66
from azureml.core import Dataset, Datastore
7-
import os
8-
import sys
9-
sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402
10-
from attach_compute import get_compute
11-
from env_variables import Env
7+
from ml_service.util.attach_compute import get_compute
8+
from ml_service.util.env_variables import Env
129

1310

1411
def main():

0 commit comments

Comments
 (0)