From da5b265cca15309fc97a99add15d9b4fa0ef1bb8 Mon Sep 17 00:00:00 2001 From: Calin Timbus Date: Thu, 18 May 2023 08:56:45 +0000 Subject: [PATCH 1/7] configuration update --- .github/workflows/nano.save | 1 + .github/workflows/workshop_unit_test.yml | 2 +- .github/workflows/workshop_unit_test.yml.save | 39 ++++++++++++++++++ src/workshop/core/scoring/deployment.yml | 2 +- src/workshop/core/scoring/endpoint.yml | 2 +- src/workshop/data/linear_regression.joblib | Bin 5731 -> 5843 bytes 6 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/nano.save create mode 100644 .github/workflows/workshop_unit_test.yml.save diff --git a/.github/workflows/nano.save b/.github/workflows/nano.save new file mode 100644 index 00000000..62d8fe9f --- /dev/null +++ b/.github/workflows/nano.save @@ -0,0 +1 @@ +X diff --git a/.github/workflows/workshop_unit_test.yml b/.github/workflows/workshop_unit_test.yml index 3c1382c3..c5208652 100644 --- a/.github/workflows/workshop_unit_test.yml +++ b/.github/workflows/workshop_unit_test.yml @@ -31,7 +31,7 @@ jobs: - name: Install AZ ML and tools run: | # SETUP line 34 to point to your own AML workspace az extension add -n ml -y --version 2.2.1 - az configure --defaults group=azureml workspace=ws01ent location=westus2 + az configure --defaults group=mlops-rg-948695 workspace=aml948695 location=eastus2 - name: Run Feature Engineering uses: ./.github/actions/aml-job-create with: diff --git a/.github/workflows/workshop_unit_test.yml.save b/.github/workflows/workshop_unit_test.yml.save new file mode 100644 index 00000000..57149c83 --- /dev/null +++ b/.github/workflows/workshop_unit_test.yml.save @@ -0,0 +1,39 @@ +name: feature_engineering_unit_test +on: + workflow_dispatch: + push: + branches-ignore: + - main + - integration + paths: + - src/workshop/core/data_engineering/* + - .github/workflows/workshop_unit_test.yml + +jobs: + unit-test: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.8' # Version range or exact version of a Python version to use, using SemVer's version range syntax + - name: Upgrade pip + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade build + python -m pip install --upgrade twine + - name: AZ Login + uses: azure/login@v1 + with: + creds: $} # SETUP: replace AZURE_SERVICE_PRINCIPAL with your own secret name + - name: Install AZ ML and tools + run: | # SETUP line 34 to point to your own AML workspace + az extension add -n ml -y --version 2.2.1 + az configure --defaults group=azureml workspace=ws01ent location=westus2 + - name: Run Feature Engineering + uses: ./.github/actions/aml-job-create + with: + jobFile: src/workshop/core/data_engineering/feature_engineering.yml + diff --git a/src/workshop/core/scoring/deployment.yml b/src/workshop/core/scoring/deployment.yml index 29c3500c..70e5584a 100644 --- a/src/workshop/core/scoring/deployment.yml +++ b/src/workshop/core/scoring/deployment.yml @@ -1,6 +1,6 @@ $schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json name: green -endpoint_name: mlops-workshop-endpoint #setup replace `mlops-workshop-endpoint` with your own endpoint name defined in endpoint.yml +endpoint_name: calin-endpoint #setup replace `mlops-workshop-endpoint` with your own endpoint name defined in endpoint.yml model: azureml:nyc_fare_prediction:1 code_configuration: code: ./ diff --git a/src/workshop/core/scoring/endpoint.yml b/src/workshop/core/scoring/endpoint.yml index 611e0721..2b65b2fa 100644 --- a/src/workshop/core/scoring/endpoint.yml +++ b/src/workshop/core/scoring/endpoint.yml @@ -1,3 +1,3 @@ $schema: https://azuremlschemas.azureedge.net/latest/managedOnlineEndpoint.schema.json -name: mlops-workshop-endpoint #setup replace `mlops-workshop-endpoint` with your own endpoint name. It has to be globally unique +name: calin-endpoint #setup replace `mlops-workshop-endpoint` with your own endpoint name. It has to be globally unique auth_mode: key diff --git a/src/workshop/data/linear_regression.joblib b/src/workshop/data/linear_regression.joblib index d6bd059086fc6a30a98af61118b41c2f0c08a9c8..da6d2c794cd14000cf8453a28efbee985f900a40 100644 GIT binary patch delta 1287 zcmZ`&ZA@EL7;d+I(H6=m9}WfsT0W`>3_5IJaLT5_c0*;rTvhZ^=z&{l@9n+4E$f`3 z!F4NMW8A&q{93?G0%mH)qGXBNi7pG<;$RM=+p=hwUz>?b=D@HyqL)ihX5#sA-aOCq zK6%f1&wG9-n0iq0qM|Q+M3JOOO4%)NN{VqqM&RU|1j0^K6L#LJux4^;WYFgi<1U7A ziM!X+LHZ!c;;ms81_UggZ;tDdr|`7ChR;(DcgM(NG9N{<1TNa>RHt6VZqZ1!;}T7$ zId%V1T@ZHCYb1uCg3Lmem`tZjwl*qAvP>9nquh|raH>Qb!QcT8Nv~Z3ZB&5Hlz=-d zDYmR7v>;_WHBv1WvK}9#P0mCSk`&_;udx%>QeKZ(&EXOyGe4*@mghLNVuGK6Z61Hq zf+JTt8Vh}X$hg=b1L-`8Sxk8t`dNa`hb(R3Z>ZA6F6hL!#(u zAs>@r8Y~7&z>JW1YyrzL1*XJQm>Sb!I!uqPf@#u`U%e`scf@IzOS6k~LdI@^z=;-dT9zO5=x>Hbxs;K) z+|?k`UZbb6tSAq%}8QjFrnxS_WYXFTo>D9&et6nVk&lJLBc&naDA zKj9L#3`FEIe$HrZ;xsSSUCM54Mie;{S|@5amIlse7r@y2VaQ`<1D-!>xIpz&Uc~vY~tw zTpXIf{}|tn#)8w=2U|j*-!GqhTeS~8Wit$y{CuJ6%%8d=pH!rw>C|sOFCGOz-ev^fEJie>+SWI4dOOV3U7K zwH!sv2NE@bR@DZ`sgT$!d7Ar4S81 zS3Rx^m!T{3J?Gmu-mc22aH6%ReyutY>@WMO)Q%3Hs;Z$(DPVQvSpJ#U%%DUT{}Hny z3t61qQyVTT&FGQdlEA^2bJ6;{qmwf`)&uqQ$nfNBHu&k>n&#pnD|##aO<`}H9VFfe zHBAunCu*m^(4f*(l+&V^vR*4hmGkXmJp)!$KYs51LkBCs$|rxZT<^|CllzO}8+x+9 zMQwTWS@UMlc;D1)gtH-c@2$d_Pg6i*>CDOF6O8~^-hC)~_4YboB)87>=RW$`QtWFL vf9=BJ)TH$MxZvRR$yU>y(=LxGw!kh#ga(1gB(j375djh=2ayq7goS?tQ}gPh delta 1185 zcmZXTeNYr-9LHHYUiOYRc~d|T1m*CBprBCHJp{q=QBbUzwF#Hw?m2eHK3I6 z2oG>EszS3)UP;MeZiTwc#5!#PUrCu*Gvs1Pr76u*4#wu<;ugrnI=Gc8m@5^Fi(df6 zNg#dIUIG-DGRUnWLuItFe3{Y8yaKngRz~iYZ1qx$81H6DY@BzL62Y$dG1b`6ebn7S^Wr1oZhXf+Tw|rjL zK$@;sjzc`}7C7dL9T#pD`Y8>Sa;>yrL0Z5Mm| zc4DNs)PLWiJmbF~TOm?Q!UMc(B$}d&UcB*0_EajoY!LT+-l#-4}uS$9`E;KEK z-jnb4wX+UJV9KFXPctA@{2(;mrF?qEj`#lR+`NnUJ@ey~qG7FJvTD@+QtM^(g=ch6 zYSKH%y``%&OQ<0FerpY{y8k1(+cA~;!_8Z0LtB3`eqoe2m~o=}p5ZzQ{4?=GOVf3v z#q>%g=-$}ep_=beR14X1?%4^{umk>)7W@&h?ex_*dA@4W14O1yQA_~zJ|L&1Y1 z1b*K2)k*!gXrdgunCWu^9eMQb*x*DvDt-oJZ#0b(SkqKO?Y3d09f{1%qGpKv`DSq~ z8Xy`i!lc-GpO`=OB|AGkh|cUSKbqq=gf@Mbf~QVRp}L6q(wE!EiQ>7Uz{!y=0&gD= z&fGss^d5IzHJ^KN7@dYa4`R=D6PxO?yKY+Nh{FlJR7ulMgzs6Yb$;?8(bZSI<&8Uc z3CXkN(-`IfI<|Gg!yEkr=mrghIBYW7vHu~ZmO5gha z?qy$hqs=j^S>gOR@mfR1=lIcIQDM}4Zu*@$q*>2*{L)g8a8>i9He2QYM!DQLC7&Z> yc0z}gXUO%+DJ||`#lzug@hjh)S$FACs6ncg3MD$0j&qjON|27vr(^0LWAfjBGSR^R From 0c4b325a82b54c87074b6526afed9763189c09bb Mon Sep 17 00:00:00 2001 From: Calin Timbus Date: Thu, 18 May 2023 09:34:14 +0000 Subject: [PATCH 2/7] New push --- .github/workflows/workshop_unit_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/workshop_unit_test.yml b/.github/workflows/workshop_unit_test.yml index c5208652..5214d3ef 100644 --- a/.github/workflows/workshop_unit_test.yml +++ b/.github/workflows/workshop_unit_test.yml @@ -1,3 +1,4 @@ +#the change name: feature_engineering_unit_test on: workflow_dispatch: From 5c1765d192777544720c1e0170e2a033507dc769 Mon Sep 17 00:00:00 2001 From: Calin Timbus Date: Thu, 18 May 2023 09:44:11 +0000 Subject: [PATCH 3/7] Modify ml_training ridge value to trigger pipeline(cd) --- src/workshop/core/training/.amlignore | 6 ++++++ src/workshop/core/training/.amlignore.amltmp | 6 ++++++ src/workshop/core/training/ml_training.py | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 src/workshop/core/training/.amlignore create mode 100644 src/workshop/core/training/.amlignore.amltmp diff --git a/src/workshop/core/training/.amlignore b/src/workshop/core/training/.amlignore new file mode 100644 index 00000000..0621f9fc --- /dev/null +++ b/src/workshop/core/training/.amlignore @@ -0,0 +1,6 @@ +## This file was auto generated by the Azure Machine Learning Studio. Please do not remove. +## Read more about the .amlignore file here: https://docs.microsoft.com/azure/machine-learning/how-to-save-write-experiment-files#storage-limits-of-experiment-snapshots + +.ipynb_aml_checkpoints/ +*.amltmp +*.amltemp \ No newline at end of file diff --git a/src/workshop/core/training/.amlignore.amltmp b/src/workshop/core/training/.amlignore.amltmp new file mode 100644 index 00000000..0621f9fc --- /dev/null +++ b/src/workshop/core/training/.amlignore.amltmp @@ -0,0 +1,6 @@ +## This file was auto generated by the Azure Machine Learning Studio. Please do not remove. +## Read more about the .amlignore file here: https://docs.microsoft.com/azure/machine-learning/how-to-save-write-experiment-files#storage-limits-of-experiment-snapshots + +.ipynb_aml_checkpoints/ +*.amltmp +*.amltemp \ No newline at end of file diff --git a/src/workshop/core/training/ml_training.py b/src/workshop/core/training/ml_training.py index 6f59dcdd..c85b8ad5 100644 --- a/src/workshop/core/training/ml_training.py +++ b/src/workshop/core/training/ml_training.py @@ -43,7 +43,7 @@ def createClassModel(algo_name, catg, nums): #--------------------------------------------- #setup: Update alpha value #--------------------------------------------- - model = Ridge(alpha=100000) #setup + model = Ridge(alpha=100) #setup elif algo_name == 'random_forest': model = RandomForestRegressor() else: @@ -100,4 +100,4 @@ def main(args): # parse args args = parse_args() # run main function - main(args) \ No newline at end of file + main(args) From 3cadf1e04b93390166031cc2d4049398885d25b8 Mon Sep 17 00:00:00 2001 From: Calin Timbus Date: Thu, 18 May 2023 10:15:19 +0000 Subject: [PATCH 4/7] Fix modification --- .github/workflows/workshop_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workshop_ci.yml b/.github/workflows/workshop_ci.yml index 09c0ea72..3213ff85 100644 --- a/.github/workflows/workshop_ci.yml +++ b/.github/workflows/workshop_ci.yml @@ -32,7 +32,7 @@ jobs: - name: Install az ml & set default values for AML run: | #setup: provide group, workspace and location az extension add -n ml -y --version 2.2.1 - az configure --defaults group=azureml workspace=ws01ent location=westus2 + az configure --defaults group=mlops-rg-948695 workspace=aml948695 location=eastus2 - name: run training and model validation run: | az ml job create -s -f src/workshop/core/pipelines/training_pipeline.yml From 381665342580d1378ac1574763cd8811e6a7071e Mon Sep 17 00:00:00 2001 From: Calin Timbus Date: Thu, 18 May 2023 10:45:18 +0000 Subject: [PATCH 5/7] Update for the CD pipeline --- .github/workflows/workshop_cd.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/workshop_cd.yml b/.github/workflows/workshop_cd.yml index df3e7214..852f5456 100644 --- a/.github/workflows/workshop_cd.yml +++ b/.github/workflows/workshop_cd.yml @@ -42,8 +42,8 @@ jobs: - name: Run deployment uses: ./.github/actions/aml-endpoint-deploy with: - resourceGroup: azureml #setup replace azureml with the name of your resource group in Azure - workspaceName: ws01ent #setup replace ws01ent with the name of your workspace in Azure + resourceGroup: mlops-rg-948695 #setup replace azureml with the name of your resource group in Azure + workspaceName: aml948695 #setup replace ws01ent with the name of your workspace in Azure endpointFile: src/workshop/core/scoring/endpoint.yml deploymentFile: src/workshop/core/scoring/deployment.yml modelVersion: latest @@ -51,14 +51,14 @@ jobs: - name: Test deployment uses: ./.github/actions/aml-endpoint-test with: - resourceGroup: azureml #setup replace azureml with the name of your resource group in Azure - workspaceName: ws01ent #setup replace ws01ent with the name of your workspace in Azure + resourceGroup: mlops-rg-948695 #setup replace azureml with the name of your resource group in Azure + workspaceName: aml948696 #setup replace ws01ent with the name of your workspace in Azure endpointFile: src/workshop/core/scoring/endpoint.yml requestFile: src/workshop/core/scoring/scoring_test_request.json - name: Swap deployment uses: ./.github/actions/aml-endpoint-swap with: - resourceGroup: azureml #setup replace azureml with the name of your resource group in Azure - workspaceName: ws01ent #setup replace ws01ent with the name of your workspace in Azure - endpointFile: src/workshop/core/scoring/endpoint.yml \ No newline at end of file + resourceGroup: mlops-rg-948695 #setup replace azureml with the name of your resource group in Azure + workspaceName: aml948695 #setup replace ws01ent with the name of your workspace in Azure + endpointFile: src/workshop/core/scoring/endpoint.yml From a34d69cd9aed7446d46d1e6d224886e8e84afd6b Mon Sep 17 00:00:00 2001 From: Calin Timbus Date: Thu, 18 May 2023 11:01:23 +0000 Subject: [PATCH 6/7] Fix small bug in workspace name --- .github/workflows/workshop_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workshop_cd.yml b/.github/workflows/workshop_cd.yml index 852f5456..ea6c3038 100644 --- a/.github/workflows/workshop_cd.yml +++ b/.github/workflows/workshop_cd.yml @@ -52,7 +52,7 @@ jobs: uses: ./.github/actions/aml-endpoint-test with: resourceGroup: mlops-rg-948695 #setup replace azureml with the name of your resource group in Azure - workspaceName: aml948696 #setup replace ws01ent with the name of your workspace in Azure + workspaceName: aml948695 #setup replace ws01ent with the name of your workspace in Azure endpointFile: src/workshop/core/scoring/endpoint.yml requestFile: src/workshop/core/scoring/scoring_test_request.json From 42fcd9131ae3362518261740c62e8cdb8ceec775 Mon Sep 17 00:00:00 2001 From: Calin Timbus Date: Thu, 18 May 2023 11:15:35 +0000 Subject: [PATCH 7/7] New stuffm --- .github/workflows/workshop_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/workshop_ci.yml b/.github/workflows/workshop_ci.yml index 3213ff85..be62ada8 100644 --- a/.github/workflows/workshop_ci.yml +++ b/.github/workflows/workshop_ci.yml @@ -1,3 +1,4 @@ +#random comment name: workshop-ci on: workflow_dispatch: