diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index bea64f5ea2eb9..fa254912c2533 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -70,6 +70,10 @@ resources: type: github endpoint: ort-examples name: microsoft/onnxruntime-inference-examples + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release variables: - template: templates/common-variables.yml @@ -82,210 +86,140 @@ variables: value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda11 }} - name: win_cuda_home value: $(Agent.TempDirectory)\v11.8 - -stages: -- template: stages/set_packaging_variables_stage.yml - parameters: - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }} - PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }} - -- template: stages/download-java-tools-stage.yml - -- template: templates/c-api-cpu.yml - parameters: - RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - ${{ if eq(parameters.NugetPackageSuffix, 'NONE') }}: - OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime' - ${{ else }}: - OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime${{ parameters.NugetPackageSuffix }}' - AdditionalBuildFlags: '' - AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos ${{parameters.AdditionalBuildFlag}}' - BuildVariant: 'default' - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - QnnSDKVersion: ${{ parameters.QnnSdk }} - -- template: stages/java-cuda-packaging-stage.yml - parameters: - CudaVersion: 11.8 - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - -- template: stages/nuget-combine-cuda-stage.yml - parameters: - DoCompliance: ${{ parameters.DoCompliance }} - CudaVersion: 11.8 - RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }} - win_trt_home: ${{ variables.win_trt_home }} - win_cuda_home: ${{ variables.win_cuda_home }} - DoEsrp: ${{ parameters.DoEsrp }} - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - buildJava: true - buildNodejs: true - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - - -- template: nuget/templates/dml-vs-2022.yml - parameters: - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - ArtifactName: 'drop-nuget-dml' - StageName: 'Windows_CI_GPU_DML_Dev' - BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --build_nodejs --cmake_generator "Visual Studio 17 2022" --use_vcpkg - BuildArch: 'x64' - msbuildArchitecture: 'amd64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'x64' - DoDebugBuild: 'false' - DoNugetPack: 'true' - DoCompliance: 'false' - DoEsrp: ${{ parameters.DoEsrp }} - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} /p:CurrentData=$(BuildDate) /p:CurrentTime=$(BuildTime) - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: nuget/templates/dml-vs-2022.yml - parameters: - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - ArtifactName: 'drop-win-dml-x86-zip' - StageName: 'Windows_CI_GPU_DML_Dev_x86' - BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --cmake_generator "Visual Studio 17 2022" --use_vcpkg - BuildArch: 'x86' - EnvSetupScript: 'setup_env_x86.bat' - sln_platform: 'Win32' - DoDebugBuild: 'false' - DoNugetPack: 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - RunTests: 'false' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-x86.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-x86.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - -- template: nuget/templates/dml-vs-2022.yml +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - ArtifactName: 'drop-win-dml-arm64-zip' - StageName: 'Windows_CI_GPU_DML_Dev_arm64' - BuildCommand: --build_dir $(Build.BinariesDirectory) --arm64 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --build_nodejs --cmake_generator "Visual Studio 17 2022" --use_vcpkg - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'arm64' - DoDebugBuild: 'false' - DoNugetPack: 'true' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - RunTests: 'false' - NuPackScript: | - msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm64 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} - cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ - ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-arm64.zip - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-arm64.zip $(Build.ArtifactStagingDirectory) - mkdir $(Build.ArtifactStagingDirectory)\testdata - copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata -- stage: NuGet_Packaging_DML - dependsOn: - - Windows_CI_GPU_DML_Dev - - Windows_CI_GPU_DML_Dev_x86 - - Windows_CI_GPU_DML_Dev_arm64 - condition: succeeded() - jobs: - - job: NuGet_Packaging_DML - workspace: - clean: all - pool: 'onnxruntime-Win2022-GPU-dml-A10' - steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet DirectML' - inputs: - artifactName: 'drop-nuget-dml' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml' + sdl: + sourceAnalysisPool: + name: onnxruntime-Win-CPU-2022 + os: windows + sourceRepositoriesToScan: + exclude: + - repository: onnxruntime-inference-examples - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet DirectML x86' - inputs: - artifactName: 'drop-win-dml-x86-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - NuGet DirectML arm64' - inputs: - artifactName: 'drop-win-dml-arm64-zip' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml' - - - script: | - pushd $(Build.BinariesDirectory)\nuget-artifact-dml - dir - powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe" - powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe" - set PATH=%CD%;%PATH% - SETLOCAL EnableDelayedExpansion - FOR /R %%i IN (*.nupkg) do ( - set filename=%%~ni - IF NOT "!filename:~25,7!"=="Managed" ( - rename %%~ni.nupkg %%~ni.zip - unzip %%~ni.zip -d %%~ni - del /Q %%~ni.zip - - unzip win-dml-x86.zip -d win-x86 - mkdir %%~ni\runtimes\win-x86 - mkdir %%~ni\runtimes\win-x86\native + stages: + - template: stages/set_packaging_variables_stage.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }} + PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }} - move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll - move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib - move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb + - template: stages/download-java-tools-stage.yml - unzip win-dml-arm64.zip -d win-arm64 - mkdir %%~ni\runtimes\win-arm64 - mkdir %%~ni\runtimes\win-arm64\native + - template: templates/c-api-cpu.yml + parameters: + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + ${{ if eq(parameters.NugetPackageSuffix, 'NONE') }}: + OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime' + ${{ else }}: + OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime${{ parameters.NugetPackageSuffix }}' + AdditionalBuildFlags: '' + AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos ${{parameters.AdditionalBuildFlag}}' + BuildVariant: 'default' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + QnnSDKVersion: ${{ parameters.QnnSdk }} + + - template: stages/java-cuda-packaging-stage.yml + parameters: + CudaVersion: 11.8 + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} - move win-arm64\runtimes\win-arm64\native\onnxruntime.dll %%~ni\runtimes\win-arm64\native\onnxruntime.dll - move win-arm64\runtimes\win-arm64\native\onnxruntime.lib %%~ni\runtimes\win-arm64\native\onnxruntime.lib - move win-arm64\runtimes\win-arm64\native\onnxruntime.pdb %%~ni\runtimes\win-arm64\native\onnxruntime.pdb + - template: stages/nuget-combine-cuda-stage.yml + parameters: + DoCompliance: ${{ parameters.DoCompliance }} + CudaVersion: 11.8 + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }} + win_trt_home: ${{ variables.win_trt_home }} + win_cuda_home: ${{ variables.win_cuda_home }} + DoEsrp: ${{ parameters.DoEsrp }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + buildJava: true + buildNodejs: true + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + + - template: nuget/templates/dml-vs-2022.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + ArtifactName: 'drop-nuget-dml' + StageName: 'Windows_CI_GPU_DML_Dev' + BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --build_nodejs --cmake_generator "Visual Studio 17 2022" --use_vcpkg + BuildArch: 'x64' + msbuildArchitecture: 'amd64' + EnvSetupScript: 'setup_env.bat' + sln_platform: 'x64' + DoDebugBuild: 'false' + DoNugetPack: 'true' + DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} /p:CurrentData=$(BuildDate) /p:CurrentTime=$(BuildTime) + copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + mkdir $(Build.ArtifactStagingDirectory)\testdata + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata - pushd %%~ni - zip -r ..\%%~ni.zip . - popd - move %%~ni.zip %%~ni.nupkg - ) - ) - popd - copy $(Build.BinariesDirectory)\nuget-artifact-dml\Microsoft.ML.OnnxRuntime.DirectML*nupkg $(Build.ArtifactStagingDirectory) - displayName: 'Bundle DML NuGet and other binaries' + - - template: templates/esrp_nuget.yml + - template: nuget/templates/dml-vs-2022.yml parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.ArtifactStagingDirectory)' + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + ArtifactName: 'drop-win-dml-x86-zip' + StageName: 'Windows_CI_GPU_DML_Dev_x86' + BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --cmake_generator "Visual Studio 17 2022" --use_vcpkg + BuildArch: 'x86' + EnvSetupScript: 'setup_env_x86.bat' + sln_platform: 'Win32' + DoDebugBuild: 'false' + DoNugetPack: 'true' + DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} - - - template: templates/validate-package.yml + RunTests: 'false' + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} + cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ + ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-x86.zip + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-x86.zip $(Build.ArtifactStagingDirectory) + mkdir $(Build.ArtifactStagingDirectory)\testdata + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata + + + + - template: nuget/templates/dml-vs-2022.yml parameters: - PackageType: 'nuget' - PackagePath: '$(Build.ArtifactStagingDirectory)' - PackageName: 'Microsoft.ML.OnnxRuntime.DirectML*nupkg' - PlatformsSupported: 'win-x64,win-x86,win-arm64' - VerifyNugetSigning: ${{ parameters.DoEsrp }} - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget-dml' - targetPath: '$(Build.ArtifactStagingDirectory)' - - template: templates/component-governance-component-detection-steps.yml + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + ArtifactName: 'drop-win-dml-arm64-zip' + StageName: 'Windows_CI_GPU_DML_Dev_arm64' + BuildCommand: --build_dir $(Build.BinariesDirectory) --arm64 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --build_nodejs --cmake_generator "Visual Studio 17 2022" --use_vcpkg + BuildArch: 'x64' + EnvSetupScript: 'setup_env.bat' + sln_platform: 'arm64' + DoDebugBuild: 'false' + DoNugetPack: 'true' + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + RunTests: 'false' + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm64 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} + cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ + ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-arm64.zip + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-arm64.zip $(Build.ArtifactStagingDirectory) + mkdir $(Build.ArtifactStagingDirectory)\testdata + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata + + + + - template: stages/nuget_dml_packaging_stage.yml parameters: - condition: 'succeeded' + DoEsrp: ${{ parameters.DoEsrp }} diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml index 4417ebce60989..fff19e69a0e46 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml @@ -189,8 +189,8 @@ jobs: - task: PublishBuildArtifacts@1 inputs: - pathtoPublish: '$(Build.SourcesDirectory)/Artifact' - artifactName: 'result-$(Build.BuildNumber)' + path: '$(Build.SourcesDirectory)/Artifact' + artifact: 'result-$(Build.BuildNumber)' - template: templates/component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/azure-pipelines/nodejs/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nodejs/templates/test_linux.yml index 1d3e92056ebe2..c4102f5ba2b49 100644 --- a/tools/ci_build/github/azure-pipelines/nodejs/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nodejs/templates/test_linux.yml @@ -10,8 +10,10 @@ stages: - job: Nodejs_Test_${{ parameters.StageSuffix }} workspace: clean: all - timeoutInMinutes: 120 - pool: ${{ parameters.AgentPool }} + timeoutInMinutes: 120 + pool: + name: ${{ parameters.AgentPool }} + os: 'linux' variables: - name: OnnxRuntimeBuildDirectory diff --git a/tools/ci_build/github/azure-pipelines/nodejs/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nodejs/templates/test_macos.yml index 4518a168879a2..42ce700b89e89 100644 --- a/tools/ci_build/github/azure-pipelines/nodejs/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nodejs/templates/test_macos.yml @@ -9,9 +9,11 @@ stages: - job: Nodejs_Test_MacOS_${{ parameters.StageSuffix }} workspace: clean: all - timeoutInMinutes: 120 + timeoutInMinutes: 120 pool: - vmImage: 'macOS-13' + name: 'Azure Pipelines' + image: 'macOS-13' + os: 'macOS' variables: - name: OnnxRuntimeBuildDirectory diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml index cf1a2809081c9..ec7b665f864a0 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml @@ -144,7 +144,7 @@ stages: displayName: 'Create NuGet Package' failOnStderr: true - - task: PublishPipelineArtifact@0 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Pipeline Artifact: ${{ parameters.ArtifactName }}' inputs: artifactName: ${{ parameters.ArtifactName }} @@ -182,7 +182,7 @@ stages: workingDirectory: '$(Build.SourcesDirectory)\js\node' displayName: 'Create NPM Package' - - task: PublishPipelineArtifact@0 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Pipeline Artifact: ${{ parameters.ArtifactName }}' inputs: artifactName: ${{ parameters.ArtifactName }} @@ -208,7 +208,7 @@ stages: DisplayName: 'ESRP - Sign Node.js binding binaries' DoEsrp: ${{ parameters.DoEsrp }} Pattern: '*.node' - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\${{ parameters.sln_platform }}' artifactName: 'drop-onnxruntime-nodejs-win-${{ parameters.sln_platform }}-dml' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index f9ecfb7cf7938..486f70a8f6b49 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -1,7 +1,7 @@ parameters: AgentPool: 'onnxruntime-Ubuntu2204-AMD-CPU' ArtifactSuffix: '' - NugetPackageName : '' + NugetPackageName: '' StageSuffix: 'CPU' # More Suffix is used to differentiate testing for GPU and GPU-Windows/GPU-Linux packages MoreSuffix: '' @@ -20,7 +20,9 @@ stages: workspace: clean: all timeoutInMinutes: 120 - pool: ${{ parameters.AgentPool }} + pool: + name: ${{ parameters.AgentPool }} + os: 'linux' variables: - template: ../../templates/common-variables.yml @@ -91,24 +93,24 @@ stages: " displayName: 'Run Package Test' - ${{ else }}: - - task: CmdLine@2 - displayName: 'Create symlink for test models' - inputs: - script: | - ln -sf /data/models $(Build.BinariesDirectory) - - task: Bash@3 - displayName: 'Run Package Test' - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh' - arguments: '$(Build.BinariesDirectory)/nuget-artifact $(NuGetPackageVersionNumber)' - workingDirectory: $(Build.BinariesDirectory) - env: - OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) - DisableContribOps: $(DisableContribOps) - DisableMlOps: $(DisableMlOps) - IsReleaseBuild: $(IsReleaseBuild) - PACKAGENAME: ${{ parameters.NugetPackageName }} + - task: CmdLine@2 + displayName: 'Create symlink for test models' + inputs: + script: | + ln -sf /data/models $(Build.BinariesDirectory) + - task: Bash@3 + displayName: 'Run Package Test' + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh' + arguments: '$(Build.BinariesDirectory)/nuget-artifact $(NuGetPackageVersionNumber)' + workingDirectory: $(Build.BinariesDirectory) + env: + OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) + DisableContribOps: $(DisableContribOps) + DisableMlOps: $(DisableMlOps) + IsReleaseBuild: $(IsReleaseBuild) + PACKAGENAME: ${{ parameters.NugetPackageName }} - template: ../../templates/component-governance-component-detection-steps.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml index 07d21333270a8..52ed380909d31 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml @@ -1,5 +1,5 @@ parameters: - IsMacOS : 'true' + IsMacOS: 'true' ArtifactSuffix: '' stages: - stage: NuGet_Test_MacOS @@ -11,7 +11,9 @@ stages: workspace: clean: all pool: - vmImage: 'macOS-13' + name: 'Azure Pipelines' + image: 'macOS-13' + os: 'macOS' variables: - name: OnnxRuntimeBuildDirectory @@ -35,24 +37,24 @@ stages: - script: | echo "TODO: Enable this test once fix this nuget test issue" - # $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ - # $(Build.BinariesDirectory)/nuget-artifact \ - # $(NuGetPackageVersionNumber) \ - # ${{ parameters.IsMacOS }} - # - # if [ $? -ne 0 ]; then - # echo "Failed to run test" - # exit 1 - # fi + # $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ + # $(Build.BinariesDirectory)/nuget-artifact \ + # $(NuGetPackageVersionNumber) \ + # ${{ parameters.IsMacOS }} + # + # if [ $? -ne 0 ]; then + # echo "Failed to run test" + # exit 1 + # fi displayName: 'Run Test' env: - OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) - DisableContribOps: $(DisableContribOps) - DisableMlOps: $(DisableMlOps) - IsReleaseBuild: $(IsReleaseBuild) + OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) + DisableContribOps: $(DisableContribOps) + DisableMlOps: $(DisableMlOps) + IsReleaseBuild: $(IsReleaseBuild) - template: ../../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'always' + parameters: + condition: 'always' - template: ../../templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/stages/download-java-tools-stage.yml b/tools/ci_build/github/azure-pipelines/stages/download-java-tools-stage.yml index 49d73c699e7b7..7e5237b32bce2 100644 --- a/tools/ci_build/github/azure-pipelines/stages/download-java-tools-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/download-java-tools-stage.yml @@ -5,6 +5,7 @@ stages: - job: Download_Java_Tools pool: name: 'onnxruntime-Ubuntu2204-AMD-CPU' + os: linux steps: - checkout: none - task: CmdLine@2 @@ -17,11 +18,13 @@ stages: wget --tries=3 https://oss.sonatype.org/service/local/repositories/releases/content/com/google/protobuf/protobuf-java/3.21.7/protobuf-java-3.21.7.jar -P ./ popd workingDirectory: '$(Agent.TempDirectory)' - - task: PublishPipelineArtifact@1 + + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Pipeline Java Tools Artifact' inputs: targetPath: '$(Agent.TempDirectory)/java-tools' artifact: 'onnxruntime-java-tools' + - template: ../templates/component-governance-component-detection-steps.yml parameters : condition : 'succeeded' \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml index b458d3045b1cf..af40d9bcdc202 100644 --- a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml @@ -73,10 +73,10 @@ stages: SourceFolder: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Pipeline Artifact' inputs: - targetPath: '$(Build.ArtifactStagingDirectory)' + path: '$(Build.ArtifactStagingDirectory)' artifact: 'onnxruntime-java-gpu' - template: ../templates/component-governance-component-detection-steps.yml @@ -136,7 +136,9 @@ stages: Jar_Packaging_GPU workspace: clean: all - pool: 'Onnxruntime-Linux-GPU' + pool: + name: 'Onnxruntime-Linux-GPU-A10' + os: linux variables: - name: runCodesignValidationInjection value: false diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml index 8fabb80a73869..ac27357b8ed62 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml @@ -218,7 +218,7 @@ stages: parameters: condition: 'succeeded' - - task: PublishPipelineArtifact@0 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Pipeline NuGet Artifact' inputs: artifactName: 'drop-signed-nuget-GPU' diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml index 73728ecabe920..126fe50291296 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml @@ -15,7 +15,9 @@ stages: workspace: clean: all timeoutInMinutes: 150 - pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + pool: + name: 'onnxruntime-Ubuntu2204-AMD-CPU' + os: linux variables: - name: CUDA_VERSION_MAJOR ${{ if eq(parameters.CudaVersion, '11.8') }}: @@ -66,7 +68,9 @@ stages: workspace: clean: all timeoutInMinutes: 180 - pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + pool: + name: 'onnxruntime-Ubuntu2204-AMD-CPU' + os: linux variables: - template: ../templates/common-variables.yml - name: CUDA_VERSION_MAJOR @@ -138,7 +142,9 @@ stages: condition: succeeded() workspace: clean: all - pool: 'Onnxruntime-Linux-GPU' + pool: + name: 'Onnxruntime-Linux-GPU-A10' + os: linux variables: - template: ../templates/common-variables.yml - name: CUDA_VERSION_MAJOR @@ -218,7 +224,7 @@ stages: /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet/run_capi_application.sh -o /src_dir/onnxruntime -p /artifact_src/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz -w /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet workingDirectory: '$(Build.ArtifactStagingDirectory)' - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: '$(Build.ArtifactStagingDirectory)/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz' artifactName: 'onnxruntime-linux-x64-gpu' diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-win-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-win-cuda-packaging-stage.yml index 095ac9e708ed0..36fb455d8ad94 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-win-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-win-cuda-packaging-stage.yml @@ -160,7 +160,7 @@ stages: arguments: $(Build.SourcesDirectory)\onnxruntime $(Build.ArtifactStagingDirectory)\onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet workingFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishPipelineArtifact@0 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Pipeline Combined GPU Package Artifact' inputs: artifactName: 'onnxruntime-win-x64-gpu' diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget_dml_packaging_stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget_dml_packaging_stage.yml new file mode 100644 index 0000000000000..51ccef1cca6c8 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/stages/nuget_dml_packaging_stage.yml @@ -0,0 +1,99 @@ +parameters: +- name: DoEsrp + type: boolean + default: true + +stages: +- stage: NuGet_Packaging_DML + dependsOn: + - Windows_CI_GPU_DML_Dev + - Windows_CI_GPU_DML_Dev_x86 + - Windows_CI_GPU_DML_Dev_arm64 + condition: succeeded() + jobs: + - job: NuGet_Packaging_DML + workspace: + clean: all + pool: 'onnxruntime-Win2022-GPU-dml-A10' + steps: + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML' + inputs: + artifactName: 'drop-nuget-dml' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML x86' + inputs: + artifactName: 'drop-win-dml-x86-zip' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML arm64' + inputs: + artifactName: 'drop-win-dml-arm64-zip' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml' + + - script: | + pushd $(Build.BinariesDirectory)\nuget-artifact-dml + dir + powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe" + powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe" + set PATH=%CD%;%PATH% + SETLOCAL EnableDelayedExpansion + FOR /R %%i IN (*.nupkg) do ( + set filename=%%~ni + IF NOT "!filename:~25,7!"=="Managed" ( + rename %%~ni.nupkg %%~ni.zip + unzip %%~ni.zip -d %%~ni + del /Q %%~ni.zip + + unzip win-dml-x86.zip -d win-x86 + mkdir %%~ni\runtimes\win-x86 + mkdir %%~ni\runtimes\win-x86\native + + move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll + move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib + move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb + + unzip win-dml-arm64.zip -d win-arm64 + mkdir %%~ni\runtimes\win-arm64 + mkdir %%~ni\runtimes\win-arm64\native + + move win-arm64\runtimes\win-arm64\native\onnxruntime.dll %%~ni\runtimes\win-arm64\native\onnxruntime.dll + move win-arm64\runtimes\win-arm64\native\onnxruntime.lib %%~ni\runtimes\win-arm64\native\onnxruntime.lib + move win-arm64\runtimes\win-arm64\native\onnxruntime.pdb %%~ni\runtimes\win-arm64\native\onnxruntime.pdb + + + pushd %%~ni + zip -r ..\%%~ni.zip . + popd + move %%~ni.zip %%~ni.nupkg + ) + ) + popd + copy $(Build.BinariesDirectory)\nuget-artifact-dml\Microsoft.ML.OnnxRuntime.DirectML*nupkg $(Build.ArtifactStagingDirectory) + displayName: 'Bundle DML NuGet and other binaries' + + - template: ../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.ArtifactStagingDirectory)' + DoEsrp: ${{ parameters.DoEsrp }} + + - template: ../templates/validate-package.yml + parameters: + PackageType: 'nuget' + PackagePath: '$(Build.ArtifactStagingDirectory)' + PackageName: 'Microsoft.ML.OnnxRuntime.DirectML*nupkg' + PlatformsSupported: 'win-x64,win-x86,win-arm64' + VerifyNugetSigning: ${{ parameters.DoEsrp }} + + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Pipeline NuGet Artifact' + inputs: + artifactName: 'drop-signed-nuget-dml' + targetPath: '$(Build.ArtifactStagingDirectory)' + - template: ../templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/stages/set_packaging_variables_stage.yml b/tools/ci_build/github/azure-pipelines/stages/set_packaging_variables_stage.yml index 2e4a673d72e00..8013396018ffa 100644 --- a/tools/ci_build/github/azure-pipelines/stages/set_packaging_variables_stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/set_packaging_variables_stage.yml @@ -8,6 +8,7 @@ stages: - job: Set_Variables pool: name: 'onnxruntime-Ubuntu2204-AMD-CPU' + os: 'linux' steps: - checkout: none - bash: | @@ -48,6 +49,7 @@ stages: - job: D1 pool: name: 'onnxruntime-Ubuntu2204-AMD-CPU' + os: 'linux' variables: MyVar: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']] BuildDate: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']] diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml index eef10d73513d9..9433cad1a6b1a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml @@ -21,7 +21,9 @@ parameters: jobs: - job: Final_AAR_Testing_Android - pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + pool: + name: 'onnxruntime-Ubuntu2204-AMD-CPU' + os: linux workspace: clean: all variables: @@ -47,11 +49,11 @@ jobs: - template: use-android-ndk.yml - script: | - set -e -x - mkdir -p android_test/android/app/libs - cd android_test/android - cp -av $(Build.SourcesDirectory)/java/src/test/android/* ./ - cp $(Build.BinariesDirectory)/final-android-aar/${{parameters.packageName}}-$(OnnxRuntimeVersion)${{parameters.ReleaseVersionSuffix}}.aar app/libs/${{parameters.packageName}}.aar + set -e -x + mkdir -p android_test/android/app/libs + cd android_test/android + cp -av $(Build.SourcesDirectory)/java/src/test/android/* ./ + cp $(Build.BinariesDirectory)/final-android-aar/${{parameters.packageName}}-$(OnnxRuntimeVersion)${{parameters.ReleaseVersionSuffix}}.aar app/libs/${{parameters.packageName}}.aar displayName: Copy Android test files and AAR to android_test directory workingDirectory: $(Build.BinariesDirectory) @@ -83,9 +85,9 @@ jobs: displayName: Trim QNN SDK version to major.minor.patch - script: | - set -e -x - # build apks for qnn package as they are not built in the emulator test step - $(Build.SourcesDirectory)/java/gradlew --no-daemon clean assembleDebug assembleAndroidTest -DqnnVersion=$(QnnMavenPackageVersion) --stacktrace + set -e -x + # build apks for qnn package as they are not built in the emulator test step + $(Build.SourcesDirectory)/java/gradlew --no-daemon clean assembleDebug assembleAndroidTest -DqnnVersion=$(QnnMavenPackageVersion) --stacktrace displayName: Build QNN APK workingDirectory: $(Build.BinariesDirectory)/android_test/android @@ -109,6 +111,6 @@ jobs: BROWSERSTACK_TOKEN: $(browserstack_access_key) - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml index 15a984d688e32..695d3968cc311 100644 --- a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml @@ -58,7 +58,10 @@ jobs: timeoutInMinutes: 120 workspace: clean: all - pool: ${{parameters.pool_name}} + pool: + name: ${{parameters.pool_name}} + os: Linux + variables: artifacts_directory: $(Build.BinariesDirectory)/.artifacts @@ -136,7 +139,7 @@ jobs: parameters: JarFileDirectory: '$(artifacts_directory)' - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: - pathtoPublish: '$(artifacts_directory)' - artifactName: '${{parameters.artifactName}}' + path: '$(artifacts_directory)' + artifact: '${{parameters.artifactName}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml index 602510424c1a7..7288d86d5e464 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml @@ -34,7 +34,7 @@ steps: archiveFile: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.tgz' replaceExistingArchive: true - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: '$(Build.ArtifactStagingDirectory)' artifactName: '${{parameters.artifactNameNoVersionString}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml index 5ee425405ac70..99437f86e6195 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml @@ -95,7 +95,7 @@ steps: archiveFile: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' replaceExistingArchive: true - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' artifactName: '${{parameters.artifactNameNoVersionString}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index a6fe5ac27749b..d344ac34a5ea2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -123,7 +123,9 @@ stages: workspace: clean: all pool: - vmImage: 'macOS-13' + name: 'Azure Pipelines' + image: 'macOS-13' + os: 'macOS' timeoutInMinutes: 300 steps: - template: set-version-number-variables-step.yml @@ -152,10 +154,10 @@ stages: --mac_catalyst_enabled displayName: "Test Apple framework" - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: - pathtoPublish: '$(Build.BinariesDirectory)/artifacts' - artifactName: 'onnxruntime-ios-full-xcframework' + path: '$(Build.BinariesDirectory)/artifacts' + artifact: 'onnxruntime-ios-full-xcframework' - template: component-governance-component-detection-steps.yml parameters: @@ -282,15 +284,15 @@ stages: SourceFolder: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Pipeline Artifact' inputs: - targetPath: '$(Build.ArtifactStagingDirectory)' + path: '$(Build.ArtifactStagingDirectory)' artifact: 'onnxruntime-java' - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + parameters: + condition: 'succeeded' - stage: NuGet_Packaging_CPU dependsOn: @@ -312,8 +314,8 @@ stages: OrtPackageId: ${{ parameters.OrtNugetPackageId }} breakCodesignValidationInjection: ${{ parameters.DoEsrp }} ReleaseVersionSuffix: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']] - BuildDate : $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']] - BuildTime : $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Time.BuildTime']] + BuildDate: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']] + BuildTime: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Time.BuildTime']] steps: - checkout: self @@ -384,7 +386,7 @@ stages: BuildId: ${{ parameters.BuildId }} - script: | - dir + dir workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' displayName: 'List artifacts' @@ -398,7 +400,7 @@ stages: filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1 - script: | - mklink /D /J models C:\local\models + mklink /D /J models C:\local\models workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Create models link' - task: NuGetToolInstaller@0 @@ -488,11 +490,12 @@ stages: PlatformsSupported: 'win-x64,win-x86,linux-x64,linux-arm64,osx-x64' VerifyNugetSigning: false - - task: PublishPipelineArtifact@0 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Pipeline NuGet Artifact' inputs: - artifactName: 'drop-signed-nuget-CPU' - targetPath: '$(Build.ArtifactStagingDirectory)' + path: '$(Build.ArtifactStagingDirectory)' + artifact: 'drop-signed-nuget-CPU' + - task: MSBuild@1 @@ -505,8 +508,8 @@ stages: workingDirectory: '$(Build.SourcesDirectory)\csharp' - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + parameters: + condition: 'succeeded' - stage: Nodejs_Packaging @@ -537,11 +540,11 @@ stages: submodules: true - script: | - echo.>>.gitattributes - echo /js/** text=auto eol=lf>>.gitattributes - rd /s /q js - git checkout -- js/** - git checkout -- .gitattributes + echo.>>.gitattributes + echo /js/** text=auto eol=lf>>.gitattributes + rd /s /q js + git checkout -- js/** + git checkout -- .gitattributes workingDirectory: '$(Build.SourcesDirectory)' displayName: 'Testing: force EOL to lf on windows for /js/**' @@ -650,20 +653,20 @@ stages: filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1 - script: | - dir + dir workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' displayName: 'List artifacts' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js' displayName: 'Install NPM packages /js' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js/common' displayName: 'Install NPM packages /js/common' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js/node' displayName: 'Install NPM packages /js/node' @@ -767,52 +770,52 @@ stages: Contents: 'onnxruntime-node-*.tgz' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishPipelineArtifact@0 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish NPM packages files' inputs: - artifactName: 'NPM_packages' - targetPath: '$(Build.ArtifactStagingDirectory)' + path: '$(Build.ArtifactStagingDirectory)' + artifact: 'NPM_packages' - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + parameters: + condition: 'succeeded' - template: ../nuget/templates/test_win.yml parameters: - AgentPool : 'onnxruntime-Win-CPU-2022' - Skipx86Tests : false - NugetPackageName : 'Microsoft.ML.OnnxRuntime' + AgentPool: 'onnxruntime-Win-CPU-2022' + Skipx86Tests: false + NugetPackageName: 'Microsoft.ML.OnnxRuntime' ArtifactSuffix: 'CPU' SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} - template: ../nuget/templates/test_linux.yml parameters: - AgentPool : onnxruntime-Ubuntu2204-AMD-CPU - NugetPackageName : 'Microsoft.ML.OnnxRuntime' + AgentPool: onnxruntime-Ubuntu2204-AMD-CPU + NugetPackageName: 'Microsoft.ML.OnnxRuntime' ArtifactSuffix: 'CPU' SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} - template: ../nuget/templates/test_macos.yml parameters: - AgentPool : macOS-13 + AgentPool: macOS-13 ArtifactSuffix: 'CPU' - template: ../nodejs/templates/test_win.yml parameters: - AgentPool : 'onnxruntime-Win-CPU-2022' - StageSuffix : 'Win_CPU_x64' + AgentPool: 'onnxruntime-Win-CPU-2022' + StageSuffix: 'Win_CPU_x64' - template: ../nodejs/templates/test_linux.yml parameters: - AgentPool : 'Azure-Pipelines-EO-Ubuntu-2004-aiinfra' - StageSuffix : 'Linux_CPU_x64' + AgentPool: 'Azure-Pipelines-EO-Ubuntu-2004-aiinfra' + StageSuffix: 'Linux_CPU_x64' - template: ../nodejs/templates/test_macos.yml parameters: - StageSuffix : 'macOS_CPU_x64' + StageSuffix: 'macOS_CPU_x64' - template: final-jar-testing.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index 12d53c1a1b8e7..ec9aff9f0663f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -35,68 +35,72 @@ jobs: workspace: clean: all - timeoutInMinutes: 210 - pool: ${{parameters.PoolName}} + timeoutInMinutes: 210 + pool: + name: ${{parameters.PoolName}} + os: linux + ${{ if eq(parameters.OnnxruntimeArch, 'aarch64') }}: + hostArchitecture: Arm64 steps: - - checkout: self - clean: true - submodules: none - - template: set-version-number-variables-step.yml - - ${{ if eq(parameters.OnnxruntimeArch, 'x64') }}: - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile - Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging - - - ${{ if eq(parameters.OnnxruntimeArch, 'aarch64') }}: - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile - Context: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging + - checkout: self + clean: true + submodules: none + - template: set-version-number-variables-step.yml + - ${{ if eq(parameters.OnnxruntimeArch, 'x64') }}: + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging - - task: CmdLine@2 - inputs: - script: | - set -e -x - mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3 \ - /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ - --skip_submodule_sync --parallel --use_vcpkg --use_binskim_compliant_compile_flags --use_vcpkg --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata - ls -al $(Build.ArtifactStagingDirectory) - displayName: 'Create Artifacts' + - ${{ if eq(parameters.OnnxruntimeArch, 'aarch64') }}: + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging - - ${{ if eq(parameters.PackageJava, 'true') }}: - - template: java-api-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: 'linux-${{parameters.OnnxruntimeArch}}' - buildConfig: 'Release' - artifactName: 'onnxruntime-java-linux-${{parameters.OnnxruntimeArch}}' - version: '$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.so' - nativeLibraryName: 'libonnxruntime4j_jni.so' + - task: CmdLine@2 + inputs: + script: | + set -e -x + mkdir -p $HOME/.onnx + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3 \ + /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ + --skip_submodule_sync --parallel --use_vcpkg --use_binskim_compliant_compile_flags --use_vcpkg --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" + mkdir $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata + ls -al $(Build.ArtifactStagingDirectory) + displayName: 'Create Artifacts' - - template: c-api-artifacts-package-and-publish-steps-posix.yml + - ${{ if eq(parameters.PackageJava, 'true') }}: + - template: java-api-artifacts-package-and-publish-steps-posix.yml parameters: - buildConfig: 'Release' - artifactName: '${{parameters.ArtifactNamePrefix}}-linux-${{parameters.OnnxruntimeArch}}-$(OnnxRuntimeVersion)' - artifactNameNoVersionString: '${{parameters.ArtifactNamePrefix}}-linux-${{parameters.OnnxruntimeArch}}' - libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' + arch: 'linux-${{parameters.OnnxruntimeArch}}' + buildConfig: 'Release' + artifactName: 'onnxruntime-java-linux-${{parameters.OnnxruntimeArch}}' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.so' + nativeLibraryName: 'libonnxruntime4j_jni.so' - - ${{ if eq(parameters.PackageNodejs, 'true') }}: - - template: nodejs-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: '${{parameters.OnnxruntimeNodejsBindingArch}}' - os: 'linux' - artifactName: 'drop-onnxruntime-nodejs-linux-${{parameters.OnnxruntimeArch}}' + - template: c-api-artifacts-package-and-publish-steps-posix.yml + parameters: + buildConfig: 'Release' + artifactName: '${{parameters.ArtifactNamePrefix}}-linux-${{parameters.OnnxruntimeArch}}-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: '${{parameters.ArtifactNamePrefix}}-linux-${{parameters.OnnxruntimeArch}}' + libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' - - ${{ if not(eq(parameters.OnnxruntimeNodejsBindingArch, 'arm64')) }}: - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + - ${{ if eq(parameters.PackageNodejs, 'true') }}: + - template: nodejs-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: '${{parameters.OnnxruntimeNodejsBindingArch}}' + os: 'linux' + artifactName: 'drop-onnxruntime-nodejs-linux-${{parameters.OnnxruntimeArch}}' + + - ${{ if not(eq(parameters.OnnxruntimeNodejsBindingArch, 'arm64')) }}: + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml b/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml index 6b469c1a2a551..4acec5e581e45 100644 --- a/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml +++ b/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml @@ -15,7 +15,7 @@ steps: args: '-r $(Build.BinariesDirectory) -c ${{parameters.buildConfig}} -a ${{parameters.artifactName}} -l ${{parameters.libraryName}} -n ${{parameters.nativeLibraryName}} -v ${{parameters.version}} -h ${{parameters.arch}}' workingDirectory: '$(Build.BinariesDirectory)/${{parameters.buildConfig}}' - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: - pathtoPublish: '$(Build.BinariesDirectory)/${{parameters.artifactName}}' - artifactName: 'drop-${{parameters.artifactName}}' + path: '$(Build.BinariesDirectory)/${{parameters.artifactName}}' + artifact: 'drop-${{parameters.artifactName}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml index ab31e592d7d71..933afd86a2d78 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml @@ -42,99 +42,101 @@ stages: - stage: MacOS_C_API_Packaging_CPU dependsOn: [] jobs: + - template: mac-cpu-packing-jobs.yml + parameters: + MacosArch: 'x86_64' + AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} + WithCache: ${{ parameters.WithCache }} + + - ${{ if eq(parameters.BuildForAllArchs, true) }}: - template: mac-cpu-packing-jobs.yml parameters: - MacosArch: 'x86_64' + MacosArch: 'arm64' + AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} + WithCache: ${{ parameters.WithCache }} + - template: mac-cpu-packing-jobs.yml + parameters: + MacosArch: 'universal2' AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} WithCache: ${{ parameters.WithCache }} - - - ${{ if eq(parameters.BuildForAllArchs, true) }}: - - template: mac-cpu-packing-jobs.yml - parameters: - MacosArch: 'arm64' - AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} - AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} - WithCache: ${{ parameters.WithCache }} - - template: mac-cpu-packing-jobs.yml - parameters: - MacosArch: 'universal2' - AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} - AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} - WithCache: ${{ parameters.WithCache }} - stage: MacOS_C_API_Package_Publish dependsOn: MacOS_C_API_Packaging_CPU jobs: - - job: MacOS_C_API_Package_Publish - pool: - vmImage: 'macOS-13' - steps: - - checkout: none + - job: MacOS_C_API_Package_Publish + pool: + name: 'Azure Pipelines' + image: 'macOS-13' + os: 'macOS' + steps: + - checkout: none + - template: flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline onnxruntime-osx-x86_64' + ArtifactName: 'onnxruntime-osx-x86_64' + TargetPath: '$(Build.ArtifactStagingDirectory)' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + + - ${{ if eq(parameters.BuildForAllArchs, true) }}: - template: flex-downloadPipelineArtifact.yml parameters: - StepName: 'Download Pipeline onnxruntime-osx-x86_64' - ArtifactName: 'onnxruntime-osx-x86_64' + StepName: 'Download Pipeline onnxruntime-osx-arm64' + ArtifactName: 'onnxruntime-osx-arm64' + TargetPath: '$(Build.ArtifactStagingDirectory)' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + - template: flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline onnxruntime-osx-universal2' + ArtifactName: 'onnxruntime-osx-universal2' TargetPath: '$(Build.ArtifactStagingDirectory)' SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} - - ${{ if eq(parameters.BuildForAllArchs, true) }}: - - template: flex-downloadPipelineArtifact.yml - parameters: - StepName: 'Download Pipeline onnxruntime-osx-arm64' - ArtifactName: 'onnxruntime-osx-arm64' - TargetPath: '$(Build.ArtifactStagingDirectory)' - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - - template: flex-downloadPipelineArtifact.yml - parameters: - StepName: 'Download Pipeline onnxruntime-osx-universal2' - ArtifactName: 'onnxruntime-osx-universal2' - TargetPath: '$(Build.ArtifactStagingDirectory)' - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - - - ${{ if eq(parameters.DoESRP, true)}}: - - script: | - pushd '$(Build.ArtifactStagingDirectory)' - find . '*.tgz' -exec tar -zxvf {} \; - rm -f *.tgz; - find . -type d -name 'onnxruntime-osx-*' -exec zip -FSr --symlinks {}.zip {} \; - find . -type d -name 'onnxruntime-osx-*' -exec rm -rf {} \; - ls -l - popd - displayName: tgz to zip - - template: mac-esrp-dylib.yml - parameters: - FolderPath: '$(Build.ArtifactStagingDirectory)' - DisplayName: 'ESRP - Sign Mac' - DoEsrp: true - Pattern: '*.zip' - - script: | - pushd '$(Build.ArtifactStagingDirectory)' - find . '*.zip' -exec unzip {} \; - rm -f *.zip; - find . -type d -name 'onnxruntime-osx-*' -exec tar -czf {}.tgz {} \; - find . -type d -name 'onnxruntime-osx-*' -exec rm -rf {} \; - ls -l - popd - displayName: zip to tgz - - bash: | - set -ex - mkdir -p $(Agent.TempDirectory)/macpackage - find $(Build.ArtifactStagingDirectory) -name "*.tgz" -exec tar -zxvf {} -C $(Agent.TempDirectory)/macpackage \; - find $(Agent.TempDirectory)/macpackage -name "*.dylib" -exec codesign -dvvv {} \; - find $(Agent.TempDirectory)/macpackage -name "*.dylib" -exec ls -l {} \; - rm -rf $(Agent.TempDirectory)/macpackage - displayName: 'Verify code signing' + - ${{ if eq(parameters.DoESRP, true)}}: + - script: | + pushd '$(Build.ArtifactStagingDirectory)' + find . '*.tgz' -exec tar -zxvf {} \; + rm -f *.tgz; + find . -type d -name 'onnxruntime-osx-*' -exec zip -FSr --symlinks {}.zip {} \; + find . -type d -name 'onnxruntime-osx-*' -exec rm -rf {} \; + ls -l + popd + displayName: tgz to zip + - template: mac-esrp-dylib.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)' + DisplayName: 'ESRP - Sign Mac' + DoEsrp: true + Pattern: '*.zip' + - script: | + pushd '$(Build.ArtifactStagingDirectory)' + find . '*.zip' -exec unzip {} \; + rm -f *.zip; + find . -type d -name 'onnxruntime-osx-*' -exec tar -czf {}.tgz {} \; + find . -type d -name 'onnxruntime-osx-*' -exec rm -rf {} \; + ls -l + popd + displayName: zip to tgz + - bash: | + set -ex + mkdir -p $(Agent.TempDirectory)/macpackage + find $(Build.ArtifactStagingDirectory) -name "*.tgz" -exec tar -zxvf {} -C $(Agent.TempDirectory)/macpackage \; + find $(Agent.TempDirectory)/macpackage -name "*.dylib" -exec codesign -dvvv {} \; + find $(Agent.TempDirectory)/macpackage -name "*.dylib" -exec ls -l {} \; + rm -rf $(Agent.TempDirectory)/macpackage + displayName: 'Verify code signing' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)' - artifact: 'onnxruntime-osx' - condition: 'succeededOrFailed()' + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)' + artifact: 'onnxruntime-osx' + condition: 'succeededOrFailed()' - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml index 3d61990ead0c2..160b4d240e5ad 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml @@ -83,8 +83,10 @@ steps: displayName: 'Copy libcustom_op_library.dylib to ArtifactStagingDirectory' condition: and(succeeded(), eq('${{ parameters.MacosArch }}', 'x86_64')) -- publish: '$(Build.ArtifactStagingDirectory)' - artifact: 'onnxruntime-osx-${{ parameters.MacosArch }}' +- task: 1ES.PublishPipelineArtifact@1 + inputs: + path: '$(Build.ArtifactStagingDirectory)' + artifact: 'onnxruntime-osx-${{ parameters.MacosArch }}' - ${{ if eq(parameters.BuildJava, true) }}: - template: java-api-artifacts-package-and-publish-steps-posix.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml index 32908753f2909..f290828d6b717 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml @@ -37,7 +37,9 @@ jobs: PROTO_CACHE_DIR: $(Pipeline.Workspace)/ccache_proto ORT_CACHE_DIR: $(Pipeline.Workspace)/ccache_ort pool: - vmImage: 'macOS-13' + name: "Azure Pipelines" + image: 'macOS-13' + os: macOS timeoutInMinutes: 300 steps: - checkout: self diff --git a/tools/ci_build/github/azure-pipelines/templates/nodejs-artifacts-package-and-publish-steps-posix.yml b/tools/ci_build/github/azure-pipelines/templates/nodejs-artifacts-package-and-publish-steps-posix.yml index 3a57982c52b1e..41c6c62e9b101 100644 --- a/tools/ci_build/github/azure-pipelines/templates/nodejs-artifacts-package-and-publish-steps-posix.yml +++ b/tools/ci_build/github/azure-pipelines/templates/nodejs-artifacts-package-and-publish-steps-posix.yml @@ -10,7 +10,7 @@ parameters: default: '' steps: - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: '$(Build.SourcesDirectory)/js/node/bin/napi-v3/${{ parameters.os }}/${{ parameters.arch }}/onnxruntime_binding.node' artifactName: '${{parameters.artifactName}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/nodejs-artifacts-package-and-publish-steps-windows.yml b/tools/ci_build/github/azure-pipelines/templates/nodejs-artifacts-package-and-publish-steps-windows.yml index 0bc067034b2fc..772fa96609ab2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/nodejs-artifacts-package-and-publish-steps-windows.yml +++ b/tools/ci_build/github/azure-pipelines/templates/nodejs-artifacts-package-and-publish-steps-windows.yml @@ -22,7 +22,7 @@ steps: DoEsrp: ${{parameters.DoEsrp}} Pattern: '*.node' - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: '$(Build.SourcesDirectory)\js\node\bin\napi-v3\win32\${{ parameters.arch }}\onnxruntime_binding.node' artifactName: '${{parameters.artifactName}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 1bda2cc1432c4..be8a2428f8152 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -177,11 +177,11 @@ stages: ${{ else }}: buildOnly: false - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Java temp binaries' inputs: - pathtoPublish: '$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}' - artifactName: 'drop-onnxruntime-java-win-${{ parameters.packageName }}${{parameters.artifact_name_suffix}}' + path: '$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}' + artifact: 'drop-onnxruntime-java-win-${{ parameters.packageName }}${{parameters.artifact_name_suffix}}' # All GPU builds will be tested in the next stage with GPU machine - ${{ if contains(parameters.ort_build_pool_name, 'CPU') }}: - task: PythonScript@0 @@ -213,11 +213,10 @@ stages: displayName: 'Copy java pad and folder for java test' workingDirectory: '$(Build.BinariesDirectory)' - - task: PublishPipelineArtifact@1 + - task: 1ES.PublishPipelineArtifact@1 inputs: targetPath: '$(Agent.TempDirectory)/RelWithDebInfo' artifactName: 'Windows_Packaging_${{ parameters.stage_name_suffix }}_build_artifacts' - publishLocation: 'pipeline' - script: | dir *.dll