Skip to content

Commit ce7f91a

Browse files
michaelgsharpsafern
michaelgsharp
and
safern
authored
Helix Integration. (dotnet#5837)
* Helix working. Squashed local commits * Updates from PR comments. * Updates from PR comments. * Update Directory.Build.targets Co-authored-by: Santiago Fernandez Madero <[email protected]> * Fixes from pr comments. * Switched tabs to spaces. * Helix changes from pr comments. * yml changes. * yml changes. * yml changes. * yml changes. * yml changes. * Indentation fixes. * Indentation fixes. Co-authored-by: Santiago Fernandez Madero <[email protected]>
1 parent 7f888c2 commit ce7f91a

File tree

54 files changed

+466
-287
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+466
-287
lines changed

Diff for: ‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
*.sln.docstates
1515
*.DS_Store
1616

17+
# Helix payload files
18+
*.payload
19+
1720
# User-specific files (MonoDevelop/Xamarin Studio)
1821
*.userprefs
1922

Diff for: ‎.vsts-dotnet-ci.yml

+7
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ jobs:
3030
innerLoop: true
3131
pool:
3232
name: Hosted Ubuntu 1604
33+
helixQueue: [email protected]/dotnet-buildtools/prereqs:centos-7-mlnet-helix-20210528183707-dde38af
3334

3435
- template: /build/ci/job-template.yml
3536
parameters:
@@ -39,6 +40,7 @@ jobs:
3940
innerLoop: true
4041
pool:
4142
name: Hosted Ubuntu 1604
43+
helixQueue: [email protected]/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-helix-20210528184647-dde38af
4244

4345
- template: /build/ci/job-template.yml
4446
parameters:
@@ -47,6 +49,7 @@ jobs:
4749
innerLoop: true
4850
pool:
4951
name: Hosted macOS
52+
helixQueue: OSX.1015.Amd64.Open
5053

5154
- template: /build/ci/job-template.yml
5255
parameters:
@@ -67,6 +70,7 @@ jobs:
6770
vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0"
6871
pool:
6972
name: Hosted VS2017
73+
helixQueue: Windows.10.Amd64.Open
7074

7175
- template: /build/ci/job-template.yml
7276
parameters:
@@ -76,6 +80,7 @@ jobs:
7680
vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
7781
pool:
7882
name: Hosted VS2017
83+
helixQueue: Windows.10.Amd64.Open
7984

8085
- template: /build/ci/job-template.yml
8186
parameters:
@@ -96,6 +101,7 @@ jobs:
96101
vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0"
97102
pool:
98103
name: Hosted VS2017
104+
helixQueue: Windows.10.Amd64.Open
99105

100106
- template: /build/ci/job-template.yml
101107
parameters:
@@ -106,3 +112,4 @@ jobs:
106112
vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
107113
pool:
108114
name: Hosted VS2017
115+
helixQueue: Windows.10.Amd64.Open

Diff for: ‎Directory.Build.targets

+20-3
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<PackageReference Include="coverlet.msbuild" Version="2.9.0" PrivateAssets="all" />
77
</ItemGroup>
88

9-
<Target Name="CopyNativeAssembiles" AfterTargets="CopyFilesToOutputDirectory">
9+
<Target Name="SetCopyProperties">
1010
<PropertyGroup>
1111
<LibPrefix Condition="'$(OS)' != 'Windows_NT'">lib</LibPrefix>
1212
<LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
@@ -33,7 +33,23 @@
3333
<NativeAssemblyReference Remove="FastTreeNative"/>
3434
<NativeAssemblyReference Remove="SymSgdNative"/>
3535
<NativeAssemblyReference Remove="MklProxyNative"/>
36+
<NativeAssemblyReference Remove="libiomp5md"/>
3637
</ItemGroup>
38+
</Target>
39+
40+
<Target Name="CopyNativeAssembliesPublish" AfterTargets="Publish" DependsOnTargets="SetCopyProperties">
41+
<Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
42+
DestinationFolder="$(OutDir)\publish"
43+
OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
44+
Retries="$(CopyRetryCount)"
45+
RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
46+
UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
47+
UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
48+
<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
49+
</Copy>
50+
</Target>
51+
52+
<Target Name="CopyNativeAssembiles" AfterTargets="CopyFilesToOutputDirectory" DependsOnTargets="SetCopyProperties">
3753

3854
<Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
3955
DestinationFolder="$(OutDir)"
@@ -42,9 +58,10 @@
4258
RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
4359
UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
4460
UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
45-
<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
61+
<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
4662
</Copy>
4763
</Target>
64+
4865
<PropertyGroup Condition="'$(Coverage)' == 'true'">
4966
<!-- https://github.com/tonerdo/coverlet/issues/363 -->
5067
<DeterministicSourcePaths>false</DeterministicSourcePaths>
@@ -54,7 +71,7 @@
5471

5572
<CollectCoverage>true</CollectCoverage>
5673
<SingleHit>true</SingleHit>
57-
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
74+
<CopyLocalLockFileAssemblies>false</CopyLocalLockFileAssemblies>
5875
<CoverletOutputFormat>opencover</CoverletOutputFormat>
5976
<CoverletOutput>$(BaseOutputPath)$(PlatformConfig)\coverage\coverage.opencover.xml</CoverletOutput>
6077
<Include></Include>

Diff for: ‎build/ci/job-template.yml

+39-12
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#TODO: Need to update build documentation.
2+
23
parameters:
34
name: ''
45
architecture: x64
@@ -11,6 +12,7 @@ parameters:
1112
runSpecific: false
1213
container: ''
1314
useVSTestTask: false
15+
helixQueue: ''
1416

1517
jobs:
1618
- job: ${{ parameters.name }}
@@ -68,7 +70,7 @@ jobs:
6870
- ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
6971
- bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
7072
displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
71-
- script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
73+
- script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:Coverage=${{ parameters.codeCoverage }}
7274
displayName: Build
7375
- ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
7476
- task: Bash@3
@@ -85,11 +87,12 @@ jobs:
8587
- ${{ if eq(parameters.buildScript, 'build.cmd') }}:
8688
- script: dir /s "$(Build.SourcesDirectory)"
8789
displayName: show bin folder disk usage
88-
- task: PowerShell@2
89-
inputs:
90-
targetType: inline
91-
script: Get-ChildItem -Path '$(Build.SourcesDirectory)\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
92-
displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
90+
# - ${{ if eq(parameters.codeCoverage, 'true') }}:
91+
# - task: PowerShell@2
92+
# inputs:
93+
# targetType: inline
94+
# script: Get-ChildItem -Path '$(Build.SourcesDirectory)\.packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
95+
# displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
9396
- ${{ if eq(parameters.nightlyBuild, 'true') }}:
9497
- script: $(dotnetPath) restore $(nightlyBuildProjPath)
9598
displayName: Restore nightly build project
@@ -113,11 +116,10 @@ jobs:
113116
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
114117
- ${{ if eq(parameters.innerLoop, 'false') }}:
115118
- ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}:
116-
# TODO: Code coverage needs to be fixed.
117-
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages /p:Coverage=${{ parameters.codeCoverage }}
119+
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:Coverage=${{ parameters.codeCoverage }}
118120
displayName: Run All Tests.
119121
- ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}:
120-
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-trait$(spaceValue)Category=RunSpecificTest' /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages /p:Coverage=${{ parameters.codeCoverage }}
122+
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-trait$(spaceValue)Category=RunSpecificTest' /p:Coverage=${{ parameters.codeCoverage }}
121123
displayName: Run Specific Tests.
122124
- ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}:
123125
- task: VSTest@2
@@ -141,8 +143,21 @@ jobs:
141143
collectDumpOn: onAbortOnly
142144
publishRunAttachments: true
143145
- ${{ if eq(parameters.innerLoop, 'true') }}:
144-
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-notrait$(spaceValue)Category=SkipInCI' /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages /p:Coverage=${{ parameters.codeCoverage }}
145-
displayName: Run CI Tests.
146+
- ${{ if eq(parameters.codeCoverage, True) }}:
147+
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-notrait$(spaceValue)Category=SkipInCI' /p:Coverage=${{ parameters.codeCoverage }}
148+
displayName: Run CI Tests.
149+
- ${{ if eq(parameters.codeCoverage, False) }}:
150+
- template: /build/ci/send-to-helix.yml
151+
parameters:
152+
HelixTargetQueues: ${{ parameters.helixQueue }}
153+
Configuration: $(_configuration)
154+
Architecture: ${{ parameters.architecture }}
155+
${{ if eq(parameters.buildScript, 'build.cmd') }}:
156+
MsBuildScript: 'powershell $(Build.SourcesDirectory)/eng/common/msbuild.ps1'
157+
WarnAsError: '-warnAsError 0'
158+
${{ if eq(parameters.buildScript, './build.sh') }}:
159+
MsBuildScript: '$(Build.SourcesDirectory)/eng/common/msbuild.sh'
160+
WarnAsError: '--warnAsError false'
146161
- script: $(dotnetPath) msbuild -restore build/Codecoverage.proj
147162
displayName: Upload coverage to codecov.io
148163
condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))
@@ -193,6 +208,18 @@ jobs:
193208
pathToPublish: $(Build.ArtifactStagingDirectory)
194209
artifactName: ${{ parameters.name }} $(_config_short)
195210
artifactType: container
211+
- ${{ if eq(parameters.buildScript, './build.sh') }}:
212+
- task: Bash@3
213+
inputs:
214+
targetType: inline
215+
script: find ./artifacts/bin -type d -path "*/runtimes" -exec rm -rv {} +;
216+
displayName: Clean up runtime folder for package (Unix)
217+
- ${{ if eq(parameters.buildScript, 'build.cmd') }}:
218+
- task: PowerShell@2
219+
inputs:
220+
targetType: inline
221+
script: Get-ChildItem -Path '.\artifacts\bin' -Recurse | Where-Object {$_.FullName -like "*runtimes*"} | Remove-Item -Recurse -Confirm:$false -Force
222+
displayName: Clean up runtime folder for package (Unix)
196223
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
197-
- script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
224+
- script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }}
198225
displayName: Build Packages

Diff for: ‎build/ci/send-to-helix.yml

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Please remember to update the documentation if you make changes to these parameters!
2+
parameters:
3+
HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/
4+
HelixType: 'tests/default' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
5+
HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
6+
HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
7+
HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group
8+
Configuration: 'Debug'
9+
Architecture: 'x64'
10+
MsBuildScript: ''
11+
WarnAsError: ''
12+
HelixConfiguration: '' # optional -- additional property attached to a job
13+
IncludeDotNetCli: true # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
14+
EnableXUnitReporter: true # optional -- true enables XUnit result reporting to Mission Control
15+
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
16+
HelixBaseUri: 'https://helix.dot.net' # optional -- sets the Helix API base URI (allows targeting int)
17+
Creator: 'ML.NET' # optional -- if the build is external, use this to specify who is sending the job
18+
DisplayNamePrefix: 'Run Helix Tests' # optional -- rename the beginning of the displayName of the steps in AzDO
19+
continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
20+
condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
21+
22+
steps:
23+
- script: ${{ parameters.MsBuildScript}}
24+
$(Build.SourcesDirectory)/eng/helix.proj
25+
/t:Test
26+
/bl:$(Build.SourcesDirectory)/artifacts/log/${{ parameters.Configuration }}/SendToHelix.binlog
27+
/p:Configuration=${{ parameters.Configuration }}
28+
/p:TargetArchitecture=${{ parameters.Architecture }}
29+
/p:BuildConfig=${{ parameters.Configuration }}
30+
/p:BuildArchitecture=${{ parameters.Architecture }}
31+
/p:HelixSource=${{ parameters.HelixSource }}
32+
/p:HelixType=${{ parameters.HelixType }}
33+
/p:HelixBuild=${{ parameters.HelixBuild }}
34+
/p:HelixConfiguration="${{ parameters.HelixConfiguration }}"
35+
/p:HelixAccessToken="${{ parameters.HelixAccessToken }}"
36+
/p:IncludeDotNetCli=${{ parameters.IncludeDotNetCli }}
37+
/p:EnableXUnitReporter=${{ parameters.EnableXUnitReporter }}
38+
/p:WaitForWorkItemCompletion=${{ parameters.WaitForWorkItemCompletion }}
39+
/p:HelixBaseUri=${{ parameters.HelixBaseUri }}
40+
${{ parameters.WarnAsError }}
41+
displayName: ${{ parameters.DisplayNamePrefix }}
42+
env:
43+
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
44+
Creator: ${{ parameters.Creator }}
45+
HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
46+
continueOnError: ${{ parameters.continueOnError }}

Diff for: ‎build/publish.proj

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<Project>
2-
2+
33
<Import Project="$(MSBuildThisFileDirectory)..\Directory.Build.props" />
44

55
<PropertyGroup>
@@ -8,8 +8,8 @@
88
<NuGetPushTimeoutSeconds Condition="'$(NuGetPushTimeoutSeconds)' == ''">600</NuGetPushTimeoutSeconds>
99
</PropertyGroup>
1010

11-
<Import Project="$(NuGetPackageRoot)\$(PublishSymbolsPackage.ToLower())\$(MicrosoftSymbolUploaderBuildTaskVersion)\build\PublishSymbols.targets" />
12-
11+
<Import Project="$(NuGetPackageRoot)$(PublishSymbolsPackage.ToLower())\$(MicrosoftSymbolUploaderBuildTaskVersion)\build\PublishSymbols.targets" />
12+
1313
<Target Name="PublishSymbolPackages"
1414
Condition="'$(EnablePublishSymbols)'=='true'"
1515
DependsOnTargets="SetupPublishSymbols">

Diff for: ‎build/vsts-ci.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -234,10 +234,10 @@ phases:
234234
# For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release"
235235
# the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable
236236
# See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version
237-
- script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:OfficialBuildId=$(OfficialBuildId) /p:DotNetFinalVersionKind=$(DotnetVersionKind) /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
237+
- script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:OfficialBuildId=$(OfficialBuildId) /p:DotNetFinalVersionKind=$(DotnetVersionKind)
238238
displayName: Build Packages
239239

240-
- script: ./sign.cmd /p:SignNugetPackages=true /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
240+
- script: ./sign.cmd /p:SignNugetPackages=true
241241
displayName: sign packages
242242
continueOnError: false
243243

@@ -267,15 +267,15 @@ phases:
267267
displayName: Publish Symbols to SymWeb Symbol Server
268268
inputs:
269269
solution: build/publish.proj
270-
msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT) /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
270+
msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT)
271271
msbuildVersion: 15.0
272272
continueOnError: true
273273

274274
- task: MSBuild@1
275275
displayName: Publish Symbols to Msdl Symbol Server
276276
inputs:
277277
solution: build/publish.proj
278-
msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT) /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
278+
msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT)
279279
msbuildVersion: 15.0
280280
continueOnError: true
281281

0 commit comments

Comments
 (0)