Skip to content

Commit aa2308b

Browse files
paulmedynskipaulmedynski
committed
Task 42088: Suppress flaky tests errors in AzDO UI
- Added targets for flaky tests that ignore failures entirely.
1 parent 47862b1 commit aa2308b

File tree

1 file changed

+130
-16
lines changed

1 file changed

+130
-16
lines changed

‎build.proj‎

Lines changed: 130 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -354,11 +354,10 @@
354354
spurious errors, while still considering the exit status of the test process to determine
355355
overall pass/fail.
356356
-->
357-
<Exec
358-
ConsoleToMsBuild="true"
359-
IgnoreExitCode="true"
360-
IgnoreStandardErrorWarningFormat="true"
361-
Command="$(TestCommand)"/>
357+
<Exec ConsoleToMsBuild="true"
358+
IgnoreExitCode="true"
359+
IgnoreStandardErrorWarningFormat="true"
360+
Command="$(TestCommand)"/>
362361
</Target>
363362

364363
<!-- Run all flaky unit tests applicable to Unix. -->
@@ -380,17 +379,16 @@
380379
<TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
381380
</PropertyGroup>
382381
<Message Text=">>> Running unit tests for Unix via command: $(TestCommand)"/>
383-
<Exec
384-
ConsoleToMsBuild="true"
385-
IgnoreExitCode="true"
386-
IgnoreStandardErrorWarningFormat="true"
387-
Command="$(TestCommand)"/>
382+
<Exec ConsoleToMsBuild="true"
383+
IgnoreExitCode="true"
384+
IgnoreStandardErrorWarningFormat="true"
385+
Command="$(TestCommand)"/>
388386
</Target>
389387

390-
<!-- Run all Functional tests applicable to the host OS. -->
388+
<!-- Run all (non-flaky) Functional tests applicable to the host OS. -->
391389
<Target Name="RunFunctionalTests" DependsOnTargets="RunFunctionalTestsWindows;RunFunctionalTestsUnix" />
392390

393-
<!-- Run all Functional tests applicable to Windows. -->
391+
<!-- Run all (non-flaky) Functional tests applicable to Windows. -->
394392
<Target Name="RunFunctionalTestsWindows" Condition="'$(IsEnabledWindows)' == 'true'">
395393
<PropertyGroup>
396394
<TestCommand>
@@ -413,7 +411,7 @@
413411
<Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
414412
</Target>
415413

416-
<!-- Run all Functional tests applicable to Unix. -->
414+
<!-- Run all (non-flaky) Functional tests applicable to Unix. -->
417415
<Target Name="RunFunctionalTestsUnix" Condition="'$(IsEnabledWindows)' != 'true'">
418416
<PropertyGroup>
419417
<TestCommand>
@@ -436,10 +434,65 @@
436434
<Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
437435
</Target>
438436

439-
<!-- Run all Manual tests applicable to the host OS. -->
437+
<!-- Run all flaky Functional tests applicable to the host OS. -->
438+
<Target Name="RunFlakyFunctionalTests" DependsOnTargets="RunFlakyFunctionalTestsWindows;RunFlakyFunctionalTestsUnix" />
439+
440+
<!-- Run all flaky Functional tests applicable to Windows. -->
441+
<Target Name="RunFlakyFunctionalTestsWindows" Condition="'$(IsEnabledWindows)' == 'true'">
442+
<PropertyGroup>
443+
<TestCommand>
444+
$(DotnetPath)dotnet test "@(FunctionalTestsProj)"
445+
-f $(TF)
446+
-p:Configuration=$(Configuration)
447+
-p:ReferenceType=$(ReferenceType)
448+
$(CollectStatement)
449+
--results-directory $(ResultsDirectory)
450+
--filter "category = Flaky"
451+
--logger:"trx;LogFilePrefix=Functional-Windows$(TargetGroup)-$(TestSet)"
452+
--blame-hang
453+
--blame-hang-dump-type full
454+
--blame-hang-timeout 10m
455+
</TestCommand>
456+
<!-- Convert more than one whitespace character into one space -->
457+
<TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
458+
</PropertyGroup>
459+
<Message Text=">>> Running Functional test for Windows via command: $(TestCommand)" />
460+
<Exec ConsoleToMsBuild="true"
461+
IgnoreExitCode="true"
462+
IgnoreStandardErrorWarningFormat="true"
463+
Command="$(TestCommand)"/>
464+
</Target>
465+
466+
<!-- Run all flaky Functional tests applicable to Unix. -->
467+
<Target Name="RunFlakyFunctionalTestsUnix" Condition="'$(IsEnabledWindows)' != 'true'">
468+
<PropertyGroup>
469+
<TestCommand>
470+
$(DotnetPath)dotnet test "@(FunctionalTestsProj)"
471+
-f $(TF)
472+
-p:Configuration=$(Configuration)
473+
-p:ReferenceType=$(ReferenceType)
474+
$(CollectStatement)
475+
--results-directory $(ResultsDirectory)
476+
--filter "category = Flaky"
477+
--logger:"trx;LogFilePrefix=Functional-Unixnetcoreapp-$(TestSet)"
478+
--blame-hang
479+
--blame-hang-dump-type full
480+
--blame-hang-timeout 10m
481+
</TestCommand>
482+
<!-- Convert more than one whitespace character into one space -->
483+
<TestCommand>$([System.Text.RegularExpressions.Regex]::Replace($(TestCommand), "\s+", " "))</TestCommand>
484+
</PropertyGroup>
485+
<Message Text=">>> Running Functional test for Unix via command: $(TestCommand)" />
486+
<Exec ConsoleToMsBuild="true"
487+
IgnoreExitCode="true"
488+
IgnoreStandardErrorWarningFormat="true"
489+
Command="$(TestCommand)"/>
490+
</Target>
491+
492+
<!-- Run all (non-flaky) Manual tests applicable to the host OS. -->
440493
<Target Name="RunManualTests" DependsOnTargets="RunManualTestsWindows;RunManualTestsUnix" />
441494

442-
<!-- Run all Manual tests applicable to Windows. -->
495+
<!-- Run all (non-flaky) Manual tests applicable to Windows. -->
443496
<Target Name="RunManualTestsWindows" Condition="'$(IsEnabledWindows)' == 'true'">
444497
<PropertyGroup>
445498
<TestCommand>
@@ -465,7 +518,7 @@
465518
<Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
466519
</Target>
467520

468-
<!-- Run all Manual tests applicable to Unix. -->
521+
<!-- Run all (non-flaky) Manual tests applicable to Unix. -->
469522
<Target Name="RunManualTestsUnix" Condition="'$(IsEnabledWindows)' != 'true'">
470523
<PropertyGroup>
471524
<TestCommand>
@@ -491,6 +544,67 @@
491544
<Exec ConsoleToMsBuild="true" Command="$(TestCommand)" />
492545
</Target>
493546

547+
<!-- Run all flaky Manual tests applicable to the host OS. -->
548+
<Target Name="RunFlakyManualTests" DependsOnTargets="RunFlakyManualTestsWindows;RunFlakyManualTestsUnix" />
549+
550+
<!-- Run all flaky Manual tests applicable to Windows. -->
551+
<Target Name="RunFlakyManualTestsWindows" Condition="'$(IsEnabledWindows)' == 'true'">
552+
<PropertyGroup>
553+
<TestCommand>
554+
$(DotnetPath)dotnet test "@(ManualTestsProj)"
555+
--no-build
556+
-v n
557+
-p:Configuration=$(Configuration)
558+
-p:Target$(TFGroup)Version=$(TF)
559+
-p:ReferenceType=$(ReferenceType)
560+
-p:TestSet=$(TestSet)
561+
-p:TestTargetOS=Windows$(TargetGroup)
562+
$(CollectStatement)
563+
--results-directory $(ResultsDirectory)
564+
--filter "category = Flaky"
565+
--logger:"trx;LogFilePrefix=Manual-Windows$(TargetGroup)-$(TestSet)"
566+
--blame-hang
567+
--blame-hang-dump-type full
568+
--blame-hang-timeout 10m
569+
</TestCommand>
570+
<TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
571+
</PropertyGroup>
572+
<Message Text=">>> Running Manual test for Windows via command: $(TestCommand)" />
573+
<Exec ConsoleToMsBuild="true"
574+
IgnoreExitCode="true"
575+
IgnoreStandardErrorWarningFormat="true"
576+
Command="$(TestCommand)"/>
577+
</Target>
578+
579+
<!-- Run all flaky Manual tests applicable to Unix. -->
580+
<Target Name="RunFlakyManualTestsUnix" Condition="'$(IsEnabledWindows)' != 'true'">
581+
<PropertyGroup>
582+
<TestCommand>
583+
$(DotnetPath)dotnet test "@(ManualTestsProj)"
584+
--no-build
585+
-v n
586+
-p:Configuration=$(Configuration)
587+
-p:TargetNetCoreVersion=$(TF)
588+
-p:ReferenceType=$(ReferenceType)
589+
-p:TestSet=$(TestSet)
590+
-p:TestTargetOS=Unixnetcoreapp
591+
$(CollectStatement)
592+
--results-directory $(ResultsDirectory)
593+
--filter "category = Flaky"
594+
--logger:"trx;LogFilePrefix=Manual-Unixnetcoreapp-$(TestSet)"
595+
--blame-hang
596+
--blame-hang-dump-type full
597+
--blame-hang-timeout 10m
598+
</TestCommand>
599+
<TestCommand>$(TestCommand.Replace($([System.Environment]::NewLine), " "))</TestCommand>
600+
</PropertyGroup>
601+
<Message Text=">>> Running Manual test for Unix via command: $(TestCommand)" />
602+
<Exec ConsoleToMsBuild="true"
603+
IgnoreExitCode="true"
604+
IgnoreStandardErrorWarningFormat="true"
605+
Command="$(TestCommand)"/>
606+
</Target>
607+
494608
<!-- Clean all build outputs. -->
495609
<Target Name="Clean">
496610
<RemoveDir Directories='$([System.IO.Directory]::GetDirectories(".", "artifacts", SearchOption.AllDirectories))' />

0 commit comments

Comments
 (0)