Skip to content

Commit

Permalink
Collect XHarness diagnostics data and upload to App Insights (#7842)
Browse files Browse the repository at this point in the history
  • Loading branch information
premun authored Sep 8, 2021
1 parent efff425 commit 3e6b160
Show file tree
Hide file tree
Showing 8 changed files with 71 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
</ItemGroup>

<ItemGroup>
<EmbeddedResource Include="tools\xharness-runner\xharness-event-reporter.py">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="tools\xharness-runner\xharness-helix-job.apple.sh">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</EmbeddedResource>
Expand Down
6 changes: 6 additions & 0 deletions src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public class MetadataName

private const string ScriptNamespace = "tools.xharness_runner.";
private const string CustomCommandsScript = "command";
private const string DiagnosticsScript = "xharness-event-reporter.py";

/// <summary>
/// Extra arguments that will be passed to the iOS/Android/... app that is being run
Expand Down Expand Up @@ -173,6 +174,11 @@ await zipArchiveManager.AddResourceFileToArchive<XHarnessTaskBase>(
payloadScript);
}

await zipArchiveManager.AddResourceFileToArchive<XHarnessTaskBase>(
outputZipPath,
ScriptNamespace + DiagnosticsScript,
DiagnosticsScript);

await zipArchiveManager.AddContentToArchive(
outputZipPath,
CustomCommandsScript + (isPosix ? ".sh" : ".ps1"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@
<HelixPreCommands>$(HelixPreCommands);export XHARNESS_DISABLE_COLORED_OUTPUT=true</HelixPreCommands>
<HelixPreCommands>$(HelixPreCommands);export XHARNESS_LOG_WITH_TIMESTAMPS=true</HelixPreCommands>
<HelixPreCommands>$(HelixPreCommands);export XHARNESS_CLI_PATH=$HELIX_CORRELATION_PAYLOAD/microsoft.dotnet.xharness.cli/$(_XHarnessPackageVersion)/tools/net6.0/any/Microsoft.DotNet.XHarness.CLI.dll</HelixPreCommands>

<HelixPreCommands Condition=" '$(EnableXHarnessTelemetry)' == 'true' ">$(HelixPreCommands);export XHARNESS_DIAGNOSTICS_PATH=$HELIX_WORKITEM_ROOT/diagnostics.json</HelixPreCommands>
<HelixPostCommands Condition=" '$(EnableXHarnessTelemetry)' == 'true' ">"$HELIX_PYTHONPATH" "$HELIX_WORKITEM_PAYLOAD/xharness-event-reporter.py";$(HelixPostCommands)</HelixPostCommands>
</PropertyGroup>

<PropertyGroup Condition="!$(IsPosixShell)">
Expand All @@ -68,6 +71,9 @@
<HelixPreCommands>$(HelixPreCommands);set XHARNESS_LOG_WITH_TIMESTAMPS=true</HelixPreCommands>
<HelixPreCommands>$(HelixPreCommands);set XHARNESS_CLI_PATH=%HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(_XHarnessPackageVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll</HelixPreCommands>
<HelixPreCommands>$(HelixPreCommands);doskey xharness="dotnet exec %XHARNESS_CLI_PATH%"</HelixPreCommands>

<HelixPreCommands Condition=" '$(EnableXHarnessTelemetry)' == 'true' ">$(HelixPreCommands);set XHARNESS_DIAGNOSTICS_PATH=%HELIX_WORKITEM_ROOT%\diagnostics.json</HelixPreCommands>
<HelixPostCommands Condition=" '$(EnableXHarnessTelemetry)' == 'true' ">"%HELIX_PYTHONPATH%" "%HELIX_WORKITEM_PAYLOAD%\xharness-event-reporter.py";$(HelixPostCommands)</HelixPostCommands>
</PropertyGroup>
</Target>

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import getopt
import json
import os
import sys

from helix.appinsights import app_insights

opts, args = getopt.gnu_getopt(sys.argv[1:], 'd:', ['diagnostics-data='])
opt_dict = dict(opts)

diagnostics_file = None

if '--data' in opt_dict:
diagnostics_file = opt_dict['--data']
elif '-d' in opt_dict:
diagnostics_file = opt_dict['-d']
else:
diagnostics_file = os.getenv('XHARNESS_DIAGNOSTICS_PATH')

if not diagnostics_file:
print('ERROR: Expected path to the diagnostics JSON file generated by XHarness')
exit(1)

if not os.path.isfile(diagnostics_file):
print(f"WARNING: Diagnostics file not found at `{diagnostics_file}`")
exit(2)

# The JSON should be an array of objects (one per each executed XHarness command)
operations = json.load(open(diagnostics_file))

for operation in operations:
custom_dimensions = dict()

custom_dimensions['command'] = operation['command']
custom_dimensions['platform'] = operation['platform']

if 'target' in operation:
if 'targetOS' in operation:
custom_dimensions['target'] = operation['target'] + '_' + operation['targetOS']
else:
custom_dimensions['target'] = operation['target']

app_insights.send_metric('XHarnessOperation', operation['exitCode'], properties=custom_dimensions)
app_insights.send_metric('XHarnessOperationDuration', operation['duration'], properties=custom_dimensions)
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,13 @@ function xharness() {
dotnet exec $Env:XHARNESS_CLI_PATH @args
}

$ErrorActionPreference="Continue"

# Act out the actual commands
. "$PSScriptRoot\command.ps1"

$ErrorActionPreference="Continue"

$exit_code=$LASTEXITCODE

$retry=$false
Expand All @@ -44,7 +48,6 @@ switch ($exit_code)
{
# ADB_DEVICE_ENUMERATION_FAILURE
85 {
$ErrorActionPreference="Continue"
Write-Error "Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices"
Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. sending x86 or x86_64 APKs to an arm64_v8a-only queue."
$retry=$true
Expand All @@ -54,7 +57,6 @@ switch ($exit_code)

# PACKAGE_INSTALLATION_FAILURE
78 {
$ErrorActionPreference="Continue"
Write-Error "Encountered PACKAGE_INSTALLATION_FAILURE. This is typically not a failure of the work item. We will try it again on another Helix agent"
Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. requesting installation on arm64_v8a-only queue for x86 or x86_64 APKs."
$retry=$true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@ set -x

# It is important we call the script via `launchctl asuser` in order to be able to spawn
# the simulator which needs to run in a user session with GUI rendering capabilities.
# The problem with launchctl is that the spawned process won't share environment variables
# so we have to pass all of them as parameters are set them again.
chmod +x xharness-runner.apple.sh
helix_runner_uid=$(id -u)
sudo launchctl asuser "$helix_runner_uid" sh ./xharness-runner.apple.sh \
$forwarded_args \
--app "$HELIX_WORKITEM_ROOT/$app" \
--xharness-cli-path "$XHARNESS_CLI_PATH" \
--output-directory "$HELIX_WORKITEM_UPLOAD_ROOT" \
--diagnostics-path "$XHARNESS_DIAGNOSTICS_PATH" \

exit_code=$?

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ while [[ $# -gt 0 ]]; do
output_directory="$2"
shift
;;
--diagnostics-path)
export XHARNESS_DIAGNOSTICS_PATH="$2"
shift
;;
--target)
target="$2"
shift
Expand Down
1 change: 1 addition & 0 deletions tests/UnitTests.XHarness.Common.props
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
<EnableXUnitReporter>true</EnableXUnitReporter>
<EnableAzurePipelinesReporter>true</EnableAzurePipelinesReporter>
<HelixBaseUri>https://helix.dot.net</HelixBaseUri>
<EnableXHarnessTelemetry>true</EnableXHarnessTelemetry>
</PropertyGroup>

<!-- For non-ci local runs -->
Expand Down

0 comments on commit 3e6b160

Please sign in to comment.