azdevops: xamarin#81.

rolfbjarne · Apr 5, 2019 · 1e7a104 · 1e7a104
1 parent 18bb46d
commit 1e7a104
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 13 deletions.
diff --git a/tests/sampletester/Makefile b/tests/sampletester/Makefile
@@ -12,6 +12,8 @@ endif
 
 TEST_RESULT=TestResult-$(shell date "+%Y%m%d-%H%M%S")
 
+build: bin/Debug/sampletester.dll
+
 run-tests: bin/Debug/sampletester.dll
 	$(Q) rm -f .failed-stamp
 	$(Q) mono64 --debug $(NUNIT_MSBUILD_DIR)/nunit3-console.exe $(WHERE_CONDITION) $(abspath $(CURDIR)/bin/Debug/sampletester.dll) "--result=$(abspath $(CURDIR)/$(TEST_RESULT).xml)" $${TEST_FIXTURE:+"$$TEST_FIXTURE"} --labels=All --inprocess || touch $(CURDIR)/.failed-stamp

diff --git a/tests/sampletester/README.md b/tests/sampletester/README.md
@@ -1,22 +1,31 @@
 # Sample testing
 
-These unit tests clone a series of known repositories with sample projects, and builds all the relevant projects in those repositories.
+These unit tests clone a series of known repositories that contain sample
+projects, and build all the relevant projects in those repositories.
 
-This is executed in Azure DevOps [here][1], but it must be triggered manually:
+It is executed automatically in Azure DevOps [here][1] every Saturday for the
+following branches:
 
-Queue:
-* Branch: the branch to test.
-* Commit: the commit in that branch to test.
+* master
+* d16-*
+* xcode*
 
-Requirements:
-* The commit in question must have already have packages (as GitHub statuses).
+It can also be triggered manually, but have in mind that the commit in
+question must already have packages (as GitHub statuses).
 
-It's possible to use the sample tests from one commit, and then test with a
-completely different commit, by setting the `PROVISION_FROM_COMMIT` variable
-to the commit that's to be tested:
+It's also possible to use the sample tests from one commit, and then test with
+Xamarin.iOS/Xamarin.Mac a completely different commit, by setting the
+`PROVISION_FROM_COMMIT` variable to the commit that's to be tested:
 
 ![screenshot](images/provision_from_commit.png)
 
-The previous point is still required: the commit to provision from must have packages.
+The previous point is still required: the commit to provision from must have
+packages.
 
-[1]: https://dev.azure.com/xamarin/public/_build?definitionId=20
+There are two ways to run these tests locally:
+
+* Launching xharness in server mode and execute the "Sample tests" (they're
+  disabled by default).
+* Executing `make` in this directory.
+
+[1]: https://dev.azure.com/xamarin/internal/_build?definitionId=23
diff --git a/tests/sampletester/Samples.cs b/tests/sampletester/Samples.cs
@@ -53,7 +53,7 @@ public class MacIosSampleTester : SampleTester {
 		const string ORG = "xamarin";
 		const string REPO = "mac-ios-samples";
 		const string CATEGORY = "maciossamples"; // categories can't contain dashes
-		const string HASH = "2ab4faf9254cecdf5766af573e508f9ac8691663";
+		const string HASH = "45182f311db77bb05971a22d58edfdef44a4b657";
 
 		static Dictionary<string, SampleTest> test_data = new Dictionary<string, SampleTest> {
 				// Build solution instead of csproj

diff --git a/tests/sampletester/packages.config b/tests/sampletester/packages.config
@@ -2,4 +2,5 @@
 <packages>
   <package id="NUnit" version="3.11.0" targetFramework="net45" />
   <package id="NUnit.ConsoleRunner" version="3.9.0" targetFramework="net45" />
+  <package id="NUnit.Extension.NUnitV2ResultWriter" version="3.6.0" targetFramework="net45" />
 </packages>
diff --git a/tests/xharness/Jenkins.cs b/tests/xharness/Jenkins.cs
@@ -33,6 +33,7 @@ public class Jenkins
 		public bool IncludeiOSMSBuild = true;
 		public bool IncludeMtouch;
 		public bool IncludeBtouch;
+		public bool IncludeSampleTests;
 		public bool IncludeMacBindingProject;
 		public bool IncludeSimulator = true;
 		public bool IncludeDevice;
@@ -666,6 +667,7 @@ void SelectTestsByLabel (int pull_request)
 			SetEnabled (labels, "ios-device", ref IncludeDevice);
 			SetEnabled (labels, "xtro", ref IncludeXtro);
 			SetEnabled (labels, "mac-32", ref IncludeMac32);
+			SetEnabled (labels, "sample", ref IncludeSampleTests);
 			SetEnabled (labels, "all", ref IncludeAll);
 
 			// enabled by default
@@ -963,6 +965,24 @@ Task PopulateTasksAsync ()
 			};
 			Tasks.Add (runDocsTests);
 
+			var buildSampleTests = new XBuildTask {
+				Jenkins = this,
+				TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.sln"))),
+				SpecifyPlatform = false,
+				Platform = TestPlatform.All,
+				ProjectConfiguration = "Debug",
+			};
+			var runSampleTests = new NUnitExecuteTask (buildSampleTests) {
+				TestLibrary = Path.Combine (Harness.RootDirectory, "sampletester", "bin", "Debug", "sampletester.dll"),
+				TestProject = new TestProject (Path.GetFullPath (Path.Combine (Harness.RootDirectory, "sampletester", "sampletester.csproj"))),
+				Platform = TestPlatform.All,
+				TestName = "Sample tests",
+				Timeout = TimeSpan.FromDays (1), // These can take quite a while to execute.
+				InProcess = true,
+				Ignored = !IncludeSampleTests,
+			};
+			Tasks.Add (runSampleTests);
+
 			Tasks.AddRange (CreateRunDeviceTasks ());
 
 			return Task.CompletedTask;