[tests] Run .apk unit tests and collect TestResults.xml (#305)

What do we want? Execution of unit tests!

When do we want it? Uh...4 months ago?

The `Xamarin.Android.NUnitLite` assembly allows writing NUnit-based
unit tests that execute on-device, and the `Mono.Android-Tests.csproj`
project contains a number of such unit tests.

The problem is that these unit tests aren't executed as part of the
Jenkins build process, so there's no way to know if a given commit
actually breaks anything.

In short, the existence of those unit tests is meaningless.

The task, then, is to fix the `make run-all-tests` target so that it
runs on-device unit tests...on an Android device.

Which raises all manner of problems. :-) (Hence 4+ months!)

For starters, our internal tooling is a horrible mish-mash of make(1),
ruby(1), bash(1), which creates an emulator, launches it, installs the
test .apk onto the emulator, and runs the tests. I don't want all of
that "cruft" in this repo, which means it needs to be rewritten in a
form more amenable to this repo: MSBuild tasks....and some make(1).

:-)

Add a new `build-tools/scripts/UnitTestApks.targets` file, which will
process an `@(UnitTestApk)` Item Group to permit deploying, running,
and undeploying test .apks from an attached Android device.

Add a slew of MSBuild tasks to support `@(UnitTestApk)`.

Update the default `$(AndroidSupportedTargetJitAbis)` value to be
`armeabi-v7a:x86`. The created Android emulator is x86.

Author: jonpryor

.gitignore

@@ -8,3 +8,4 @@ packages
 .DS_Store
 .nuget
 TestResult.xml
+TestResult-*.xml

Configuration.props

@@ -38,7 +38,7 @@
     <AndroidSdkDirectory>$(AndroidToolchainDirectory)\sdk</AndroidSdkDirectory>
     <AndroidNdkDirectory>$(AndroidToolchainDirectory)\ndk</AndroidNdkDirectory>
     <AndroidSupportedHostJitAbis Condition=" '$(AndroidSupportedHostJitAbis)' == '' ">$(HostOS)</AndroidSupportedHostJitAbis>
-    <AndroidSupportedTargetJitAbis Condition=" '$(AndroidSupportedTargetJitAbis)' == '' ">armeabi-v7a</AndroidSupportedTargetJitAbis>
+    <AndroidSupportedTargetJitAbis Condition=" '$(AndroidSupportedTargetJitAbis)' == '' ">armeabi-v7a:x86</AndroidSupportedTargetJitAbis>
     <JavaInteropSourceDirectory Condition=" '$(JavaInteropSourceDirectory)' == '' ">$(MSBuildThisFileDirectory)external\Java.Interop</JavaInteropSourceDirectory>
     <LlvmSourceDirectory Condition=" '$(LlvmSourceDirectory)' == '' ">$(MSBuildThisFileDirectory)external\llvm</LlvmSourceDirectory>
     <MonoSourceDirectory>$(MSBuildThisFileDirectory)external\mono</MonoSourceDirectory>
@@ -70,6 +70,14 @@
     <LibZipSourceFullPath>$([System.IO.Path]::GetFullPath ('$(LibZipSourceDirectory)'))</LibZipSourceFullPath>
     <LibZipSharpSourceFullPath>$([System.IO.Path]::GetFullPath ('$(LibZipSharpSourceDirectory)'))</LibZipSharpSourceFullPath>
   </PropertyGroup>
+  <PropertyGroup>
+    <AdbToolPath Condition=" '$(AdbToolPath)' == '' ">$(AndroidSdkFullPath)\platform-tools</AdbToolPath>
+    <AdbToolExe Condition=" '$(AdbToolExe)' == '' ">adb</AdbToolExe>
+    <AndroidToolPath Condition=" '$(AndroidToolPath)' == '' ">$(AndroidSdkFullPath)\tools</AndroidToolPath>
+    <AndroidToolExe Condition=" '$(AndroidToolExe)' == '' ">android</AndroidToolExe>
+    <EmulatorToolPath Condition=" '$(EmulatorToolPath)' == '' ">$(AndroidSdkFullPath)\tools</EmulatorToolPath>
+    <EmulatorToolExe Condition=" '$(EmulatorToolExe)' == '' ">emulator</EmulatorToolExe>
+  </PropertyGroup>
   <!--
     "Fixup" $(AndroidSupportedHostJitAbis) so that Condition attributes elsewhere
     can use `:ABI-NAME:`, to avoid substring mismatches.

Documentation/DevelopmentTips.md

@@ -0,0 +1,77 @@
+# Development Tips
+
+Tips and tricks while developing Xamarin.Android.
+
+# How do I rebuild the Mono Runtime and Native Binaries?
+
+The various Mono runtimes -- over *20* of them (!) -- all store object code
+within `build-tools/mono-runtimes/obj/$(Configuration)/TARGET`.
+
+If you change sources within `external/mono`, a top-level `make`/`xbuild`
+invocation may not rebuild those mono native binaries. To explicitly rebuild
+*all* Mono runtimes, use the `ForceBuild` target:
+
+	# Build and install all runtimes
+	$ xbuild /t:ForceBuild build-tools/mono-runtimes/mono-runtimes.mdproj
+
+To build Mono for a specific target, run `make` from the relevant directory
+and invoke the `_InstallRuntimes` target. For example, to rebuild
+Mono for armeabi-v7a:
+
+	$ cd build-tools/mono-runtimes
+	$ make -C obj/Debug/armeabi-v7a
+	
+	# This updates bin/$(Configuration)/lib/xbuild/Xamarin/Android/lib/armeabi-v7a/libmonosgen-2.0.so
+	$ xbuild /t:_InstallRuntimes
+
+# How do I rebuild BCL assemblies?
+
+The Xamarin.Android Base Class Library assemblies, such as `mscorlib.dll`,
+are built within `external/mono`, using Mono's normal build system:
+
+	# This updates external/mono/mcs/class/lib/monodroid/ASSEMBLY.dll
+	$ make -C external/mono/mcs/class/ASSEMBLY PROFILE=monodroid
+
+Alternatively, if you want to rebuild *all* the assemblies, the "host"
+Mono needs to be rebuilt. Note that the name of the "host" directory
+varies based on the operating system you're building from:
+
+	$ make -C build-tools/mono-runtimes/obj/Debug/host-Darwin
+
+Once the assemblies have been rebuilt, they can be copied into the appropriate
+Xamarin.Android SDK directory by using the `_InstallBcl` target:
+
+	# This updates bin/$(Configuration)/lib/xbuild-frameworks/MonoAndroid/v1.0/ASSEMBLY.dll
+	$ xbuild build-tools/mono-runtimes/mono-runtimes.mdproj /t:_InstallBcl
+
+# Testing Updated Assemblies
+
+The `xamarin-android` repo does not support [fast deployment][fastdep],
+which means that, *normally*, if you wanted to e.g. test a fix within
+`Mono.Android.dll` you would need to:
+
+[fastdev]: https://developer.xamarin.com/guides/android/under_the_hood/build_process/#Fast_Deployment
+
+1. Build `src/Mono.Android/Mono.Android.csproj`
+2. Rebuild your test project, e.g.
+    `src/Mono.Android/Test/Mono.Android-Tests.csproj`
+3. Reinstall the test project
+4. Re-run the test project.
+
+The resulting `.apk`s can be quite big, e.g.
+`bin/TestDebug/Mono.Android_Tests-Signed.apk` is 59MB, so steps
+(2) through (4) can be annoyingly time consuming.
+
+Fortunately, a key part of fast deployment *is* part of the `xamarin-android`:
+an "update directory" is created by `libmono-android*.so` during process
+startup, in *non*-`RELEASE` builds. This directory is printed to `adb logcat`:
+
+	 W/monodroid( 2796): Creating public update directory: `/data/data/Mono.Android_Tests/files/.__override__`
+
+Assemblies located within the "update directory" are used *in preference to*
+assemblies located within the executing `.apk`. Assemblies can be `adb push`ed
+into the update directory:
+
+	adb push bin/Debug/lib/xbuild-frameworks/MonoAndroid/v7.1/Mono.Android.dll /data/data/Mono.Android_Tests/files/.__override__
+
+When the process restarts the new assembly will be used.

Makefile

@@ -25,7 +25,7 @@ all::
 	$(MSBUILD) $(MSBUILD_FLAGS) $(SOLUTION)
 
 all-tests::
-	tools/scripts/xabuild $(MSBUILD_FLAGS) Xamarin.Android-Tests.sln
+	MSBUILD="$(MSBUILD)" tools/scripts/xabuild $(MSBUILD_FLAGS) Xamarin.Android-Tests.sln
 
 prepare:: prepare-external prepare-props
 
@@ -142,27 +142,25 @@ define RUN_NUNIT_TEST
 	$(RUNTIME) --runtime=v4.0.0 \
 		$(NUNIT_CONSOLE) $(NUNIT_EXTRA) $(1) \
 		$(if $(RUN),-run:$(RUN)) \
+		--result=TestResult-$(basename $(notdir $(1))).xml \
 		-output=bin/Test$(CONFIGURATION)/TestOutput-$(basename $(notdir $(1))).txt ;
 endef
 
 run-nunit-tests: $(NUNIT_TESTS)
 	$(foreach t,$(NUNIT_TESTS), $(call RUN_NUNIT_TEST,$(t),1))
 
-# Test .apk projects must satisfy the following requirements:
-# 1. They must have a UnDeploy target
-# 2. They must have a Deploy target
-# 3. They must have a RunTests target
+# .apk files to test on-device need to:
+# (1) Have their .csproj files listed here
+# (2) Add a `@(UnitTestApk)` entry to `tests/RunApkTests.targets`
 TEST_APK_PROJECTS = \
 	src/Mono.Android/Test/Mono.Android-Tests.csproj
 
-# Syntax: $(call RUN_TEST_APK,path/to/project.csproj)
-define RUN_TEST_APK
+# Syntax: $(call BUILD_TEST_APK,path/to/project.csproj)
+define BUILD_TEST_APK
 	# Must use xabuild to ensure correct assemblies are resolved
-	tools/scripts/xabuild /t:SignAndroidPackage $(1) && \
-	$(MSBUILD) $(MSBUILD_FLAGS) /t:UnDeploy $(1) && \
-	$(MSBUILD) $(MSBUILD_FLAGS) /t:Deploy $(1) && \
-	$(MSBUILD) $(MSBUILD_FLAGS) /t:RunTests $(1) $(if $(ADB_TARGET),"/p:AdbTarget=$(ADB_TARGET)",)
-endef
+	MSBUILD="$(MSBUILD)" tools/scripts/xabuild /t:SignAndroidPackage $(1)
+endef	# BUILD_TEST_APK
 
 run-apk-tests:
-	$(foreach p, $(TEST_APK_PROJECTS), $(call RUN_TEST_APK, $(p)))
+	$(foreach p, $(TEST_APK_PROJECTS), $(call BUILD_TEST_APK, $(p)))
+	$(MSBUILD) $(MSBUILD_FLAGS) tests/RunApkTests.targets

README.md

@@ -290,44 +290,4 @@ We use [Bugzilla](https://bugzilla.xamarin.com/enter_bug.cgi?product=Android) to
 
 # Maintainer FAQ
 
-## How do I rebuild the Mono Runtime and Native Binaries?
-
-The various Mono runtimes -- over *20* of them (!) -- all store object code
-within `build-tools/mono-runtimes/obj/$(Configuration)/TARGET`.
-
-If you change sources within `external/mono`, a top-level `make`/`xbuild`
-invocation may not rebuild those mono native binaries. To explicitly rebuild
-*all* Mono runtimes, use the `ForceBuild` target:
-
-	# Build and install all runtimes
-	$ xbuild /t:ForceBuild build-tools/mono-runtimes/mono-runtimes.mdproj
-
-To build Mono for a specific target, run `make` from the relevant directory
-and invoke the `_InstallRuntimes` target. For example, to rebuild
-Mono for armeabi-v7a:
-
-	$ cd build-tools/mono-runtimes
-	$ make -C obj/Debug/armeabi-v7a
-	
-	# This updates bin/$(Configuration)/lib/xbuild/Xamarin/Android/lib/armeabi-v7a/libmonosgen-2.0.so
-	$ xbuild /t:_InstallRuntimes
-
-## How do I rebuild BCL assemblies?
-
-The Xamarin.Android Base Class Library assemblies, such as `mscorlib.dll`,
-are built within `external/mono`, using Mono's normal build system:
-
-	# This updates external/mono/mcs/class/lib/monodroid/ASSEMBLY.dll
-	$ make -C external/mono/mcs/class/ASSEMBLY PROFILE=monodroid
-
-Alternatively, if you want to rebuild *all* the assemblies, the "host"
-Mono needs to be rebuilt. Note that the name of the "host" directory
-varies based on the operating system you're building from:
-
-	$ make -C build-tools/mono-runtimes/obj/Debug/host-Darwin
-
-Once the assemblies have been rebuilt, they can be copied into the appropriate
-Xamarin.Android SDK directory by using the `_InstallBcl` target:
-
-	# This updates bin/$(Configuration)/lib/xbuild-frameworks/MonoAndroid/v1.0/ASSEMBLY.dll
-	$ xbuild build-tools/mono-runtimes/mono-runtimes.mdproj /t:_InstallBcl
+See [DevelopmentTips.md](Documentation/DevelopmentTips.md).

build-tools/android-toolchain/android-toolchain.projitems

@@ -86,6 +86,11 @@
       <HostOS></HostOS>
       <DestDir>extras\android\m2repository</DestDir>
     </AndroidSdkItem>
+    <AndroidSdkItem Include="sysimg_x86-21_r03.zip">
+      <HostOS></HostOS>
+      <RelUrl>sys-img/android/</RelUrl>
+      <DestDir>system-images\android-21\x86</DestDir>
+    </AndroidSdkItem>
   </ItemGroup>
   <ItemGroup>
     <_NdkToolchain Include="arm-linux-androideabi-clang" Condition="$(AndroidSupportedTargetJitAbisForConditionalChecks.Contains(':armeabi:')) Or $(AndroidSupportedTargetJitAbisForConditionalChecks.Contains(':armeabi-v7a:')) Or $(AndroidSupportedTargetAotAbisForConditionalChecks.Contains (':win-armeabi:'))">

build-tools/android-toolchain/android-toolchain.targets

@@ -32,7 +32,7 @@
       Outputs="@(_PlatformAndroidSdkItem->'$(AndroidToolchainCacheDirectory)\%(Identity)');@(_PlatformAndroidNdkItem->'$(AndroidToolchainCacheDirectory)\%(Identity)')">
     <MakeDir Directories="$(AndroidToolchainCacheDirectory)" />
     <DownloadUri
-        SourceUris="@(_PlatformAndroidSdkItem->'$(AndroidUri)/%(Identity)');@(_PlatformAndroidNdkItem->'$(AndroidUri)/%(Identity)')"
+        SourceUris="@(_PlatformAndroidSdkItem->'$(AndroidUri)/%(RelUrl)%(Identity)');@(_PlatformAndroidNdkItem->'$(AndroidUri)/%(RelUrl)%(Identity)')"
         DestinationFiles="@(_PlatformAndroidSdkItem->'$(AndroidToolchainCacheDirectory)\%(Identity)');@(_PlatformAndroidNdkItem->'$(AndroidToolchainCacheDirectory)\%(Identity)')"
     />
   </Target>

build-tools/scripts/UnitTestApks.targets

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.Adb" />
+  <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.CheckAdbTarget" />
+  <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.CreateAndroidEmulator" />
+  <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.RunInstrumentationTests" />
+  <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\Xamarin.Android.Tools.BootstrapTasks.dll" TaskName="Xamarin.Android.Tools.BootstrapTasks.StartAndroidEmulator" />
+
+  <PropertyGroup>
+    <_TestImageName>XamarinAndroidUnitTestRunner</_TestImageName>
+    <_AdbEmulatorPort>5600</_AdbEmulatorPort>
+  </PropertyGroup>
+
+  <Target Name="AcquireAndroidTarget">
+    <CheckAdbTarget
+        Condition=" '$(RequireNewEmulator)' != 'True' "
+        AdbTarget="$(AdbTarget)"
+        ToolPath="$(AdbToolPath)">
+      <Output TaskParameter="AdbTarget"     PropertyName="_AdbTarget" />
+      <Output TaskParameter="IsValidTarget" PropertyName="_ValidAdbTarget"  />
+    </CheckAdbTarget>
+    <CreateAndroidEmulator
+        Condition=" '$(_ValidAdbTarget)' != 'True' "
+        AndroidAbi="x86"
+        AndroidSdkHome="$(AndroidSdkDirectory)"
+        SdkVersion="21"
+        ImageName="$(_TestImageName)"
+        ToolExe="$(AndroidToolExe)"
+        ToolPath="$(AndroidToolPath)"
+    />
+    <StartAndroidEmulator
+        Condition=" '$(_ValidAdbTarget)' != 'True' "
+        AndroidSdkHome="$(AndroidSdkDirectory)"
+        ImageName="$(_TestImageName)"
+        Port="$(_AdbEmulatorPort)"
+        ToolExe="$(EmulatorToolExe)"
+        ToolPath="$(EmulatorToolPath)">
+      <Output TaskParameter="AdbTarget" PropertyName="_AdbTarget" />
+      <Output TaskParameter="AdbTarget" PropertyName="_EmuTarget" />
+      <Output TaskParameter="AdbProcess" PropertyName="_EmuProcess" />
+    </StartAndroidEmulator>
+    <Adb
+        Condition=" '$(_ValidAdbTarget)' != 'True' "
+        Arguments="$(_AdbTarget) wait-for-device"
+    />
+    <Message
+        Condition=" '$(_EmuTarget)' != '' "
+        Text="Launched Android emulator; `adb` target: '$(_AdbTarget)'"
+    />
+  </Target>
+
+  <Target Name="ReleaseAndroidTarget">
+    <Adb
+        Condition=" '$(_EmuTarget)' != '' "
+        Arguments="$(_EmuTarget) emu kill"
+    />
+  </Target>
+
+  <!--
+    <ItemGroup>
+      <UnitTestApk Include="ApkFile">
+        <Package></Package>
+        <InstrumentationType></InstrumentationType>
+        <ResultsPath></ResultsPath>
+      </UnitTestApk>
+    </ItemGroup>
+    -->
+
+  <Target Name="DeployUnitTestApks"
+      Condition=" '@(UnitTestApk)' != '' ">
+    <Adb
+        Arguments="$(_AdbTarget) $(AdbOptions) install -r &quot;%(UnitTestApk.Identity)&quot;"
+        ToolExe="$(AdbToolExe)"
+        ToolPath="$(AdbToolPath)"
+    />
+  </Target>
+
+  <Target Name="UndeployUnitTestApks"
+      Condition=" '@(UnitTestApk)' != '' ">
+    <Adb
+        Arguments="$(_AdbTarget) $(AdbOptions) uninstall &quot;%(UnitTestApk.Package)&quot;"
+        ToolExe="$(AdbToolExe)"
+        ToolPath="$(AdbToolPath)"
+    />
+  </Target>
+
+  <Target Name="RunUnitTestApks"
+      Condition=" '@(UnitTestApk)' != '' ">
+    <RunInstrumentationTests
+        AdbTarget="$(_AdbTarget)"
+        AdbOptions="$(AdbOptions)"
+        Component="%(UnitTestApk.Package)/%(UnitTestApk.InstrumentationType)"
+        NUnit2TestResultsFile="%(UnitTestApk.ResultsPath)"
+        ToolExe="$(AdbToolExe)"
+        ToolPath="$(AdbToolPath)">
+      <Output TaskParameter="NUnit2TestResultsFile" PropertyName="_ResultsFile "/>
+    </RunInstrumentationTests>
+  </Target>
+</Project>

build-tools/xa-prep-tasks/Xamarin.Android.BuildTools.PrepTasks/Git.cs

@@ -12,7 +12,7 @@
 
 namespace Xamarin.Android.BuildTools.PrepTasks
 {
-	public class Git : ToolTask
+	public class Git : PathToolTask
 	{
 		[Required]
 		public                  ITaskItem       WorkingDirectory            { get; set; }
@@ -26,10 +26,8 @@ protected   virtual     bool            LogTaskMessages {
 			get { return true; }
 		}
 
-		protected   override    string          ToolName {
-			get {
-				return "git" + (Path.DirectorySeparatorChar == '\\' ? ".exe" : "");
-			}
+		protected   override    string          ToolBaseName {
+			get { return "git"; }
 		}
 
 		List<string> lines;
@@ -57,23 +55,6 @@ public override bool Execute ()
 			return !Log.HasLoggedErrors;
 		}
 
-		protected override string GenerateFullPathToTool ()
-		{
-			var app     = string.IsNullOrEmpty (ToolExe) ? ToolName: ToolExe;
-			var path    = !string.IsNullOrEmpty (ToolPath) ? ToolPath : GetDirectoryFromPath (app);
-			return Path.Combine (path, app);
-		}
-
-		string GetDirectoryFromPath (string app)
-		{
-			foreach (var p in Environment.GetEnvironmentVariable ("PATH").Split (new [] { Path.PathSeparator.ToString () }, StringSplitOptions.RemoveEmptyEntries)) {
-				if (File.Exists (Path.Combine (p, app))) {
-					return p;
-				}
-			}
-			return null;
-		}
-
 		protected override string GenerateCommandLineCommands ()
 		{
 			return Arguments;
@@ -86,6 +67,7 @@ protected override string GetWorkingDirectory ()
 
 		protected override void LogEventsFromTextOutput (string singleLine, MessageImportance messageImportance)
 		{
+			base.LogEventsFromTextOutput (singleLine, messageImportance);
 			Lines.Add (singleLine);
 		}
 	}