Move common test code into AppVeyor module (#961)

* move test code of appveyor build into its own function to expose test differences

* remove setting erroraction in appveyor module to have the same behaviour as in yaml

* fix tests due to scoping bug in PowerShell

* fix test due to powershell bug wherby $error is not defined in the local scope when being executed inside a module

* another similar test fix (same as before) and restore erroractionpreference in module again

Author: bergmeister

Tests/Engine/GetScriptAnalyzerRule.tests.ps1

@@ -142,7 +142,7 @@ Describe "Test RuleExtension" {
             }
             catch
             {
-                $Error[0].FullyQualifiedErrorId | Should -Match "PathNotFound,Microsoft.Windows.PowerShell.ScriptAnalyzer.Commands.GetScriptAnalyzerRuleCommand"
+                $_.FullyQualifiedErrorId | Should -Match "PathNotFound,Microsoft.Windows.PowerShell.ScriptAnalyzer.Commands.GetScriptAnalyzerRuleCommand"
             }
         }
 

Tests/Engine/InvokeScriptAnalyzer.tests.ps1

@@ -477,7 +477,7 @@ Describe "Test CustomizedRulePath" {
             {
                 if (-not $testingLibraryUsage)
 			    {
-                    $Error[0].FullyQualifiedErrorId | Should -Match "PathNotFound,Microsoft.Windows.PowerShell.ScriptAnalyzer.Commands.InvokeScriptAnalyzerCommand"
+                    $_.FullyQualifiedErrorId | Should -Match "PathNotFound,Microsoft.Windows.PowerShell.ScriptAnalyzer.Commands.InvokeScriptAnalyzerCommand"
                 }
             }
         }

Tests/Rules/AvoidAssignmentToAutomaticVariable.tests.ps1

@@ -59,7 +59,8 @@ Describe "AvoidAssignmentToAutomaticVariables" {
             {
                 try
                 {
-                    Set-Variable -Name $VariableName -Value 'foo' -ErrorVariable errorVariable -ErrorAction Stop
+                    # Global scope has to be used due to a bug in PS. https://github.com/PowerShell/PowerShell/issues/6378
+                    Set-Variable -Name $VariableName -Value 'foo' -ErrorVariable errorVariable -ErrorAction Stop -Scope Global
                     throw "Expected exception did not occur when assigning value to read-only variable '$VariableName'"
                 }
                 catch

appveyor.yml

@@ -47,31 +47,15 @@ build_script:
                 Invoke-AppveyorBuild -CheckoutPath $env:APPVEYOR_BUILD_FOLDER -BuildConfiguration $env:BuildConfiguration -BuildType 'NetStandard'
             }
 
-# Test scripts are not in a module function because the tests behave differently for unknown reasons in AppVeyor
 test_script:
   - ps:   |
             if ($env:PowerShellEdition -eq 'WindowsPowerShell') {
-                $modulePath = $env:PSModulePath.Split([System.IO.Path]::PathSeparator) | Where-Object { Test-Path $_} | Select-Object -First 1
-                Copy-Item "${env:APPVEYOR_BUILD_FOLDER}\out\PSScriptAnalyzer" "$modulePath\" -Recurse -Force
-                $testResultsFile = ".\TestResults.xml"
-                $testScripts = "${env:APPVEYOR_BUILD_FOLDER}\Tests\Engine","${env:APPVEYOR_BUILD_FOLDER}\Tests\Rules"
-                $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsFile -PassThru
-                (New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}", (Resolve-Path $testResultsFile))
-                if ($testResults.FailedCount -gt 0) {
-                    throw "$($testResults.FailedCount) tests failed."
-                }
+                Invoke-AppveyorTest -CheckoutPath $env:APPVEYOR_BUILD_FOLDER
             }
   - pwsh: |
             if ($env:PowerShellEdition -eq 'PowerShellCore') {
-                $modulePath = $env:PSModulePath.Split([System.IO.Path]::PathSeparator) | Where-Object { Test-Path $_} | Select-Object -First 1
-                Copy-Item "${env:APPVEYOR_BUILD_FOLDER}\out\PSScriptAnalyzer" "$modulePath\" -Recurse -Force
-                $testResultsFile = ".\TestResults.xml"
-                $testScripts = "${env:APPVEYOR_BUILD_FOLDER}\Tests\Engine","${env:APPVEYOR_BUILD_FOLDER}\Tests\Rules"
-                $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsFile -PassThru
-                (New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}", (Resolve-Path $testResultsFile))
-                if ($testResults.FailedCount -gt 0) {
-                    throw "$($testResults.FailedCount) tests failed."
-                }
+                Import-Module .\tools\appveyor.psm1 # Appveyor does not persist pwsh sessions like it does for ps
+                Invoke-AppveyorTest -CheckoutPath $env:APPVEYOR_BUILD_FOLDER
             }
 
 # Upload the project along with test results as a zip archive

tools/appveyor.psm1

@@ -67,6 +67,25 @@ function Invoke-AppVeyorBuild {
     Pop-Location
 }
 
+# Implements AppVeyor 'test_script' step
+function Invoke-AppveyorTest {
+    Param(
+        [Parameter(Mandatory)]
+        [ValidateScript( {Test-Path $_})]
+        $CheckoutPath
+    )
+
+    $modulePath = $env:PSModulePath.Split([System.IO.Path]::PathSeparator) | Where-Object { Test-Path $_} | Select-Object -First 1
+    Copy-Item "${CheckoutPath}\out\PSScriptAnalyzer" "$modulePath\" -Recurse -Force
+    $testResultsFile = ".\TestResults.xml"
+    $testScripts = "${CheckoutPath}\Tests\Engine","${CheckoutPath}\Tests\Rules"
+    $testResults = Invoke-Pester -Script $testScripts -OutputFormat NUnitXml -OutputFile $testResultsFile -PassThru
+    (New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/${env:APPVEYOR_JOB_ID}", (Resolve-Path $testResultsFile))
+    if ($testResults.FailedCount -gt 0) {
+        throw "$($testResults.FailedCount) tests failed."
+    }
+}
+
 # Implements AppVeyor 'on_finish' step
 function Invoke-AppveyorFinish {
     $stagingDirectory = (Resolve-Path ..).Path