diff --git a/.yamato/com.unity.ml-agents-performance.yml b/.yamato/com.unity.ml-agents-performance.yml new file mode 100644 index 0000000000..62351afae9 --- /dev/null +++ b/.yamato/com.unity.ml-agents-performance.yml @@ -0,0 +1,37 @@ +test_editors: + - version: 2019.3 + - version: 2020.1 +--- +{% for editor in test_editors %} +Run_Mac_Perfomance_Tests{{ editor.version }}: + name: Run Mac Performance Tests {{ editor.version }} + agent: + type: Unity::VM::osx + image: package-ci/mac:stable + flavor: b1.small + variables: + UNITY_VERSION: {{ editor.version }} + commands: + - python -m pip install unity-downloader-cli --extra-index-url https://artifactory.eu-cph-1.unityops.net/api/pypi/common-python/simple + - unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast + - curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr + - chmod +x ./utr + - ./utr --suite=editor --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=com.unity.ml-agents --zero-tests-are-ok=1 + triggers: + cancel_old_ci: true + expression: | + (pull_request.target eq "master" OR + pull_request.target match "release.+") AND + NOT pull_request.draft AND + (pull_request.changes.any match "com.unity.ml-agents/**" OR + pull_request.changes.any match "DevProject/**" OR + pull_request.changes.any match "ml-agents/**" OR + pull_request.changes.any match "ml-agents-envs/**" OR + pull_request.changes.any match ".yamato/com.unity.ml-agents-performance.yml") AND + NOT pull_request.changes.all match "**/*.md" + artifacts: + logs: + paths: + - "build/test-results/**" + - "*.log" +{% endfor %} diff --git a/DevProject/Assets/ML-Agents.meta b/DevProject/Assets/ML-Agents.meta new file mode 100644 index 0000000000..9aec789642 --- /dev/null +++ b/DevProject/Assets/ML-Agents.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 8498a7556d0b348cbaff70bd8635db8b +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Assets/ML-Agents/Scripts.meta b/DevProject/Assets/ML-Agents/Scripts.meta new file mode 100644 index 0000000000..664ce4a064 --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 45f48c01abd5d47048bc3edcc6db1dd0 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests.meta b/DevProject/Assets/ML-Agents/Scripts/Tests.meta new file mode 100644 index 0000000000..bfb93955f2 --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: eb96b88fc6ae14a41949d1bd75e68d38 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta new file mode 100644 index 0000000000..a9852ddda7 --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: fb5960f38a3fc42e8a6b52c9bf47d83d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs new file mode 100644 index 0000000000..a1961bf963 --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs @@ -0,0 +1,194 @@ +using NUnit.Framework; +using Unity.MLAgents; +using Unity.MLAgents.Policies; +using Unity.MLAgents.Sensors; +using Unity.MLAgents.Sensors.Reflection; +using Unity.PerformanceTesting; +using UnityEngine; + +namespace MLAgentsExamples.Tests.Performance +{ + [TestFixture] + public class SensorPerformanceTests + { + string[] s_Markers = + { + "root.InitializeSensors", + "root.AgentSendState.CollectObservations", + "root.AgentSendState.RequestDecision" + }; + const int k_NumAgentSteps = 10; + const int k_MeasurementCount = 25; + const int k_MarkerTestSteps = 10; + + [SetUp] + public void SetUp() + { + // Step a dummy agent here, so that we don't time the Academy initialization connection attempt and + // any other static setup costs. + RunAgent(1, 0, ObservableAttributeOptions.ExamineAll); + } + + /// + /// Simple Agent just used for "burning in" the Academy for testing. + /// + class DummyAgent : Agent + { + public override void CollectObservations(VectorSensor sensor) + { + } + + public override void Heuristic(float[] actionsOut) + { + } + } + + /// + /// Agent used for performance testing that uses the CollectObservations interface. + /// + class CollectObservationsAgent : Agent + { + public override void CollectObservations(VectorSensor sensor) + { + sensor.AddObservation(new Vector3(1, 2, 3)); + sensor.AddObservation(new Quaternion(1, 2, 3, 4)); + } + + public override void Heuristic(float[] actionsOut) + { + } + } + + /// + /// Agent used for performance testing that uses the ObservableAttributes on fields. + /// + class ObservableFieldAgent : Agent + { + [Observable] + public Vector3 Vector3Field = new Vector3(1, 2, 3); + + [Observable] + public Quaternion QuaternionField = new Quaternion(1, 2, 3, 4); + + public override void Heuristic(float[] actionsOut) + { + } + } + + /// + /// Agent used for performance testing that uses the ObservableAttributes on properties. + /// + class ObservablePropertyAgent : Agent + { + Vector3 m_Vector3Field = new Vector3(1, 2, 3); + + [Observable] + Vector3 Vector3Property + { + get { return m_Vector3Field; } + } + + Quaternion m_QuaternionField = new Quaternion(1, 2, 3, 4); + + [Observable] + Quaternion QuaternionProperty + { + get { return m_QuaternionField; } + } + + public override void Heuristic(float[] actionsOut) + { + } + } + + void RunAgent(int numSteps, int obsSize, ObservableAttributeOptions obsOptions) where T : Agent + { + var agentGameObj = new GameObject(); + var agent = agentGameObj.AddComponent(); + + var behaviorParams = agent.GetComponent(); + behaviorParams.BrainParameters.VectorObservationSize = obsSize; + behaviorParams.ObservableAttributeHandling = obsOptions; + + agent.LazyInitialize(); + for (var i = 0; i < numSteps; i++) + { + agent.RequestDecision(); + Academy.Instance.EnvironmentStep(); + } + Object.DestroyImmediate(agentGameObj); + } + + [Test, Performance] + public void TestCollectObservationsAgent() + { + Measure.Method(() => + { + RunAgent(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore); + }) + .MeasurementCount(k_MeasurementCount) + .GC() + .Run(); + } + + [Test, Performance] + public void TestObservableFieldAgent() + { + Measure.Method(() => + { + RunAgent(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); + }) + .MeasurementCount(k_MeasurementCount) + .GC() + .Run(); + } + + [Test, Performance] + public void TestObservablePropertyAgent() + { + Measure.Method(() => + { + RunAgent(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); + }) + .MeasurementCount(k_MeasurementCount) + .GC() + .Run(); + } + + [Test, Performance] + public void TestCollectObservationsAgentMarkers() + { + using (Measure.ProfilerMarkers(s_Markers)) + { + for(var i=0; i(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore); + } + } + } + + [Test, Performance] + public void TestObservableFieldAgentMarkers() + { + using (Measure.ProfilerMarkers(s_Markers)) + { + for (var i = 0; i < k_MarkerTestSteps; i++) + { + RunAgent(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); + } + } + } + + [Test, Performance] + public void TestObservablePropertyAgentMarkers() + { + using (Measure.ProfilerMarkers(s_Markers)) + { + for (var i = 0; i < k_MarkerTestSteps; i++) + { + RunAgent(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); + } + } + } + } +} diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs.meta b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs.meta new file mode 100644 index 0000000000..5945b01951 --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c11a031f0b2d94ec3a90c1bd54713399 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef b/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef new file mode 100755 index 0000000000..84a661b81e --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef @@ -0,0 +1,28 @@ +{ + "name": "Unity.ML-Agents.Performance.Tests", + "references": [ + "Unity.ML-Agents.Editor", + "Unity.ML-Agents", + "Unity.Barracuda", + "Unity.ML-Agents.CommunicatorObjects", + "Unity.PerformanceTesting" + ], + "optionalUnityReferences": [ + "TestAssemblies" + ], + "includePlatforms": [ + "Editor" + ], + "excludePlatforms": [], + "allowUnsafeCode": false, + "overrideReferences": true, + "precompiledReferences": [ + "System.IO.Abstractions.dll", + "System.IO.Abstractions.TestingHelpers.dll", + "Google.Protobuf.dll" + ], + "autoReferenced": false, + "defineConstraints": [ + "UNITY_INCLUDE_TESTS" + ] +} diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta b/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta new file mode 100644 index 0000000000..297863d31c --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 1bad3ff1107ca4051af2c89caabd728c +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Packages/manifest.json b/DevProject/Packages/manifest.json index 520f51a8fd..40022eeead 100644 --- a/DevProject/Packages/manifest.json +++ b/DevProject/Packages/manifest.json @@ -15,6 +15,7 @@ "com.unity.package-validation-suite": "0.11.0-preview", "com.unity.purchasing": "2.0.6", "com.unity.test-framework": "1.1.13", + "com.unity.test-framework.performance": "2.2.0-preview", "com.unity.testtools.codecoverage": "0.2.2-preview", "com.unity.textmeshpro": "2.0.1", "com.unity.timeline": "1.2.12", diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index b80e70bd9b..545de9af0c 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -926,7 +926,10 @@ void SendInfoToBrain() m_Info.maxStepReached = false; m_Info.episodeId = m_EpisodeId; - m_Brain.RequestDecision(m_Info, sensors); + using (TimerStack.Instance.Scoped("RequestDecision")) + { + m_Brain.RequestDecision(m_Info, sensors); + } // If we have any DemonstrationWriters, write the AgentInfo and sensors to them. foreach (var demoWriter in DemonstrationWriters)