[MLA-1009] observable performance tests

.yamato/com.unity.ml-agents-performance.yml

@@ -0,0 +1,37 @@
+test_editors:
+  - version: 2019.3
+  - version: 2020.1
+---
+{% for editor in test_editors %}
+Run_Mac_Perfomance_Tests{{ editor.version }}:
+  name: Run Mac Performance Tests {{ editor.version }}
+  agent:
+    type: Unity::VM::osx
+    image: package-ci/mac:stable
+    flavor: b1.small
+  variables:
+    UNITY_VERSION: {{ editor.version }}
+  commands:
+    - python -m pip install unity-downloader-cli --extra-index-url https://artifactory.eu-cph-1.unityops.net/api/pypi/common-python/simple
+    - unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast
+    - curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr
+    - chmod +x ./utr
+    - ./utr --suite=editor --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=com.unity.ml-agents --zero-tests-are-ok=1
+  triggers:
+    cancel_old_ci: true
+    expression: |
+      (pull_request.target eq "master" OR
+      pull_request.target match "release.+") AND
+      NOT pull_request.draft AND
+      (pull_request.changes.any match "com.unity.ml-agents/**" OR
+      pull_request.changes.any match "DevProject/**" OR
+      pull_request.changes.any match "ml-agents/**" OR
+      pull_request.changes.any match "ml-agents-envs/**" OR
+      pull_request.changes.any match ".yamato/com.unity.ml-agents-performance.yml") AND
+      NOT pull_request.changes.all match "**/*.md"
+  artifacts:
+    logs:
+      paths:
+        - "build/test-results/**"
+        - "*.log"
+{% endfor %}

DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs

@@ -0,0 +1,194 @@
+using NUnit.Framework;
+using Unity.MLAgents;
+using Unity.MLAgents.Policies;
+using Unity.MLAgents.Sensors;
+using Unity.MLAgents.Sensors.Reflection;
+using Unity.PerformanceTesting;
+using UnityEngine;
+
+namespace MLAgentsExamples.Tests.Performance
+{
+    [TestFixture]
+    public class SensorPerformanceTests
+    {
+        string[] s_Markers =
+        {
+            "root.InitializeSensors",
+            "root.AgentSendState.CollectObservations",
+            "root.AgentSendState.RequestDecision"
+        };
+        const int k_NumAgentSteps = 10;
+        const int k_MeasurementCount = 25;
+        const int k_MarkerTestSteps = 10;
+
+        [SetUp]
+        public void SetUp()
+        {
+            // Step a dummy agent here, so that we don't time the Academy initialization connection attempt and
+            // any other static setup costs.
+            RunAgent<DummyAgent>(1, 0, ObservableAttributeOptions.ExamineAll);
+        }
+
+        /// <summary>
+        /// Simple Agent just used for "burning in" the Academy for testing.
+        /// </summary>
+        class DummyAgent : Agent
+        {
+            public override void CollectObservations(VectorSensor sensor)
+            {
+            }
+
+            public override void Heuristic(float[] actionsOut)
+            {
+            }
+        }
+
+        /// <summary>
+        /// Agent used for performance testing that uses the CollectObservations interface.
+        /// </summary>
+        class CollectObservationsAgent : Agent
+        {
+            public override void CollectObservations(VectorSensor sensor)
+            {
+                sensor.AddObservation(new Vector3(1, 2, 3));
+                sensor.AddObservation(new Quaternion(1, 2, 3, 4));
+            }
+
+            public override void Heuristic(float[] actionsOut)
+            {
+            }
+        }
+
+        /// <summary>
+        /// Agent used for performance testing that uses the ObservableAttributes on fields.
+        /// </summary>
+        class ObservableFieldAgent : Agent
+        {
+            [Observable]
+            public Vector3 Vector3Field = new Vector3(1, 2, 3);
+
+            [Observable]
+            public Quaternion QuaternionField = new Quaternion(1, 2, 3, 4);
+
+            public override void Heuristic(float[] actionsOut)
+            {
+            }
+        }
+
+        /// <summary>
+        /// Agent used for performance testing that uses the ObservableAttributes on properties.
+        /// </summary>
+        class ObservablePropertyAgent : Agent
+        {
+            Vector3 m_Vector3Field = new Vector3(1, 2, 3);
+
+            [Observable]
+            Vector3 Vector3Property
+            {
+                get { return m_Vector3Field; }
+            }
+
+            Quaternion m_QuaternionField = new Quaternion(1, 2, 3, 4);
+
+            [Observable]
+            Quaternion QuaternionProperty
+            {
+                get { return m_QuaternionField; }
+            }
+
+            public override void Heuristic(float[] actionsOut)
+            {
+            }
+        }
+
+        void RunAgent<T>(int numSteps, int obsSize, ObservableAttributeOptions obsOptions) where T : Agent
+        {
+            var agentGameObj = new GameObject();
+            var agent = agentGameObj.AddComponent<T>();
+
+            var behaviorParams = agent.GetComponent<BehaviorParameters>();
+            behaviorParams.BrainParameters.VectorObservationSize = obsSize;
+            behaviorParams.ObservableAttributeHandling = obsOptions;
+
+            agent.LazyInitialize();
+            for (var i = 0; i < numSteps; i++)
+            {
+                agent.RequestDecision();
+                Academy.Instance.EnvironmentStep();
+            }
+            Object.DestroyImmediate(agentGameObj);
+        }
+
+        [Test, Performance]
+        public void TestCollectObservationsAgent()
+        {
+            Measure.Method(() =>
+            {
+                RunAgent<CollectObservationsAgent>(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore);
+            })
+                .MeasurementCount(k_MeasurementCount)
+                .GC()
+                .Run();
+        }
+
+        [Test, Performance]
+        public void TestObservableFieldAgent()
+        {
+            Measure.Method(() =>
+            {
+                RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited);
+            })
+                .MeasurementCount(k_MeasurementCount)
+                .GC()
+                .Run();
+        }
+
+        [Test, Performance]
+        public void TestObservablePropertyAgent()
+        {
+            Measure.Method(() =>
+                {
+                    RunAgent<ObservablePropertyAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited);
+                })
+                .MeasurementCount(k_MeasurementCount)
+                .GC()
+                .Run();
+        }
+
+        [Test, Performance]
+        public void TestCollectObservationsAgentMarkers()
+        {
+            using (Measure.ProfilerMarkers(s_Markers))
+            {
+                for(var i=0; i<k_MarkerTestSteps; i++)
+                {
+                    RunAgent<CollectObservationsAgent>(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore);
+                }
+            }
+        }
+
+        [Test, Performance]
+        public void TestObservableFieldAgentMarkers()
+        {
+            using (Measure.ProfilerMarkers(s_Markers))
+            {
+                for (var i = 0; i < k_MarkerTestSteps; i++)
+                {
+                    RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited);
+                }
+            }
+        }
+
+        [Test, Performance]
+        public void TestObservablePropertyAgentMarkers()
+        {
+            using (Measure.ProfilerMarkers(s_Markers))
+            {
+                for (var i = 0; i < k_MarkerTestSteps; i++)
+                {
+                    RunAgent<ObservableFieldAgent>(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited);
+                }
+            }
+        }
+    }
+}

DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef

@@ -0,0 +1,28 @@
+{
+    "name": "Unity.ML-Agents.Performance.Tests",
+    "references": [
+        "Unity.ML-Agents.Editor",
+        "Unity.ML-Agents",
+        "Unity.Barracuda",
+        "Unity.ML-Agents.CommunicatorObjects",
+        "Unity.PerformanceTesting"
+    ],
+    "optionalUnityReferences": [
+        "TestAssemblies"
+    ],
+    "includePlatforms": [
+        "Editor"
+    ],
+    "excludePlatforms": [],
+    "allowUnsafeCode": false,
+    "overrideReferences": true,
+    "precompiledReferences": [
+        "System.IO.Abstractions.dll",
+        "System.IO.Abstractions.TestingHelpers.dll",
+        "Google.Protobuf.dll"
+    ],
+    "autoReferenced": false,
+    "defineConstraints": [
+        "UNITY_INCLUDE_TESTS"
+    ]
+}

DevProject/Packages/manifest.json

@@ -15,6 +15,7 @@
     "com.unity.package-validation-suite": "0.11.0-preview",
     "com.unity.purchasing": "2.0.6",
     "com.unity.test-framework": "1.1.13",
+    "com.unity.test-framework.performance": "2.2.0-preview",
     "com.unity.testtools.codecoverage": "0.2.2-preview",
     "com.unity.textmeshpro": "2.0.1",
     "com.unity.timeline": "1.2.12",

com.unity.ml-agents/Runtime/Agent.cs

@@ -926,7 +926,10 @@ void SendInfoToBrain()
             m_Info.maxStepReached = false;
             m_Info.episodeId = m_EpisodeId;
 
-            m_Brain.RequestDecision(m_Info, sensors);
+            using (TimerStack.Instance.Scoped("RequestDecision"))
+            {
+                m_Brain.RequestDecision(m_Info, sensors);
+            }
 
             // If we have any DemonstrationWriters, write the AgentInfo and sensors to them.
             foreach (var demoWriter in DemonstrationWriters)