From 2dbf68b51e4299f992bb3f7afacd6a97c371438b Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Wed, 27 May 2020 13:36:16 -0700 Subject: [PATCH 01/14] WIP perf tests --- com.unity.ml-agents/Tests/Editor/Performance.meta | 8 ++++++++ .../Editor/Performance/SensorPerformanceTests.cs | 6 ++++++ .../Editor/Performance/SensorPerformanceTests.cs.meta | 11 +++++++++++ 3 files changed, 25 insertions(+) create mode 100644 com.unity.ml-agents/Tests/Editor/Performance.meta create mode 100644 com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs create mode 100644 com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs.meta diff --git a/com.unity.ml-agents/Tests/Editor/Performance.meta b/com.unity.ml-agents/Tests/Editor/Performance.meta new file mode 100644 index 0000000000..6ef60bbaa6 --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Performance.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: f570de01b4d63477e8021f0f3a745d72 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs b/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs new file mode 100644 index 0000000000..bc040ae6a4 --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs @@ -0,0 +1,6 @@ +namespace Unity.MLAgents.Tests.Performance +{ + public class SensorPerformanceTests + { + } +} diff --git a/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs.meta b/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs.meta new file mode 100644 index 0000000000..5945b01951 --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c11a031f0b2d94ec3a90c1bd54713399 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: From 3b8e49faa4e17e915e5fa493705d3580807ff430 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Wed, 27 May 2020 13:38:10 -0700 Subject: [PATCH 02/14] WIP perf test --- DevProject/Packages/manifest.json | 7 +- DevProject/ProjectSettings/ProjectVersion.txt | 4 +- .../Performance/SensorPerformanceTests.cs | 84 +++++++++++++++++++ .../Unity.ML-Agents.Editor.Tests.asmdef | 3 +- 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/DevProject/Packages/manifest.json b/DevProject/Packages/manifest.json index c5be46dc46..ce216ec913 100644 --- a/DevProject/Packages/manifest.json +++ b/DevProject/Packages/manifest.json @@ -2,7 +2,7 @@ "dependencies": { "com.unity.2d.sprite": "1.0.0", "com.unity.2d.tilemap": "1.0.0", - "com.unity.ads": "2.0.8", + "com.unity.ads": "3.4.4", "com.unity.analytics": "3.3.5", "com.unity.coding": "0.1.0-preview.13", "com.unity.collab-proxy": "1.2.16", @@ -13,12 +13,13 @@ "com.unity.package-manager-doctools": "1.1.1-preview.3", "com.unity.package-validation-suite": "0.7.15-preview", "com.unity.purchasing": "2.0.6", - "com.unity.test-framework": "1.1.11", + "com.unity.test-framework": "1.1.13", + "com.unity.test-framework.performance": "2.2.0-preview", "com.unity.testtools.codecoverage": "0.2.2-preview", "com.unity.textmeshpro": "2.0.1", "com.unity.timeline": "1.2.12", "com.unity.ugui": "1.0.0", - "com.unity.xr.legacyinputhelpers": "1.3.8", + "com.unity.xr.legacyinputhelpers": "1.3.11", "com.unity.modules.ai": "1.0.0", "com.unity.modules.androidjni": "1.0.0", "com.unity.modules.animation": "1.0.0", diff --git a/DevProject/ProjectSettings/ProjectVersion.txt b/DevProject/ProjectSettings/ProjectVersion.txt index a8d6725795..c7d3fa9045 100644 --- a/DevProject/ProjectSettings/ProjectVersion.txt +++ b/DevProject/ProjectSettings/ProjectVersion.txt @@ -1,2 +1,2 @@ -m_EditorVersion: 2019.3.3f1 -m_EditorVersionWithRevision: 2019.3.3f1 (7ceaae5f7503) +m_EditorVersion: 2019.3.11f1 +m_EditorVersionWithRevision: 2019.3.11f1 (ceef2d848e70) diff --git a/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs b/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs index bc040ae6a4..f90bd0702f 100644 --- a/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs @@ -1,6 +1,90 @@ +using NUnit.Framework; +using Unity.MLAgents.Policies; +using Unity.MLAgents.Sensors; +using Unity.MLAgents.Sensors.Reflection; +using Unity.PerformanceTesting; +using UnityEngine; + namespace Unity.MLAgents.Tests.Performance { + [TestFixture] public class SensorPerformanceTests { + [SetUp] + public void SetUp() + { + // Run Academy initialization here, so that we don't time the connection attempt. + Academy.Instance.LazyInitialize(); + } + + class CollectObservationsAgent : Agent + { + public override void CollectObservations(VectorSensor sensor) + { + sensor.AddObservation(new Vector3(1, 2, 3)); + sensor.AddObservation(new Quaternion(1, 2, 3, 4)); + } + + public override void Heuristic(float[] actionsOut) + { + } + } + + class ObservableFieldAgent : Agent + { + [Observable] + public Vector3 Vector3Field = new Vector3(1, 2, 3); + + [Observable] + public Quaternion QuaternionField = new Quaternion(1, 2, 3, 4); + + public override void Heuristic(float[] actionsOut) + { + } + } + + void RunAgent(int numSteps, int obsSize) where T : Agent + { + var agentGameObj = new GameObject(); + var agent = agentGameObj.AddComponent(); + + var decisionRequester = agent.gameObject.AddComponent(); + decisionRequester.DecisionPeriod = 1; + decisionRequester.Awake(); + + var behaviorParams = agent.GetComponent(); + behaviorParams.BrainParameters.VectorObservationSize = obsSize; + + agent.LazyInitialize(); + for (var i = 0; i < numSteps; i++) + { + Academy.Instance.EnvironmentStep(); + } + Object.DestroyImmediate(agentGameObj); + } + + [Test, Performance] + public void TestCollectObservationsAgent() + { + Measure.Method(() => + { + RunAgent(10, 7); + }) + .MeasurementCount(10) + //.WarmupCount(5) + .Run(); + } + + [Test, Performance] + public void TestObservableFieldAgent() + { + Measure.Method(() => + { + RunAgent(10, 0); + }) + .MeasurementCount(10) + //.WarmupCount(5) + .Run(); + } } } diff --git a/com.unity.ml-agents/Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef b/com.unity.ml-agents/Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef index 6035c9b3e0..0bdb5dc9a6 100755 --- a/com.unity.ml-agents/Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef +++ b/com.unity.ml-agents/Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef @@ -4,7 +4,8 @@ "Unity.ML-Agents.Editor", "Unity.ML-Agents", "Unity.Barracuda", - "Unity.ML-Agents.CommunicatorObjects" + "Unity.ML-Agents.CommunicatorObjects", + "Unity.PerformanceTesting" ], "optionalUnityReferences": [ "TestAssemblies" From 6f2dae7a954c9c4a6ae187fd3f21fac5932aeed6 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Wed, 27 May 2020 15:21:21 -0700 Subject: [PATCH 03/14] add marker tests too --- .../Reflection/EnumReflectionSensor.cs | 4 +-- .../Performance/SensorPerformanceTests.cs | 31 +++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Sensors/Reflection/EnumReflectionSensor.cs b/com.unity.ml-agents/Runtime/Sensors/Reflection/EnumReflectionSensor.cs index 92632b16ff..99f72bc177 100644 --- a/com.unity.ml-agents/Runtime/Sensors/Reflection/EnumReflectionSensor.cs +++ b/com.unity.ml-agents/Runtime/Sensors/Reflection/EnumReflectionSensor.cs @@ -3,7 +3,7 @@ namespace Unity.MLAgents.Sensors.Reflection { - internal class EnumReflectionSensor: ReflectionSensorBase + internal class EnumReflectionSensor : ReflectionSensorBase { Array m_Values; bool m_IsFlags; @@ -24,7 +24,7 @@ internal override void WriteReflectedField(ObservationWriter writer) var enumValue = (Enum)GetReflectedValue(); int i = 0; - foreach(var val in m_Values) + foreach (var val in m_Values) { if (m_IsFlags) { diff --git a/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs b/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs index f90bd0702f..33db923045 100644 --- a/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs @@ -71,7 +71,6 @@ public void TestCollectObservationsAgent() RunAgent(10, 7); }) .MeasurementCount(10) - //.WarmupCount(5) .Run(); } @@ -83,8 +82,36 @@ public void TestObservableFieldAgent() RunAgent(10, 0); }) .MeasurementCount(10) - //.WarmupCount(5) .Run(); } + + [Test, Performance] + public void TestCollectObservationsAgentMarkers() + { + string[] markers = + { + "root.InitializeSensors", + "root.AgentSendState.CollectObservations" + }; + using (Measure.ProfilerMarkers(markers)) + { + RunAgent(10, 7); + } + } + + [Test, Performance] + public void TestObservableFieldAgentMarkers() + { + string[] markers = + { + "root.InitializeSensors", + "root.AgentSendState.CollectObservations" + }; + + using (Measure.ProfilerMarkers(markers)) + { + RunAgent(10, 0); + } + } } } From 013880615135f565c3306d12a80407bbb975d760 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Wed, 27 May 2020 17:58:33 -0700 Subject: [PATCH 04/14] move to devproject --- DevProject/Assets/ML-Agents.meta | 8 ++++++ DevProject/Assets/ML-Agents/Scripts.meta | 8 ++++++ .../Assets/ML-Agents/Scripts/Tests.meta | 8 ++++++ .../ML-Agents/Scripts/Tests/Performance.meta | 8 ++++++ .../Performance/SensorPerformanceTests.cs | 0 .../SensorPerformanceTests.cs.meta | 0 .../ML-Agents/Scripts/Tests/Tests.asmdef | 28 +++++++++++++++++++ .../ML-Agents/Scripts/Tests/Tests.asmdef.meta | 7 +++++ .../Unity.ML-Agents.Runtime.Tests.asmdef | 3 +- 9 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 DevProject/Assets/ML-Agents.meta create mode 100644 DevProject/Assets/ML-Agents/Scripts.meta create mode 100644 DevProject/Assets/ML-Agents/Scripts/Tests.meta create mode 100644 DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta rename {com.unity.ml-agents/Tests/Editor => DevProject/Assets/ML-Agents/Scripts/Tests}/Performance/SensorPerformanceTests.cs (100%) rename {com.unity.ml-agents/Tests/Editor => DevProject/Assets/ML-Agents/Scripts/Tests}/Performance/SensorPerformanceTests.cs.meta (100%) create mode 100755 DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef create mode 100644 DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta diff --git a/DevProject/Assets/ML-Agents.meta b/DevProject/Assets/ML-Agents.meta new file mode 100644 index 0000000000..9aec789642 --- /dev/null +++ b/DevProject/Assets/ML-Agents.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 8498a7556d0b348cbaff70bd8635db8b +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Assets/ML-Agents/Scripts.meta b/DevProject/Assets/ML-Agents/Scripts.meta new file mode 100644 index 0000000000..664ce4a064 --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 45f48c01abd5d47048bc3edcc6db1dd0 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests.meta b/DevProject/Assets/ML-Agents/Scripts/Tests.meta new file mode 100644 index 0000000000..bfb93955f2 --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: eb96b88fc6ae14a41949d1bd75e68d38 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta new file mode 100644 index 0000000000..a9852ddda7 --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: fb5960f38a3fc42e8a6b52c9bf47d83d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs similarity index 100% rename from com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs rename to DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs diff --git a/com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs.meta b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs.meta similarity index 100% rename from com.unity.ml-agents/Tests/Editor/Performance/SensorPerformanceTests.cs.meta rename to DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs.meta diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef b/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef new file mode 100755 index 0000000000..84a661b81e --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef @@ -0,0 +1,28 @@ +{ + "name": "Unity.ML-Agents.Performance.Tests", + "references": [ + "Unity.ML-Agents.Editor", + "Unity.ML-Agents", + "Unity.Barracuda", + "Unity.ML-Agents.CommunicatorObjects", + "Unity.PerformanceTesting" + ], + "optionalUnityReferences": [ + "TestAssemblies" + ], + "includePlatforms": [ + "Editor" + ], + "excludePlatforms": [], + "allowUnsafeCode": false, + "overrideReferences": true, + "precompiledReferences": [ + "System.IO.Abstractions.dll", + "System.IO.Abstractions.TestingHelpers.dll", + "Google.Protobuf.dll" + ], + "autoReferenced": false, + "defineConstraints": [ + "UNITY_INCLUDE_TESTS" + ] +} diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta b/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta new file mode 100644 index 0000000000..297863d31c --- /dev/null +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Tests.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 1bad3ff1107ca4051af2c89caabd728c +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/com.unity.ml-agents/Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef b/com.unity.ml-agents/Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef index f136f55054..7d4dac4455 100644 --- a/com.unity.ml-agents/Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef +++ b/com.unity.ml-agents/Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef @@ -4,7 +4,8 @@ "Unity.ML-Agents", "Unity.Barracuda", "Unity.ML-Agents.CommunicatorObjects", - "Unity.ML-Agents.Editor" + "Unity.ML-Agents.Editor", + "Unity.PerformanceTesting" ], "optionalUnityReferences": [ "TestAssemblies" From c26abecd0640afbde2352a51e8a2f452c7ee9fda Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Wed, 27 May 2020 18:31:34 -0700 Subject: [PATCH 05/14] yamato first pass --- .yamato/com.unity.ml-agents-performance.yml | 40 +++++++++++++++++++ .../Performance/SensorPerformanceTests.cs | 27 +++++-------- 2 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 .yamato/com.unity.ml-agents-performance.yml diff --git a/.yamato/com.unity.ml-agents-performance.yml b/.yamato/com.unity.ml-agents-performance.yml new file mode 100644 index 0000000000..411fff22d6 --- /dev/null +++ b/.yamato/com.unity.ml-agents-performance.yml @@ -0,0 +1,40 @@ +test_editors: + - version: 2019.3 + - version: 2020.1 +--- +{% for editor in test_editors %} +Run_Mac_Perfomance_Tests{{ editor.version }}: + name: Run Mac Performance Tests {{ editor.version }} + agent: + type: Unity::VM::osx + image: package-ci/mac:stable + flavor: b1.small + variables: + UNITY_VERSION: {{ editor.version }} + commands: + - python -m pip install unity-downloader-cli --extra-index-url https://artifactory.eu-cph-1.unityops.net/api/pypi/common-python/simple + - unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast + - curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr + - ./utr --suite=editmode --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=ML-Agents --zero-tests-are-ok=1 + triggers: + cancel_old_ci: true + expression: | + (pull_request.target eq "master" OR + pull_request.target match "release.+") AND + NOT pull_request.draft AND + (pull_request.changes.any match "com.unity.ml-agents/**" OR + pull_request.changes.any match "DevProject/**" OR + pull_request.changes.any match "ml-agents/**" OR + pull_request.changes.any match "ml-agents-envs/**" OR + pull_request.changes.any match ".yamato/training-int-tests.yml") AND + NOT pull_request.changes.all match "**/*.md" + artifacts: + logs: + paths: + - "*.log" + - "*.xml" + - "artifacts/**/*" + standalonebuild: + paths: + - "artifacts/testplayer*/**" +{% endfor %} diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs index 33db923045..3210e28962 100644 --- a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs @@ -1,20 +1,27 @@ using NUnit.Framework; +using Unity.MLAgents; using Unity.MLAgents.Policies; using Unity.MLAgents.Sensors; using Unity.MLAgents.Sensors.Reflection; using Unity.PerformanceTesting; using UnityEngine; -namespace Unity.MLAgents.Tests.Performance +namespace MLAgentsExamples.Tests.Performance { [TestFixture] public class SensorPerformanceTests { + string[] s_Markers = + { + "root.InitializeSensors", + "root.AgentSendState.CollectObservations" + }; + [SetUp] public void SetUp() { // Run Academy initialization here, so that we don't time the connection attempt. - Academy.Instance.LazyInitialize(); + var academy = Academy.Instance; } class CollectObservationsAgent : Agent @@ -50,7 +57,6 @@ void RunAgent(int numSteps, int obsSize) where T : Agent var decisionRequester = agent.gameObject.AddComponent(); decisionRequester.DecisionPeriod = 1; - decisionRequester.Awake(); var behaviorParams = agent.GetComponent(); behaviorParams.BrainParameters.VectorObservationSize = obsSize; @@ -88,12 +94,7 @@ public void TestObservableFieldAgent() [Test, Performance] public void TestCollectObservationsAgentMarkers() { - string[] markers = - { - "root.InitializeSensors", - "root.AgentSendState.CollectObservations" - }; - using (Measure.ProfilerMarkers(markers)) + using (Measure.ProfilerMarkers(s_Markers)) { RunAgent(10, 7); } @@ -102,13 +103,7 @@ public void TestCollectObservationsAgentMarkers() [Test, Performance] public void TestObservableFieldAgentMarkers() { - string[] markers = - { - "root.InitializeSensors", - "root.AgentSendState.CollectObservations" - }; - - using (Measure.ProfilerMarkers(markers)) + using (Measure.ProfilerMarkers(s_Markers)) { RunAgent(10, 0); } From b029d4203e8fb8f1d98dadab784e77c2038b7161 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Thu, 28 May 2020 08:27:57 -0700 Subject: [PATCH 06/14] chmod --- .yamato/com.unity.ml-agents-performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.yamato/com.unity.ml-agents-performance.yml b/.yamato/com.unity.ml-agents-performance.yml index 411fff22d6..84cb0d9bfd 100644 --- a/.yamato/com.unity.ml-agents-performance.yml +++ b/.yamato/com.unity.ml-agents-performance.yml @@ -15,6 +15,7 @@ Run_Mac_Perfomance_Tests{{ editor.version }}: - python -m pip install unity-downloader-cli --extra-index-url https://artifactory.eu-cph-1.unityops.net/api/pypi/common-python/simple - unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast - curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr + - chmod +x ./utr - ./utr --suite=editmode --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=ML-Agents --zero-tests-are-ok=1 triggers: cancel_old_ci: true From fe19e57d528c20ac09f133eb3ad8521f72cb459d Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Thu, 28 May 2020 09:28:29 -0700 Subject: [PATCH 07/14] fix trigger, fix meta files --- .yamato/com.unity.ml-agents-performance.yml | 2 +- com.unity.ml-agents/Tests/Editor/Performance.meta | 8 -------- utils/validate_meta_files.py | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) delete mode 100644 com.unity.ml-agents/Tests/Editor/Performance.meta diff --git a/.yamato/com.unity.ml-agents-performance.yml b/.yamato/com.unity.ml-agents-performance.yml index 84cb0d9bfd..8c7eeef022 100644 --- a/.yamato/com.unity.ml-agents-performance.yml +++ b/.yamato/com.unity.ml-agents-performance.yml @@ -27,7 +27,7 @@ Run_Mac_Perfomance_Tests{{ editor.version }}: pull_request.changes.any match "DevProject/**" OR pull_request.changes.any match "ml-agents/**" OR pull_request.changes.any match "ml-agents-envs/**" OR - pull_request.changes.any match ".yamato/training-int-tests.yml") AND + pull_request.changes.any match ".yamato/com.unity.ml-agents-performance.yml") AND NOT pull_request.changes.all match "**/*.md" artifacts: logs: diff --git a/com.unity.ml-agents/Tests/Editor/Performance.meta b/com.unity.ml-agents/Tests/Editor/Performance.meta deleted file mode 100644 index 6ef60bbaa6..0000000000 --- a/com.unity.ml-agents/Tests/Editor/Performance.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: f570de01b4d63477e8021f0f3a745d72 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/utils/validate_meta_files.py b/utils/validate_meta_files.py index 4457181dd3..20fb76253c 100644 --- a/utils/validate_meta_files.py +++ b/utils/validate_meta_files.py @@ -2,7 +2,7 @@ def main(): - asset_paths = ["Project/Assets", "com.unity.ml-agents"] + asset_paths = ["Project/Assets", "com.unity.ml-agents", "DevProject/Assets"] meta_suffix = ".meta" python_suffix = ".py" whitelist = frozenset( From 75e1b24502b5b45eff3eeb19850b9a63bba1bba1 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Thu, 28 May 2020 09:58:19 -0700 Subject: [PATCH 08/14] fix utr command --- .yamato/com.unity.ml-agents-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/com.unity.ml-agents-performance.yml b/.yamato/com.unity.ml-agents-performance.yml index 8c7eeef022..380b3ab19e 100644 --- a/.yamato/com.unity.ml-agents-performance.yml +++ b/.yamato/com.unity.ml-agents-performance.yml @@ -16,7 +16,7 @@ Run_Mac_Perfomance_Tests{{ editor.version }}: - unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast - curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr - chmod +x ./utr - - ./utr --suite=editmode --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=ML-Agents --zero-tests-are-ok=1 + - ./utr --suite=editor --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=ML-Agents --zero-tests-are-ok=1 triggers: cancel_old_ci: true expression: | From c08aaa4b861ca1da7c93e8573a6e4d13b6fa13ed Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Thu, 28 May 2020 10:26:16 -0700 Subject: [PATCH 09/14] fix artifact paths --- .yamato/com.unity.ml-agents-performance.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.yamato/com.unity.ml-agents-performance.yml b/.yamato/com.unity.ml-agents-performance.yml index 380b3ab19e..4cb79581c6 100644 --- a/.yamato/com.unity.ml-agents-performance.yml +++ b/.yamato/com.unity.ml-agents-performance.yml @@ -32,10 +32,6 @@ Run_Mac_Perfomance_Tests{{ editor.version }}: artifacts: logs: paths: + - "build/test-results/**" - "*.log" - - "*.xml" - - "artifacts/**/*" - standalonebuild: - paths: - - "artifacts/testplayer*/**" {% endfor %} From 030cc20781964b982dc16d6a9f0283f3f53df09c Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 29 May 2020 09:26:10 -0700 Subject: [PATCH 10/14] Update com.unity.ml-agents-performance.yml --- .yamato/com.unity.ml-agents-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/com.unity.ml-agents-performance.yml b/.yamato/com.unity.ml-agents-performance.yml index 4cb79581c6..62351afae9 100644 --- a/.yamato/com.unity.ml-agents-performance.yml +++ b/.yamato/com.unity.ml-agents-performance.yml @@ -16,7 +16,7 @@ Run_Mac_Perfomance_Tests{{ editor.version }}: - unity-downloader-cli -u {{ editor.version }} -c editor --wait --fast - curl -s https://artifactory.internal.unity3d.com/core-automation/tools/utr-standalone/utr --output utr - chmod +x ./utr - - ./utr --suite=editor --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=ML-Agents --zero-tests-are-ok=1 + - ./utr --suite=editor --platform=StandaloneOSX --editor-location=.Editor --testproject=DevProject --artifacts_path=build/test-results --report-performance-data --performance-project-id=com.unity.ml-agents --zero-tests-are-ok=1 triggers: cancel_old_ci: true expression: | From 692656fa8698f19e12fc8b731d504a83c0f08555 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 1 Jun 2020 11:31:59 -0700 Subject: [PATCH 11/14] test properties, reduce some noise --- .../Performance/SensorPerformanceTests.cs | 102 ++++++++++++++++-- 1 file changed, 91 insertions(+), 11 deletions(-) diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs index 3210e28962..c31fb87cb9 100644 --- a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs @@ -16,14 +16,35 @@ public class SensorPerformanceTests "root.InitializeSensors", "root.AgentSendState.CollectObservations" }; + const int k_NumAgentSteps = 10; + const int k_MeasurementCount = 25; + const int k_MarkerTestSteps = 10; [SetUp] public void SetUp() { - // Run Academy initialization here, so that we don't time the connection attempt. - var academy = Academy.Instance; + // Step a dummy agent here, so that we don't time the Academy initialization connection attempt and + // any other static setup costs. + RunAgent(1, 0); } + /// + /// Simple Agent just used for "burning in" the Academy for testing. + /// + class DummyAgent : Agent + { + public override void CollectObservations(VectorSensor sensor) + { + } + + public override void Heuristic(float[] actionsOut) + { + } + } + + /// + /// Agent used for performance testing that uses the CollectObservations interface. + /// class CollectObservationsAgent : Agent { public override void CollectObservations(VectorSensor sensor) @@ -37,6 +58,9 @@ public override void Heuristic(float[] actionsOut) } } + /// + /// Agent used for performance testing that uses the ObservableAttributes on fields. + /// class ObservableFieldAgent : Agent { [Observable] @@ -50,20 +74,44 @@ public override void Heuristic(float[] actionsOut) } } + /// + /// Agent used for performance testing that uses the ObservableAttributes on properties. + /// + class ObservablePropertyAgent : Agent + { + Vector3 m_Vector3Field = new Vector3(1, 2, 3); + + [Observable] + Vector3 Vector3Property + { + get { return m_Vector3Field; } + } + + Quaternion m_QuaternionField = new Quaternion(1, 2, 3, 4); + + [Observable] + Quaternion QuaternionProperty + { + get { return m_QuaternionField; } + } + + public override void Heuristic(float[] actionsOut) + { + } + } + void RunAgent(int numSteps, int obsSize) where T : Agent { var agentGameObj = new GameObject(); var agent = agentGameObj.AddComponent(); - var decisionRequester = agent.gameObject.AddComponent(); - decisionRequester.DecisionPeriod = 1; - var behaviorParams = agent.GetComponent(); behaviorParams.BrainParameters.VectorObservationSize = obsSize; agent.LazyInitialize(); for (var i = 0; i < numSteps; i++) { + agent.RequestDecision(); Academy.Instance.EnvironmentStep(); } Object.DestroyImmediate(agentGameObj); @@ -74,9 +122,10 @@ public void TestCollectObservationsAgent() { Measure.Method(() => { - RunAgent(10, 7); + RunAgent(k_NumAgentSteps, 7); }) - .MeasurementCount(10) + .MeasurementCount(k_MeasurementCount) + .GC() .Run(); } @@ -85,9 +134,22 @@ public void TestObservableFieldAgent() { Measure.Method(() => { - RunAgent(10, 0); + RunAgent(k_NumAgentSteps, 0); }) - .MeasurementCount(10) + .MeasurementCount(k_MeasurementCount) + .GC() + .Run(); + } + + [Test, Performance] + public void TestObservablePropertyAgent() + { + Measure.Method(() => + { + RunAgent(k_NumAgentSteps, 0); + }) + .MeasurementCount(k_MeasurementCount) + .GC() .Run(); } @@ -96,7 +158,10 @@ public void TestCollectObservationsAgentMarkers() { using (Measure.ProfilerMarkers(s_Markers)) { - RunAgent(10, 7); + for(var i=0; i(k_NumAgentSteps, 7); + } } } @@ -105,7 +170,22 @@ public void TestObservableFieldAgentMarkers() { using (Measure.ProfilerMarkers(s_Markers)) { - RunAgent(10, 0); + for (var i = 0; i < k_MarkerTestSteps; i++) + { + RunAgent(k_NumAgentSteps, 0); + } + } + } + + [Test, Performance] + public void TestObservablePropertyAgentMarkers() + { + using (Measure.ProfilerMarkers(s_Markers)) + { + for (var i = 0; i < k_MarkerTestSteps; i++) + { + RunAgent(k_NumAgentSteps, 0); + } } } } From 06fb18c5d8fdd6065b9ed6a5c507cc985c5f90b0 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 1 Jun 2020 13:20:39 -0700 Subject: [PATCH 12/14] timer around RequestDecision --- .../Scripts/Tests/Performance/SensorPerformanceTests.cs | 3 ++- com.unity.ml-agents/Runtime/Agent.cs | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs index c31fb87cb9..3ec7c643aa 100644 --- a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs @@ -14,7 +14,8 @@ public class SensorPerformanceTests string[] s_Markers = { "root.InitializeSensors", - "root.AgentSendState.CollectObservations" + "root.AgentSendState.CollectObservations", + "root.AgentSendState.RequestDecision" }; const int k_NumAgentSteps = 10; const int k_MeasurementCount = 25; diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index 815a7bd917..b611ee2041 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -924,7 +924,10 @@ void SendInfoToBrain() m_Info.maxStepReached = false; m_Info.episodeId = m_EpisodeId; - m_Brain.RequestDecision(m_Info, sensors); + using (TimerStack.Instance.Scoped("RequestDecision")) + { + m_Brain.RequestDecision(m_Info, sensors); + } // If we have any DemonstrationWriters, write the AgentInfo and sensors to them. foreach (var demoWriter in DemonstrationWriters) From a1ef66ac4c4e4c26001c61944dd031ec59c9b9be Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Tue, 2 Jun 2020 18:36:56 -0700 Subject: [PATCH 13/14] actually set ObservableAttributeHandling --- .../Tests/Performance/SensorPerformanceTests.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs index 3ec7c643aa..a1961bf963 100644 --- a/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs +++ b/DevProject/Assets/ML-Agents/Scripts/Tests/Performance/SensorPerformanceTests.cs @@ -26,7 +26,7 @@ public void SetUp() { // Step a dummy agent here, so that we don't time the Academy initialization connection attempt and // any other static setup costs. - RunAgent(1, 0); + RunAgent(1, 0, ObservableAttributeOptions.ExamineAll); } /// @@ -101,13 +101,14 @@ public override void Heuristic(float[] actionsOut) } } - void RunAgent(int numSteps, int obsSize) where T : Agent + void RunAgent(int numSteps, int obsSize, ObservableAttributeOptions obsOptions) where T : Agent { var agentGameObj = new GameObject(); var agent = agentGameObj.AddComponent(); var behaviorParams = agent.GetComponent(); behaviorParams.BrainParameters.VectorObservationSize = obsSize; + behaviorParams.ObservableAttributeHandling = obsOptions; agent.LazyInitialize(); for (var i = 0; i < numSteps; i++) @@ -123,7 +124,7 @@ public void TestCollectObservationsAgent() { Measure.Method(() => { - RunAgent(k_NumAgentSteps, 7); + RunAgent(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore); }) .MeasurementCount(k_MeasurementCount) .GC() @@ -135,7 +136,7 @@ public void TestObservableFieldAgent() { Measure.Method(() => { - RunAgent(k_NumAgentSteps, 0); + RunAgent(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); }) .MeasurementCount(k_MeasurementCount) .GC() @@ -147,7 +148,7 @@ public void TestObservablePropertyAgent() { Measure.Method(() => { - RunAgent(k_NumAgentSteps, 0); + RunAgent(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); }) .MeasurementCount(k_MeasurementCount) .GC() @@ -161,7 +162,7 @@ public void TestCollectObservationsAgentMarkers() { for(var i=0; i(k_NumAgentSteps, 7); + RunAgent(k_NumAgentSteps, 7, ObservableAttributeOptions.Ignore); } } } @@ -173,7 +174,7 @@ public void TestObservableFieldAgentMarkers() { for (var i = 0; i < k_MarkerTestSteps; i++) { - RunAgent(k_NumAgentSteps, 0); + RunAgent(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); } } } @@ -185,7 +186,7 @@ public void TestObservablePropertyAgentMarkers() { for (var i = 0; i < k_MarkerTestSteps; i++) { - RunAgent(k_NumAgentSteps, 0); + RunAgent(k_NumAgentSteps, 0, ObservableAttributeOptions.ExcludeInherited); } } } From b3fd4de889d95b143b422ed03e4cd9ec6ba6d5e2 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Wed, 3 Jun 2020 15:50:11 -0700 Subject: [PATCH 14/14] undo asmdef changes --- .../Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef | 3 +-- .../Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/com.unity.ml-agents/Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef b/com.unity.ml-agents/Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef index 0bdb5dc9a6..6035c9b3e0 100755 --- a/com.unity.ml-agents/Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef +++ b/com.unity.ml-agents/Tests/Editor/Unity.ML-Agents.Editor.Tests.asmdef @@ -4,8 +4,7 @@ "Unity.ML-Agents.Editor", "Unity.ML-Agents", "Unity.Barracuda", - "Unity.ML-Agents.CommunicatorObjects", - "Unity.PerformanceTesting" + "Unity.ML-Agents.CommunicatorObjects" ], "optionalUnityReferences": [ "TestAssemblies" diff --git a/com.unity.ml-agents/Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef b/com.unity.ml-agents/Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef index 7d4dac4455..f136f55054 100644 --- a/com.unity.ml-agents/Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef +++ b/com.unity.ml-agents/Tests/Runtime/Unity.ML-Agents.Runtime.Tests.asmdef @@ -4,8 +4,7 @@ "Unity.ML-Agents", "Unity.Barracuda", "Unity.ML-Agents.CommunicatorObjects", - "Unity.ML-Agents.Editor", - "Unity.PerformanceTesting" + "Unity.ML-Agents.Editor" ], "optionalUnityReferences": [ "TestAssemblies"