From 0a3b1c2583907ca309128b22ab621f8b7c724e92 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Tue, 3 Nov 2020 14:36:57 -0800 Subject: [PATCH 01/22] WIP --- com.unity.ml-agents/Runtime/Analytics.meta | 3 + .../Runtime/Analytics/Events.cs | 36 ++++++++++ .../Runtime/Analytics/Events.cs.meta | 3 + .../Runtime/Analytics/InferenceAnalytics.cs | 72 +++++++++++++++++++ .../Analytics/InferenceAnalytics.cs.meta | 3 + 5 files changed, 117 insertions(+) create mode 100644 com.unity.ml-agents/Runtime/Analytics.meta create mode 100644 com.unity.ml-agents/Runtime/Analytics/Events.cs create mode 100644 com.unity.ml-agents/Runtime/Analytics/Events.cs.meta create mode 100644 com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs create mode 100644 com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta diff --git a/com.unity.ml-agents/Runtime/Analytics.meta b/com.unity.ml-agents/Runtime/Analytics.meta new file mode 100644 index 0000000000..260b85a9b3 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 8b12ac54c5224758af88c67e2af4a01e +timeCreated: 1604359666 \ No newline at end of file diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs new file mode 100644 index 0000000000..96f75f8298 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; + +namespace Unity.MLAgents.Analytics +{ + internal struct InferenceEvent + { + public string BarracudaModelSource; + public string BarracudaModelVersion; + public string BarracudaModelProducer; + public string BarracudaPackageVersion; + public int InferenceDevice; + public EventObservationSpec ObservationSpec; + public EventActionSpec ActionSpec; + public int MemorySize; + // public Int64 ModelHash; TODO ? + } + + /// + /// Simplified version of ActionSpec struct for use in analytics + /// + internal struct EventActionSpec + { + public int NumContinuousActions; + public int NumDiscreteActions; + } + + /// + /// Simplified summary of Agent observations for use in analytics + /// + internal struct EventObservationSpec + { + public int NumVectorObservations; + public List> VisualObservationSizes; + } +} diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta b/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta new file mode 100644 index 0000000000..347eebcd51 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 0a0d7cda6d74425a80775769a9283ba6 +timeCreated: 1604359798 \ No newline at end of file diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs new file mode 100644 index 0000000000..da98704555 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -0,0 +1,72 @@ +using Unity.Barracuda; +using Unity.MLAgents.Policies; +using UnityEditor; +using UnityEditor.Analytics; +using UnityEngine.Analytics; + +namespace Unity.MLAgents.Analytics +{ + internal class InferenceAnalytics + { + static bool s_EventRegistered = false; + const int k_MaxEventsPerHour = 1000; + const int k_MaxNumberOfElements = 1000; + const string k_VendorKey = "unity.ml-agents"; + const string k_EventName = "InferenceModelSet"; + + static bool EnableAnalytics() + { + if (s_EventRegistered) + { + return true; + } + + AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey); + if (result == AnalyticsResult.Ok) + { + s_EventRegistered = true; + } + + return s_EventRegistered; + } + + public static void InferenceModelSet(NNModel nnModel, InferenceDevice inferenceDevice) + { + //The event shouldn't be able to report if this is disabled but if we know we're not going to report + //Lets early out and not waste time gathering all the data + if (!EditorAnalytics.enabled) + return; + + if (!EnableAnalytics()) + return; + + var data = GetEventForModel(nnModel, inferenceDevice); + EditorAnalytics.SendEventWithLimit(k_EventName, data); + } + + static InferenceEvent GetEventForModel(NNModel nnModel, InferenceDevice inferenceDevice) + { + var barracudaModel = ModelLoader.Load(nnModel); + var inferenceEvent = new InferenceEvent(); + inferenceEvent.BarracudaModelSource = barracudaModel.IrSource; + inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion; + inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName; + + if (barracudaModel.ProducerName == "Script") + { + // .nn files don't have these fields set correctly. Make up some values. + inferenceEvent.BarracudaModelSource = "NN"; + inferenceEvent.BarracudaModelProducer = "tf2bc.py"; + } + +#if UNITY_2019_3_OR_NEWER + var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly); + inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version; +#else + inferenceEvent.BarracudaPackageVersion = "unknown"; +#endif + + return inferenceEvent; + } + } +} diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta new file mode 100644 index 0000000000..e81b2ecbb6 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: ac4c40c2394d481ebf602caa600a32f3 +timeCreated: 1604359787 \ No newline at end of file From 0c4c9d8a76118e004333a5aa6c5746aca6e2431b Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Wed, 4 Nov 2020 18:14:32 -0800 Subject: [PATCH 02/22] WIP --- com.unity.ml-agents/Runtime/Analytics/Events.cs | 1 + .../Runtime/Analytics/InferenceAnalytics.cs | 10 ++++++---- .../Runtime/Policies/BarracudaPolicy.cs | 4 +++- .../Runtime/Policies/BehaviorParameters.cs | 4 ++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs index 96f75f8298..8126f8936e 100644 --- a/com.unity.ml-agents/Runtime/Analytics/Events.cs +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs @@ -5,6 +5,7 @@ namespace Unity.MLAgents.Analytics { internal struct InferenceEvent { + public string BehaviorName; public string BarracudaModelSource; public string BarracudaModelVersion; public string BarracudaModelProducer; diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index da98704555..2b26b0c110 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -30,7 +30,7 @@ static bool EnableAnalytics() return s_EventRegistered; } - public static void InferenceModelSet(NNModel nnModel, InferenceDevice inferenceDevice) + public static void InferenceModelSet(NNModel nnModel, string behaviorName, InferenceDevice inferenceDevice) { //The event shouldn't be able to report if this is disabled but if we know we're not going to report //Lets early out and not waste time gathering all the data @@ -40,17 +40,19 @@ public static void InferenceModelSet(NNModel nnModel, InferenceDevice inferenceD if (!EnableAnalytics()) return; - var data = GetEventForModel(nnModel, inferenceDevice); - EditorAnalytics.SendEventWithLimit(k_EventName, data); + var data = GetEventForModel(nnModel, behaviorName, inferenceDevice); + //EditorAnalytics.SendEventWithLimit(k_EventName, data); } - static InferenceEvent GetEventForModel(NNModel nnModel, InferenceDevice inferenceDevice) + static InferenceEvent GetEventForModel(NNModel nnModel, string behaviorName, InferenceDevice inferenceDevice) { var barracudaModel = ModelLoader.Load(nnModel); var inferenceEvent = new InferenceEvent(); + inferenceEvent.BehaviorName = behaviorName; inferenceEvent.BarracudaModelSource = barracudaModel.IrSource; inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion; inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName; + inferenceEvent.InferenceDevice = (int)inferenceDevice; if (barracudaModel.ProducerName == "Script") { diff --git a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs index 0914566637..010d4da96b 100644 --- a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs +++ b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs @@ -45,7 +45,9 @@ internal class BarracudaPolicy : IPolicy public BarracudaPolicy( ActionSpec actionSpec, NNModel model, - InferenceDevice inferenceDevice) + InferenceDevice inferenceDevice, + string behaviorName + ) { var modelRunner = Academy.Instance.GetOrCreateModelRunner(model, actionSpec, inferenceDevice); m_ModelRunner = modelRunner; diff --git a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs index add0dc359e..1ff7542b86 100644 --- a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs @@ -212,7 +212,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen "Either assign a model, or change to a different Behavior Type." ); } - return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice); + return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName); } case BehaviorType.Default: if (Academy.Instance.IsCommunicatorOn) @@ -221,7 +221,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen } if (m_Model != null) { - return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice); + return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName); } else { From ea8dec8791ef819e189bd47b678082a882f7c530 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Thu, 19 Nov 2020 09:52:28 -0800 Subject: [PATCH 03/22] action and sensor events --- com.unity.ml-agents/Runtime/Agent.cs | 9 +++-- .../Runtime/Analytics/Events.cs | 31 +++++++++++++--- .../Runtime/Analytics/InferenceAnalytics.cs | 35 +++++++++++++++---- .../Runtime/Policies/BehaviorParameters.cs | 33 +++++++++++++++++ 4 files changed, 96 insertions(+), 12 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index a5a018d0d2..78c0fcfbe1 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -421,7 +421,8 @@ public void LazyInitialize() InitializeActuators(); } - m_Brain = m_PolicyFactory.GeneratePolicy(m_ActuatorManager.GetCombinedActionSpec(), Heuristic); + var combinedActionSpec = m_ActuatorManager.GetCombinedActionSpec(); + m_Brain = m_PolicyFactory.GeneratePolicy(combinedActionSpec, Heuristic); ResetData(); Initialize(); @@ -430,6 +431,8 @@ public void LazyInitialize() InitializeSensors(); } + m_PolicyFactory.UpdateAnalytics(sensors, combinedActionSpec); + m_Info.storedVectorActions = new float[m_ActuatorManager.TotalNumberOfActions]; // The first time the Academy resets, all Agents in the scene will be @@ -595,7 +598,9 @@ internal void ReloadPolicy() return; } m_Brain?.Dispose(); - m_Brain = m_PolicyFactory.GeneratePolicy(m_ActuatorManager.GetCombinedActionSpec(), Heuristic); + var combinedActionSpec = m_ActuatorManager.GetCombinedActionSpec(); + m_Brain = m_PolicyFactory.GeneratePolicy(combinedActionSpec, Heuristic); + m_PolicyFactory.UpdateAnalytics(sensors, combinedActionSpec); } /// diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs index 8126f8936e..ecb3faf42a 100644 --- a/com.unity.ml-agents/Runtime/Analytics/Events.cs +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using Unity.MLAgents.Actuators; +using Unity.MLAgents.Sensors; namespace Unity.MLAgents.Analytics { @@ -11,10 +13,10 @@ internal struct InferenceEvent public string BarracudaModelProducer; public string BarracudaPackageVersion; public int InferenceDevice; - public EventObservationSpec ObservationSpec; + public List ObservationSpecs; public EventActionSpec ActionSpec; public int MemorySize; - // public Int64 ModelHash; TODO ? + // public Int64 ModelHash; //TODO ? } /// @@ -24,6 +26,18 @@ internal struct EventActionSpec { public int NumContinuousActions; public int NumDiscreteActions; + public int[] BranchSizes; + + public static EventActionSpec FromActionSpec(ActionSpec actionSpec) + { + var branchSizes = actionSpec.BranchSizes ?? Array.Empty(); + return new EventActionSpec + { + NumContinuousActions = actionSpec.NumContinuousActions, + NumDiscreteActions = actionSpec.NumDiscreteActions, + BranchSizes = branchSizes, + }; + } } /// @@ -31,7 +45,16 @@ internal struct EventActionSpec /// internal struct EventObservationSpec { - public int NumVectorObservations; - public List> VisualObservationSizes; + public string SensorName; + public int[] ObservationShape; + + public static EventObservationSpec FromSensor(ISensor sensor) + { + return new EventObservationSpec + { + SensorName = sensor.GetName(), + ObservationShape = sensor.GetObservationShape(), + }; + } } } diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 2b26b0c110..22b551c143 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -1,5 +1,9 @@ +using System.Collections.Generic; using Unity.Barracuda; +using Unity.MLAgents.Actuators; +using Unity.MLAgents.Inference; using Unity.MLAgents.Policies; +using Unity.MLAgents.Sensors; using UnityEditor; using UnityEditor.Analytics; using UnityEngine.Analytics; @@ -30,21 +34,33 @@ static bool EnableAnalytics() return s_EventRegistered; } - public static void InferenceModelSet(NNModel nnModel, string behaviorName, InferenceDevice inferenceDevice) + public static void InferenceModelSet( + NNModel nnModel, + string behaviorName, + InferenceDevice inferenceDevice, + IList sensors, + ActionSpec actionSpec + ) { - //The event shouldn't be able to report if this is disabled but if we know we're not going to report - //Lets early out and not waste time gathering all the data + // The event shouldn't be able to report if this is disabled but if we know we're not going to report + // Lets early out and not waste time gathering all the data if (!EditorAnalytics.enabled) return; if (!EnableAnalytics()) return; - var data = GetEventForModel(nnModel, behaviorName, inferenceDevice); + var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); //EditorAnalytics.SendEventWithLimit(k_EventName, data); } - static InferenceEvent GetEventForModel(NNModel nnModel, string behaviorName, InferenceDevice inferenceDevice) + static InferenceEvent GetEventForModel( + NNModel nnModel, + string behaviorName, + InferenceDevice inferenceDevice, + IList sensors, + ActionSpec actionSpec + ) { var barracudaModel = ModelLoader.Load(nnModel); var inferenceEvent = new InferenceEvent(); @@ -56,7 +72,7 @@ static InferenceEvent GetEventForModel(NNModel nnModel, string behaviorName, Inf if (barracudaModel.ProducerName == "Script") { - // .nn files don't have these fields set correctly. Make up some values. + // .nn files don't have these fields set correctly. Assign some placeholder values. inferenceEvent.BarracudaModelSource = "NN"; inferenceEvent.BarracudaModelProducer = "tf2bc.py"; } @@ -68,6 +84,13 @@ static InferenceEvent GetEventForModel(NNModel nnModel, string behaviorName, Inf inferenceEvent.BarracudaPackageVersion = "unknown"; #endif + inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec); + inferenceEvent.ObservationSpecs = new List(sensors.Count); + foreach (var sensor in sensors) + { + inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); + } + return inferenceEvent; } } diff --git a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs index 1ff7542b86..39e76be325 100644 --- a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs @@ -1,8 +1,11 @@ using Unity.Barracuda; using System; +using System.Collections.Generic; using UnityEngine; using UnityEngine.Serialization; using Unity.MLAgents.Actuators; +using Unity.MLAgents.Analytics; +using Unity.MLAgents.Sensors; using Unity.MLAgents.Sensors.Reflection; namespace Unity.MLAgents.Policies @@ -241,5 +244,35 @@ internal void UpdateAgentPolicy() } agent.ReloadPolicy(); } + + internal void UpdateAnalytics(IList sensors, ActionSpec actionSpec) + { + switch (m_BehaviorType) + { + case BehaviorType.HeuristicOnly: + return; + case BehaviorType.InferenceOnly: + { + InferenceAnalytics.InferenceModelSet(m_Model, m_BehaviorName, m_InferenceDevice, sensors, actionSpec); + return; + } + case BehaviorType.Default: + if (Academy.Instance.IsCommunicatorOn) + { + return; + } + if (m_Model != null) + { + InferenceAnalytics.InferenceModelSet(m_Model, m_BehaviorName, m_InferenceDevice, sensors, actionSpec); + return; + } + else + { + return; + } + default: + return; + } + } } } From 333cc870c66cee2a844562e86fc8c74cde4032ab Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 20 Nov 2020 14:26:31 -0800 Subject: [PATCH 04/22] send from barracudapolicy instead, keep set of models logged --- com.unity.ml-agents/Runtime/Agent.cs | 3 -- .../Runtime/Analytics/InferenceAnalytics.cs | 15 ++++++++++ .../Runtime/Inference/ModelRunner.cs | 10 +++++++ .../Runtime/Policies/BarracudaPolicy.cs | 25 +++++++++++++++- .../Runtime/Policies/BehaviorParameters.cs | 29 ------------------- 5 files changed, 49 insertions(+), 33 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index 78c0fcfbe1..402bfc429c 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -431,8 +431,6 @@ public void LazyInitialize() InitializeSensors(); } - m_PolicyFactory.UpdateAnalytics(sensors, combinedActionSpec); - m_Info.storedVectorActions = new float[m_ActuatorManager.TotalNumberOfActions]; // The first time the Academy resets, all Agents in the scene will be @@ -600,7 +598,6 @@ internal void ReloadPolicy() m_Brain?.Dispose(); var combinedActionSpec = m_ActuatorManager.GetCombinedActionSpec(); m_Brain = m_PolicyFactory.GeneratePolicy(combinedActionSpec, Heuristic); - m_PolicyFactory.UpdateAnalytics(sensors, combinedActionSpec); } /// diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 22b551c143..8f4148c6a9 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -18,6 +18,8 @@ internal class InferenceAnalytics const string k_VendorKey = "unity.ml-agents"; const string k_EventName = "InferenceModelSet"; + private static HashSet s_SentModels; + static bool EnableAnalytics() { if (s_EventRegistered) @@ -25,6 +27,11 @@ static bool EnableAnalytics() return true; } + if (s_SentModels == null) + { + s_SentModels = new HashSet(); + } + AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey); if (result == AnalyticsResult.Ok) { @@ -50,6 +57,14 @@ ActionSpec actionSpec if (!EnableAnalytics()) return; + var added = s_SentModels.Add(nnModel); + + if (!added) + { + // We previously added this model. Exit so we don't resend. + return; + } + var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); //EditorAnalytics.SendEventWithLimit(k_EventName, data); } diff --git a/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs b/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs index d24c4af9c2..c5234d2ad9 100644 --- a/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs +++ b/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs @@ -86,6 +86,16 @@ public ModelRunner( actionSpec, seed, m_TensorAllocator, m_Memories, barracudaModel); } + public InferenceDevice InferenceDevice + { + get { return m_InferenceDevice; } + } + + public NNModel Model + { + get { return m_Model; } + } + static Dictionary PrepareBarracudaInputs(IEnumerable infInputs) { var inputs = new Dictionary(); diff --git a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs index 010d4da96b..a5db52a39c 100644 --- a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs +++ b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs @@ -41,23 +41,46 @@ internal class BarracudaPolicy : IPolicy List m_SensorShapes; SpaceType m_SpaceType; + private ActionSpec m_ActionSpec; + + private string m_BehaviorName; + + /// + /// Whether or not we've tried to send analytics for this model. We only ever try to send once per policy, + /// and do additional deduplication in the analytics code. + /// + private bool m_AnalyticsSent; + /// public BarracudaPolicy( ActionSpec actionSpec, NNModel model, InferenceDevice inferenceDevice, string behaviorName - ) + ) { var modelRunner = Academy.Instance.GetOrCreateModelRunner(model, actionSpec, inferenceDevice); m_ModelRunner = modelRunner; actionSpec.CheckNotHybrid(); m_SpaceType = actionSpec.NumContinuousActions > 0 ? SpaceType.Continuous : SpaceType.Discrete; + m_BehaviorName = behaviorName; + m_ActionSpec = actionSpec; } /// public void RequestDecision(AgentInfo info, List sensors) { + if (!m_AnalyticsSent) + { + m_AnalyticsSent = true; + Analytics.InferenceAnalytics.InferenceModelSet( + m_ModelRunner.Model, + m_BehaviorName, + m_ModelRunner.InferenceDevice, + sensors, + m_ActionSpec + ); + } m_AgentId = info.episodeId; m_ModelRunner?.PutObservations(info, sensors); } diff --git a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs index 39e76be325..3de7f6c0f6 100644 --- a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs @@ -245,34 +245,5 @@ internal void UpdateAgentPolicy() agent.ReloadPolicy(); } - internal void UpdateAnalytics(IList sensors, ActionSpec actionSpec) - { - switch (m_BehaviorType) - { - case BehaviorType.HeuristicOnly: - return; - case BehaviorType.InferenceOnly: - { - InferenceAnalytics.InferenceModelSet(m_Model, m_BehaviorName, m_InferenceDevice, sensors, actionSpec); - return; - } - case BehaviorType.Default: - if (Academy.Instance.IsCommunicatorOn) - { - return; - } - if (m_Model != null) - { - InferenceAnalytics.InferenceModelSet(m_Model, m_BehaviorName, m_InferenceDevice, sensors, actionSpec); - return; - } - else - { - return; - } - default: - return; - } - } } } From 2db88ff2ef59b68f02807c89d98301b24ea3ae89 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 20 Nov 2020 14:28:34 -0800 Subject: [PATCH 05/22] cleanup, undo some changes --- com.unity.ml-agents/Runtime/Agent.cs | 6 ++---- com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs | 3 --- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Agent.cs b/com.unity.ml-agents/Runtime/Agent.cs index 402bfc429c..a5a018d0d2 100644 --- a/com.unity.ml-agents/Runtime/Agent.cs +++ b/com.unity.ml-agents/Runtime/Agent.cs @@ -421,8 +421,7 @@ public void LazyInitialize() InitializeActuators(); } - var combinedActionSpec = m_ActuatorManager.GetCombinedActionSpec(); - m_Brain = m_PolicyFactory.GeneratePolicy(combinedActionSpec, Heuristic); + m_Brain = m_PolicyFactory.GeneratePolicy(m_ActuatorManager.GetCombinedActionSpec(), Heuristic); ResetData(); Initialize(); @@ -596,8 +595,7 @@ internal void ReloadPolicy() return; } m_Brain?.Dispose(); - var combinedActionSpec = m_ActuatorManager.GetCombinedActionSpec(); - m_Brain = m_PolicyFactory.GeneratePolicy(combinedActionSpec, Heuristic); + m_Brain = m_PolicyFactory.GeneratePolicy(m_ActuatorManager.GetCombinedActionSpec(), Heuristic); } /// diff --git a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs index 3de7f6c0f6..100220e320 100644 --- a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs @@ -1,11 +1,8 @@ using Unity.Barracuda; using System; -using System.Collections.Generic; using UnityEngine; using UnityEngine.Serialization; using Unity.MLAgents.Actuators; -using Unity.MLAgents.Analytics; -using Unity.MLAgents.Sensors; using Unity.MLAgents.Sensors.Reflection; namespace Unity.MLAgents.Policies From b7748da5aecda8e520dfd4716e512e4272af2398 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 20 Nov 2020 15:59:15 -0800 Subject: [PATCH 06/22] model memory and hashing --- .../Runtime/Analytics/Events.cs | 2 +- .../Runtime/Analytics/InferenceAnalytics.cs | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs index ecb3faf42a..f06b8f490b 100644 --- a/com.unity.ml-agents/Runtime/Analytics/Events.cs +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs @@ -16,7 +16,7 @@ internal struct InferenceEvent public List ObservationSpecs; public EventActionSpec ActionSpec; public int MemorySize; - // public Int64 ModelHash; //TODO ? + public string ModelHash; } /// diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 8f4148c6a9..c66017388c 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Unity.Barracuda; using Unity.MLAgents.Actuators; @@ -6,6 +7,7 @@ using Unity.MLAgents.Sensors; using UnityEditor; using UnityEditor.Analytics; +using UnityEngine; using UnityEngine.Analytics; namespace Unity.MLAgents.Analytics @@ -83,6 +85,7 @@ ActionSpec actionSpec inferenceEvent.BarracudaModelSource = barracudaModel.IrSource; inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion; inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName; + inferenceEvent.MemorySize = (int)barracudaModel.GetTensorByName(TensorNames.MemorySize)[0]; inferenceEvent.InferenceDevice = (int)inferenceDevice; if (barracudaModel.ProducerName == "Script") @@ -106,7 +109,72 @@ ActionSpec actionSpec inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); } + inferenceEvent.ModelHash = GetModelHash(barracudaModel); return inferenceEvent; } + + internal class FNVHash + { + const ulong kFNV_prime = 1099511628211; + const ulong kFNV_offset_basis = 14695981039346656037; + private const int kMaxBytes = 1024; + + public ulong hash; + + public FNVHash() + { + hash = kFNV_offset_basis; + } + + public void Append(float[] values) + { + // Limit the max number of float bytes that we hash for performance. + // This increases the chance of a collision, but this should still be extremely rare. + var bytesToHash = Mathf.Min(kMaxBytes, Buffer.ByteLength(values)); + for (var i = 0; i < bytesToHash; i++) + { + var b = Buffer.GetByte(values, i); + Update(b); + } + } + + public void Append(string value) + { + foreach (var c in value) + { + Update((byte)c); + } + } + + private void Update(byte b) + { + hash *= kFNV_prime; + hash ^= b; + } + + public override string ToString() + { + return hash.ToString(); + } + } + + static string GetModelHash(Model barracudaModel) + { + // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings) + // For these versions, we'll use a simple FNV-1 hash. + // https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function +#if UNITY_2020_1_OR_NEWER + var hash = new Hash128(); +#else + var hash = new FNVHash(); +#endif + foreach (var layer in barracudaModel.layers) + { + hash.Append(layer.name); + hash.Append(layer.weights); + } + + return hash.ToString(); + } } } From 0f24a161167f400f87ebb5ae6dbcb769e814efc6 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 4 Dec 2020 13:42:20 -0800 Subject: [PATCH 07/22] rename event and add [Serialize] --- com.unity.ml-agents/Runtime/Analytics/Events.cs | 2 ++ com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs index f06b8f490b..2558e6c0d4 100644 --- a/com.unity.ml-agents/Runtime/Analytics/Events.cs +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs @@ -22,6 +22,7 @@ internal struct InferenceEvent /// /// Simplified version of ActionSpec struct for use in analytics /// + [Serializable] internal struct EventActionSpec { public int NumContinuousActions; @@ -43,6 +44,7 @@ public static EventActionSpec FromActionSpec(ActionSpec actionSpec) /// /// Simplified summary of Agent observations for use in analytics /// + [Serializable] internal struct EventObservationSpec { public string SensorName; diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index c66017388c..19fbe1262a 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -18,7 +18,7 @@ internal class InferenceAnalytics const int k_MaxEventsPerHour = 1000; const int k_MaxNumberOfElements = 1000; const string k_VendorKey = "unity.ml-agents"; - const string k_EventName = "InferenceModelSet"; + const string k_EventName = "ml_agents_inferencemodelset"; private static HashSet s_SentModels; @@ -67,7 +67,10 @@ ActionSpec actionSpec return; } + // Note - to debug, use JsonUtility.ToJson on the event. var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); + // var s = JsonUtility.ToJson(data, true); + // Debug.Log(s); //EditorAnalytics.SendEventWithLimit(k_EventName, data); } From 59c4b3875475b48f547a56917785c15502d13117 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 4 Dec 2020 13:56:49 -0800 Subject: [PATCH 08/22] make non-editor friendly --- .../Runtime/Analytics/InferenceAnalytics.cs | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 19fbe1262a..6b8db97f21 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -5,11 +5,15 @@ using Unity.MLAgents.Inference; using Unity.MLAgents.Policies; using Unity.MLAgents.Sensors; -using UnityEditor; -using UnityEditor.Analytics; using UnityEngine; using UnityEngine.Analytics; +#if UNITY_EDITOR +using UnityEditor; +using UnityEditor.Analytics; +#endif + + namespace Unity.MLAgents.Analytics { internal class InferenceAnalytics @@ -29,20 +33,33 @@ static bool EnableAnalytics() return true; } - if (s_SentModels == null) - { - s_SentModels = new HashSet(); - } - +#if UNITY_EDITOR AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey); +#else + AnalyticsResult result = AnalyticsResult.UnsupportedPlatform; +#endif if (result == AnalyticsResult.Ok) { s_EventRegistered = true; } + if (s_EventRegistered && s_SentModels == null) + { + s_SentModels = new HashSet(); + } + return s_EventRegistered; } + public static bool IsAnalyticsEnabled() + { +#if UNITY_EDITOR + return EditorAnalytics.enabled; +#else + return false; +#endif + } + public static void InferenceModelSet( NNModel nnModel, string behaviorName, @@ -53,7 +70,7 @@ ActionSpec actionSpec { // The event shouldn't be able to report if this is disabled but if we know we're not going to report // Lets early out and not waste time gathering all the data - if (!EditorAnalytics.enabled) + if (!IsAnalyticsEnabled()) return; if (!EnableAnalytics()) @@ -71,7 +88,11 @@ ActionSpec actionSpec var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); // var s = JsonUtility.ToJson(data, true); // Debug.Log(s); +#if UNITY_EDITOR //EditorAnalytics.SendEventWithLimit(k_EventName, data); +#else + return; +#endif } static InferenceEvent GetEventForModel( From 405c4ef8f5c7a1be1a53f3adc40b07871ed6415e Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 4 Dec 2020 15:42:45 -0800 Subject: [PATCH 09/22] update hashing - use Hash128, always limit number of floats --- .../Runtime/Analytics/InferenceAnalytics.cs | 60 +++++++++++++++---- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 6b8db97f21..122ae6cceb 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -84,12 +84,12 @@ ActionSpec actionSpec return; } - // Note - to debug, use JsonUtility.ToJson on the event. var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); - // var s = JsonUtility.ToJson(data, true); - // Debug.Log(s); + // Note - to debug, use JsonUtility.ToJson on the event. + var s = JsonUtility.ToJson(data, true); + Debug.Log(s); #if UNITY_EDITOR - //EditorAnalytics.SendEventWithLimit(k_EventName, data); + EditorAnalytics.SendEventWithLimit(k_EventName, data); #else return; #endif @@ -133,7 +133,10 @@ ActionSpec actionSpec inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); } + var startTick = DateTime.Now.Ticks; inferenceEvent.ModelHash = GetModelHash(barracudaModel); + var endTick = DateTime.Now.Ticks; + Debug.Log($"hash took {(endTick - startTick) * 1e-7} seconds"); return inferenceEvent; } @@ -150,11 +153,9 @@ public FNVHash() hash = kFNV_offset_basis; } - public void Append(float[] values) + public void Append(float[] values, int startUnused, int count) { - // Limit the max number of float bytes that we hash for performance. - // This increases the chance of a collision, but this should still be extremely rare. - var bytesToHash = Mathf.Min(kMaxBytes, Buffer.ByteLength(values)); + var bytesToHash = sizeof(float) * count; for (var i = 0; i < bytesToHash; i++) { var b = Buffer.GetByte(values, i); @@ -182,20 +183,55 @@ public override string ToString() } } + struct MLAgentsHash128 + { + private Hash128 m_Hash; + + public void Append(float[] values, int startUnused, int count) + { + if (values == null) + { + return; + } + + for (var i = 0; i < count; i++) + { + var tempHash = new Hash128(); + HashUtilities.ComputeHash128(ref values[i], ref tempHash); + HashUtilities.AppendHash(ref tempHash, ref m_Hash); + } + } + + public void Append(string value) + { + var tempHash = Hash128.Compute(value); + HashUtilities.AppendHash(ref tempHash, ref m_Hash); + } + + public override string ToString() + { + return m_Hash.ToString(); + } + } + static string GetModelHash(Model barracudaModel) { // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings) - // For these versions, we'll use a simple FNV-1 hash. - // https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function + // For these versions, we'll use a simple wrapper that supports arrays of floats. #if UNITY_2020_1_OR_NEWER var hash = new Hash128(); #else - var hash = new FNVHash(); + var hash = new MLAgentsHash128(); #endif + // Limit the max number of float bytes that we hash for performance. + // This increases the chance of a collision, but this should still be extremely rare. + const int kMaxFloats = 256; + foreach (var layer in barracudaModel.layers) { hash.Append(layer.name); - hash.Append(layer.weights); + var numFloatsToHash = Mathf.Min(layer.weights.Length, kMaxFloats); + hash.Append(layer.weights, 0, numFloatsToHash); } return hash.ToString(); From 140632e93956c5f7e80ade63bb71cbe4686e3989 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 4 Dec 2020 16:16:02 -0800 Subject: [PATCH 10/22] more observation fields, use struct for dimensions --- .../Runtime/Analytics/Events.cs | 25 +++++++++++++++++-- .../Runtime/Analytics/InferenceAnalytics.cs | 22 +++++++++++----- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs index 2558e6c0d4..37ecbcd904 100644 --- a/com.unity.ml-agents/Runtime/Analytics/Events.cs +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs @@ -16,6 +16,7 @@ internal struct InferenceEvent public List ObservationSpecs; public EventActionSpec ActionSpec; public int MemorySize; + public long TotalWeightSizeBytes; public string ModelHash; } @@ -41,6 +42,16 @@ public static EventActionSpec FromActionSpec(ActionSpec actionSpec) } } + /// + /// Information about one dimension of an observation. + /// + [Serializable] + internal struct EventObservationDimensionInfo + { + public int Size; + public int Flags; + } + /// /// Simplified summary of Agent observations for use in analytics /// @@ -48,14 +59,24 @@ public static EventActionSpec FromActionSpec(ActionSpec actionSpec) internal struct EventObservationSpec { public string SensorName; - public int[] ObservationShape; + public string CompressionType; + public EventObservationDimensionInfo[] DimensionInfos; public static EventObservationSpec FromSensor(ISensor sensor) { + var shape = sensor.GetObservationShape(); + var dimInfos = new EventObservationDimensionInfo[shape.Length]; + for (var i = 0; i < shape.Length; i++) + { + dimInfos[i].Size = shape[i]; + // TODO copy flags when we have them + } + return new EventObservationSpec { SensorName = sensor.GetName(), - ObservationShape = sensor.GetObservationShape(), + CompressionType = sensor.GetCompressionType().ToString(), + DimensionInfos = dimInfos, }; } } diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 122ae6cceb..f9cc96e4d9 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -86,10 +86,9 @@ ActionSpec actionSpec var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); // Note - to debug, use JsonUtility.ToJson on the event. - var s = JsonUtility.ToJson(data, true); - Debug.Log(s); + //Debug.Log(JsonUtility.ToJson(data, true)); #if UNITY_EDITOR - EditorAnalytics.SendEventWithLimit(k_EventName, data); + //EditorAnalytics.SendEventWithLimit(k_EventName, data); #else return; #endif @@ -133,10 +132,8 @@ ActionSpec actionSpec inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); } - var startTick = DateTime.Now.Ticks; + inferenceEvent.TotalWeightSizeBytes = GetModelWeightSize(barracudaModel); inferenceEvent.ModelHash = GetModelHash(barracudaModel); - var endTick = DateTime.Now.Ticks; - Debug.Log($"hash took {(endTick - startTick) * 1e-7} seconds"); return inferenceEvent; } @@ -214,6 +211,19 @@ public override string ToString() } } + static long GetModelWeightSize(Model barracudaModel) + { + long totalWeightsSizeInBytes = 0; + for (var l = 0; l < barracudaModel.layers.Count; ++l) + { + for (var d = 0; d < barracudaModel.layers[l].datasets.Length; ++d) + { + totalWeightsSizeInBytes += barracudaModel.layers[l].datasets[d].length; + } + } + return totalWeightsSizeInBytes; + } + static string GetModelHash(Model barracudaModel) { // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings) From add10c47b707eb05c14343d99e8cc57b39f2b665 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 4 Dec 2020 17:13:28 -0800 Subject: [PATCH 11/22] docstrings --- .../Runtime/Analytics/InferenceAnalytics.cs | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index f9cc96e4d9..4becd59eb5 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -18,12 +18,27 @@ namespace Unity.MLAgents.Analytics { internal class InferenceAnalytics { + const string k_VendorKey = "unity.ml-agents"; + const string k_EventName = "ml_agents_inferencemodelset"; + + /// + /// Whether or not we've registered this particular event yet + /// static bool s_EventRegistered = false; + + /// + /// Hourly limit for this event name + /// const int k_MaxEventsPerHour = 1000; + + /// + /// Maximum number of items in this event. + /// const int k_MaxNumberOfElements = 1000; - const string k_VendorKey = "unity.ml-agents"; - const string k_EventName = "ml_agents_inferencemodelset"; + /// + /// Models that we've already sent events for. + /// private static HashSet s_SentModels; static bool EnableAnalytics() @@ -88,6 +103,7 @@ ActionSpec actionSpec // Note - to debug, use JsonUtility.ToJson on the event. //Debug.Log(JsonUtility.ToJson(data, true)); #if UNITY_EDITOR + // TODO re-enable when we're ready to merge. //EditorAnalytics.SendEventWithLimit(k_EventName, data); #else return; @@ -137,6 +153,10 @@ ActionSpec actionSpec return inferenceEvent; } + /// + /// Simple implementation of the Fowler–Noll–Vo hash function. + /// https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function + /// internal class FNVHash { const ulong kFNV_prime = 1099511628211; @@ -180,6 +200,9 @@ public override string ToString() } } + /// + /// Wrapper around Hash128 that supports Append(float[], int, int) + /// struct MLAgentsHash128 { private Hash128 m_Hash; @@ -213,6 +236,8 @@ public override string ToString() static long GetModelWeightSize(Model barracudaModel) { + // This corresponds to the "Total weight size" display in the Barracuda inspector + // and the calculations are the same. long totalWeightsSizeInBytes = 0; for (var l = 0; l < barracudaModel.layers.Count; ++l) { @@ -227,7 +252,7 @@ static long GetModelWeightSize(Model barracudaModel) static string GetModelHash(Model barracudaModel) { // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings) - // For these versions, we'll use a simple wrapper that supports arrays of floats. + // For these versions, we'll use a simple wrapper that just supports arrays of floats. #if UNITY_2020_1_OR_NEWER var hash = new Hash128(); #else From e85fd506d82bf5d2c63a3d77b96ce3bdc4041766 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 4 Dec 2020 17:33:04 -0800 Subject: [PATCH 12/22] more docstrings --- .../Runtime/Analytics/InferenceAnalytics.cs | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 4becd59eb5..3bd01cca93 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -110,6 +110,17 @@ ActionSpec actionSpec #endif } + /// + /// Send an analytics event for the NNModel when it is set up for inference. + /// No events will be sent if analytics are disabled, and at most one event + /// will be sent per model instance. + /// + /// The NNModel being used for inference. + /// The BehaviorName of the Agent using the model + /// Whether inference is being performed on the CPU or GPU + /// List of ISensors for the Agent. Used to generate information about the observation space. + /// ActionSpec for the Agent. Used to generate information about the action space. + /// static InferenceEvent GetEventForModel( NNModel nnModel, string behaviorName, @@ -234,10 +245,15 @@ public override string ToString() } } + /// + /// Compute the total model weight size in bytes. + /// This corresponds to the "Total weight size" display in the Barracuda inspector, + /// and the calculations are the same. + /// + /// + /// static long GetModelWeightSize(Model barracudaModel) { - // This corresponds to the "Total weight size" display in the Barracuda inspector - // and the calculations are the same. long totalWeightsSizeInBytes = 0; for (var l = 0; l < barracudaModel.layers.Count; ++l) { @@ -249,6 +265,13 @@ static long GetModelWeightSize(Model barracudaModel) return totalWeightsSizeInBytes; } + /// + /// Compute a hash of the model's layer data and return it as a string. + /// A subset of the layer weights are used for performance. + /// This increases the chance of a collision, but this should still be extremely rare. + /// + /// + /// static string GetModelHash(Model barracudaModel) { // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings) @@ -259,7 +282,6 @@ static string GetModelHash(Model barracudaModel) var hash = new MLAgentsHash128(); #endif // Limit the max number of float bytes that we hash for performance. - // This increases the chance of a collision, but this should still be extremely rare. const int kMaxFloats = 256; foreach (var layer in barracudaModel.layers) From 8c07e5ef5153259ad012e43ca36df841bc6491dd Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 4 Dec 2020 17:36:21 -0800 Subject: [PATCH 13/22] update docstrings --- .../Runtime/Analytics/InferenceAnalytics.cs | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 3bd01cca93..bb2e691aeb 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -75,6 +75,17 @@ public static bool IsAnalyticsEnabled() #endif } + /// + /// Send an analytics event for the NNModel when it is set up for inference. + /// No events will be sent if analytics are disabled, and at most one event + /// will be sent per model instance. + /// + /// The NNModel being used for inference. + /// The BehaviorName of the Agent using the model + /// Whether inference is being performed on the CPU or GPU + /// List of ISensors for the Agent. Used to generate information about the observation space. + /// ActionSpec for the Agent. Used to generate information about the action space. + /// public static void InferenceModelSet( NNModel nnModel, string behaviorName, @@ -110,16 +121,15 @@ ActionSpec actionSpec #endif } + /// - /// Send an analytics event for the NNModel when it is set up for inference. - /// No events will be sent if analytics are disabled, and at most one event - /// will be sent per model instance. + /// Generate an InferenceEvent for the model. /// - /// The NNModel being used for inference. - /// The BehaviorName of the Agent using the model - /// Whether inference is being performed on the CPU or GPU - /// List of ISensors for the Agent. Used to generate information about the observation space. - /// ActionSpec for the Agent. Used to generate information about the action space. + /// + /// + /// + /// + /// /// static InferenceEvent GetEventForModel( NNModel nnModel, From 9b3ece7fc3eb15f371a6835aa004c465b1fae5b8 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 7 Dec 2020 15:12:20 -0800 Subject: [PATCH 14/22] fix builds, add tests --- .../Runtime/Analytics/InferenceAnalytics.cs | 93 +++++-------------- .../Tests/Editor/Analytics.meta | 3 + .../Analytics/InferenceAnalyticsTests.cs | 63 +++++++++++++ .../Analytics/InferenceAnalyticsTests.cs.meta | 3 + 4 files changed, 92 insertions(+), 70 deletions(-) create mode 100644 com.unity.ml-agents/Tests/Editor/Analytics.meta create mode 100644 com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs create mode 100644 com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index bb2e691aeb..62bc7f1e96 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -121,7 +121,6 @@ ActionSpec actionSpec #endif } - /// /// Generate an InferenceEvent for the model. /// @@ -131,7 +130,7 @@ ActionSpec actionSpec /// /// /// - static InferenceEvent GetEventForModel( + internal static InferenceEvent GetEventForModel( NNModel nnModel, string behaviorName, InferenceDevice inferenceDevice, @@ -155,11 +154,11 @@ ActionSpec actionSpec inferenceEvent.BarracudaModelProducer = "tf2bc.py"; } -#if UNITY_2019_3_OR_NEWER +#if UNITY_2019_3_OR_NEWER && UNITY_EDITOR var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly); inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version; #else - inferenceEvent.BarracudaPackageVersion = "unknown"; + inferenceEvent.BarracudaPackageVersion = null; #endif inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec); @@ -175,50 +174,23 @@ ActionSpec actionSpec } /// - /// Simple implementation of the Fowler–Noll–Vo hash function. - /// https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function + /// Compute the total model weight size in bytes. + /// This corresponds to the "Total weight size" display in the Barracuda inspector, + /// and the calculations are the same. /// - internal class FNVHash + /// + /// + static long GetModelWeightSize(Model barracudaModel) { - const ulong kFNV_prime = 1099511628211; - const ulong kFNV_offset_basis = 14695981039346656037; - private const int kMaxBytes = 1024; - - public ulong hash; - - public FNVHash() - { - hash = kFNV_offset_basis; - } - - public void Append(float[] values, int startUnused, int count) - { - var bytesToHash = sizeof(float) * count; - for (var i = 0; i < bytesToHash; i++) - { - var b = Buffer.GetByte(values, i); - Update(b); - } - } - - public void Append(string value) + long totalWeightsSizeInBytes = 0; + for (var l = 0; l < barracudaModel.layers.Count; ++l) { - foreach (var c in value) + for (var d = 0; d < barracudaModel.layers[l].datasets.Length; ++d) { - Update((byte)c); + totalWeightsSizeInBytes += barracudaModel.layers[l].datasets[d].length; } } - - private void Update(byte b) - { - hash *= kFNV_prime; - hash ^= b; - } - - public override string ToString() - { - return hash.ToString(); - } + return totalWeightsSizeInBytes; } /// @@ -228,19 +200,25 @@ struct MLAgentsHash128 { private Hash128 m_Hash; - public void Append(float[] values, int startUnused, int count) + public void Append(float[] values, int count) { if (values == null) { return; } + // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings and scalars) + // For these versions, we'll hash element by element. +#if UNITY_2020_1_OR_NEWER + m_Hash.Append(layer.weights, 0, count); +#else for (var i = 0; i < count; i++) { var tempHash = new Hash128(); HashUtilities.ComputeHash128(ref values[i], ref tempHash); HashUtilities.AppendHash(ref tempHash, ref m_Hash); } +#endif } public void Append(string value) @@ -255,26 +233,6 @@ public override string ToString() } } - /// - /// Compute the total model weight size in bytes. - /// This corresponds to the "Total weight size" display in the Barracuda inspector, - /// and the calculations are the same. - /// - /// - /// - static long GetModelWeightSize(Model barracudaModel) - { - long totalWeightsSizeInBytes = 0; - for (var l = 0; l < barracudaModel.layers.Count; ++l) - { - for (var d = 0; d < barracudaModel.layers[l].datasets.Length; ++d) - { - totalWeightsSizeInBytes += barracudaModel.layers[l].datasets[d].length; - } - } - return totalWeightsSizeInBytes; - } - /// /// Compute a hash of the model's layer data and return it as a string. /// A subset of the layer weights are used for performance. @@ -284,13 +242,8 @@ static long GetModelWeightSize(Model barracudaModel) /// static string GetModelHash(Model barracudaModel) { - // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings) - // For these versions, we'll use a simple wrapper that just supports arrays of floats. -#if UNITY_2020_1_OR_NEWER - var hash = new Hash128(); -#else var hash = new MLAgentsHash128(); -#endif + // Limit the max number of float bytes that we hash for performance. const int kMaxFloats = 256; @@ -298,7 +251,7 @@ static string GetModelHash(Model barracudaModel) { hash.Append(layer.name); var numFloatsToHash = Mathf.Min(layer.weights.Length, kMaxFloats); - hash.Append(layer.weights, 0, numFloatsToHash); + hash.Append(layer.weights, numFloatsToHash); } return hash.ToString(); diff --git a/com.unity.ml-agents/Tests/Editor/Analytics.meta b/com.unity.ml-agents/Tests/Editor/Analytics.meta new file mode 100644 index 0000000000..473f2be08f --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Analytics.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: adbf291ff40848a296523d69a5be65a5 +timeCreated: 1607379470 \ No newline at end of file diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs new file mode 100644 index 0000000000..fb8126f84b --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; +using NUnit.Framework; +using Unity.MLAgents.Sensors; +using UnityEngine; +using Unity.Barracuda; +using Unity.MLAgents.Actuators; +using Unity.MLAgents.Analytics; +using Unity.MLAgents.Policies; +using UnityEditor; + +namespace Unity.MLAgents.Tests.Analytics +{ + [TestFixture] + public class InferenceAnalyticsTests + { + const string k_continuous2vis8vec2actionPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.nn"; + NNModel continuous2vis8vec2actionModel; + Test3DSensorComponent sensor_21_20_3; + Test3DSensorComponent sensor_20_22_3; + + ActionSpec GetContinuous2vis8vec2actionActionSpec() + { + return ActionSpec.MakeContinuous(2); + } + + [SetUp] + public void SetUp() + { + continuous2vis8vec2actionModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuous2vis8vec2actionPath, typeof(NNModel)); + var go = new GameObject("SensorA"); + sensor_21_20_3 = go.AddComponent(); + sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3); + sensor_20_22_3 = go.AddComponent(); + sensor_20_22_3.Sensor = new Test3DSensor("SensorB", 20, 22, 3); + } + + [Test] + public void TestModelEvent() + { + var sensors = new List { sensor_21_20_3.Sensor, sensor_20_22_3.Sensor }; + + var continuousEvent = InferenceAnalytics.GetEventForModel( + continuous2vis8vec2actionModel, "continuousModel", + InferenceDevice.CPU, sensors, GetContinuous2vis8vec2actionActionSpec() + ); + Assert.AreEqual(2, continuousEvent.ActionSpec.NumContinuousActions); + Assert.AreEqual(0, continuousEvent.ActionSpec.NumDiscreteActions); + Assert.AreEqual(2, continuousEvent.ObservationSpecs.Count); + Assert.AreEqual(3, continuousEvent.ObservationSpecs[0].DimensionInfos.Length); + Assert.AreEqual(20, continuousEvent.ObservationSpecs[0].DimensionInfos[0].Size); + Assert.AreEqual("None", continuousEvent.ObservationSpecs[0].CompressionType); + Assert.AreNotEqual(null, continuousEvent.ModelHash); + + // Make sure nested fields get serialized + var jsonString = JsonUtility.ToJson(continuousEvent, true); + Assert.IsTrue(jsonString.Contains("ObservationSpecs")); + Assert.IsTrue(jsonString.Contains("ActionSpec")); + Assert.IsTrue(jsonString.Contains("NumDiscreteActions")); + Assert.IsTrue(jsonString.Contains("SensorName")); + Assert.IsTrue(jsonString.Contains("Flags")); + } + } +} diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta new file mode 100644 index 0000000000..20f024f03b --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 9f054f620b8b468bbd8ccf7d2cc14ccd +timeCreated: 1607379491 \ No newline at end of file From 2ba6d3504125c269c415b87754165fd9d374cdd0 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 7 Dec 2020 15:22:02 -0800 Subject: [PATCH 15/22] fix 2020 compile --- com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 62bc7f1e96..cb99bd8e70 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -210,7 +210,7 @@ public void Append(float[] values, int count) // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings and scalars) // For these versions, we'll hash element by element. #if UNITY_2020_1_OR_NEWER - m_Hash.Append(layer.weights, 0, count); + m_Hash.Append(values, 0, count); #else for (var i = 0; i < count; i++) { From b577ec2dc8858384118355707ad620f9ee92b278 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 7 Dec 2020 18:02:51 -0800 Subject: [PATCH 16/22] Uncomment EditorAnalytics.SendEventWithLimit --- com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index cb99bd8e70..83e36be5e0 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -112,10 +112,9 @@ ActionSpec actionSpec var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); // Note - to debug, use JsonUtility.ToJson on the event. - //Debug.Log(JsonUtility.ToJson(data, true)); + // Debug.Log(JsonUtility.ToJson(data, true)); #if UNITY_EDITOR - // TODO re-enable when we're ready to merge. - //EditorAnalytics.SendEventWithLimit(k_EventName, data); + EditorAnalytics.SendEventWithLimit(k_EventName, data); #else return; #endif From 3b7f7425f38936ee95f8f1a73471c3c598b22648 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Tue, 8 Dec 2020 09:47:48 -0800 Subject: [PATCH 17/22] changelog: --- com.unity.ml-agents/CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index c68212bf91..54e36b7672 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -14,8 +14,11 @@ and this project adheres to ### Minor Changes #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#) +- Analytics events are now collected when performing inference in the editor. For more details on what is collected and +how to opt out, see the [package README](Documentation~/com.unity.ml-agents.md). (#4677) + #### ml-agents / ml-agents-envs / gym-unity (Python) -- `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`. +- `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`. (#4691) ### Bug Fixes #### com.unity.ml-agents (C#) From b9a4acaac23eec0317d9bf4def997e056c24178f Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Wed, 9 Dec 2020 17:13:56 -0800 Subject: [PATCH 18/22] better changelog --- com.unity.ml-agents/CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index 54e36b7672..1e9a939c64 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -14,8 +14,10 @@ and this project adheres to ### Minor Changes #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#) -- Analytics events are now collected when performing inference in the editor. For more details on what is collected and -how to opt out, see the [package README](Documentation~/com.unity.ml-agents.md). (#4677) +- If [Unity Editor Analytics](https://docs.unity3d.com/Manual/EditorAnalytics.html) are enabled, ML-Agents collects +anonymous data about models that are used for inference. This helps us focus our efforts on the most commonly used +features, and better understand the needs of our users. We compute a hash of a subset of model weights but cannot +recreate your model from the collected information. (#4677) #### ml-agents / ml-agents-envs / gym-unity (Python) - `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`. (#4691) From de08a28df109fd1a20e1c011d8f86ff8ad58a94d Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Fri, 11 Dec 2020 11:41:31 -0800 Subject: [PATCH 19/22] hash behavior name --- com.unity.ml-agents/Runtime/Analytics/Events.cs | 6 ++++++ .../Runtime/Analytics/InferenceAnalytics.cs | 8 ++++++-- .../Tests/Editor/Analytics/InferenceAnalyticsTests.cs | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs index 37ecbcd904..9dd5292318 100644 --- a/com.unity.ml-agents/Runtime/Analytics/Events.cs +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs @@ -7,11 +7,17 @@ namespace Unity.MLAgents.Analytics { internal struct InferenceEvent { + /// + /// Hash of the BehaviorName. + /// public string BehaviorName; public string BarracudaModelSource; public string BarracudaModelVersion; public string BarracudaModelProducer; public string BarracudaPackageVersion; + /// + /// Whether inference is performed on CPU (0) or GPU (1). + /// public int InferenceDevice; public List ObservationSpecs; public EventActionSpec ActionSpec; diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs index 83e36be5e0..0b509eb491 100644 --- a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -139,7 +139,11 @@ ActionSpec actionSpec { var barracudaModel = ModelLoader.Load(nnModel); var inferenceEvent = new InferenceEvent(); - inferenceEvent.BehaviorName = behaviorName; + + // Hash the behavior name so that there's no concern about PII or "secret" data being leaked. + var behaviorNameHash = Hash128.Compute(behaviorName); + inferenceEvent.BehaviorName = behaviorNameHash.ToString(); + inferenceEvent.BarracudaModelSource = barracudaModel.IrSource; inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion; inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName; @@ -150,7 +154,7 @@ ActionSpec actionSpec { // .nn files don't have these fields set correctly. Assign some placeholder values. inferenceEvent.BarracudaModelSource = "NN"; - inferenceEvent.BarracudaModelProducer = "tf2bc.py"; + inferenceEvent.BarracudaModelProducer = "tensorflow_to_barracuda.py"; } #if UNITY_2019_3_OR_NEWER && UNITY_EDITOR diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs index fb8126f84b..90d0398a3d 100644 --- a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs @@ -38,11 +38,16 @@ public void SetUp() public void TestModelEvent() { var sensors = new List { sensor_21_20_3.Sensor, sensor_20_22_3.Sensor }; + var behaviorName = "continuousModel"; var continuousEvent = InferenceAnalytics.GetEventForModel( - continuous2vis8vec2actionModel, "continuousModel", + continuous2vis8vec2actionModel, behaviorName, InferenceDevice.CPU, sensors, GetContinuous2vis8vec2actionActionSpec() ); + + // The behavior name should be hashed, not pass-through. + Assert.AreNotEqual(behaviorName, continuousEvent.BehaviorName); + Assert.AreEqual(2, continuousEvent.ActionSpec.NumContinuousActions); Assert.AreEqual(0, continuousEvent.ActionSpec.NumDiscreteActions); Assert.AreEqual(2, continuousEvent.ObservationSpecs.Count); From c69589eb0d2634c994f7fcea5adea512208c1f71 Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 14 Dec 2020 14:18:46 -0800 Subject: [PATCH 20/22] update analytics notice --- com.unity.ml-agents/CHANGELOG.md | 7 +++---- com.unity.ml-agents/Documentation~/com.unity.ml-agents.md | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index 1e9a939c64..396ef68cda 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -14,10 +14,9 @@ and this project adheres to ### Minor Changes #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#) -- If [Unity Editor Analytics](https://docs.unity3d.com/Manual/EditorAnalytics.html) are enabled, ML-Agents collects -anonymous data about models that are used for inference. This helps us focus our efforts on the most commonly used -features, and better understand the needs of our users. We compute a hash of a subset of model weights but cannot -recreate your model from the collected information. (#4677) +- In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics. +Please refer to "Information that is passively collected by Unity" in the +[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy). (#4677) #### ml-agents / ml-agents-envs / gym-unity (Python) - `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`. (#4691) diff --git a/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md b/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md index e5c368d284..02e6233547 100755 --- a/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md +++ b/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md @@ -111,6 +111,10 @@ If you are new to the Unity ML-Agents package, or have a question after reading the documentation, you can checkout our [GitHub Repository], which also includes a number of ways to [connect with us] including our [ML-Agents Forum]. +In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics. +Please refer to "Information that is passively collected by Unity" in the +[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy). + [unity ML-Agents Toolkit]: https://github.com/Unity-Technologies/ml-agents [unity inference engine]: https://docs.unity3d.com/Packages/com.unity.barracuda@latest/index.html [package manager documentation]: https://docs.unity3d.com/Manual/upm-ui-install.html From b8cec8060fa0d0357328261c803843c427a3641b Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 14 Dec 2020 14:32:52 -0800 Subject: [PATCH 21/22] readme too --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 84cb810276..c7c7e704f6 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,12 @@ minutes to For any other questions or feedback, connect directly with the ML-Agents team at ml-agents@unity3d.com. +## Privacy + +In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics. +Please refer to "Information that is passively collected by Unity" in the +[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy). + ## License [Apache License 2.0](LICENSE) From c958b8a1d264791da77addcfb3d88c6243b44f7f Mon Sep 17 00:00:00 2001 From: Chris Elion Date: Mon, 14 Dec 2020 18:16:03 -0800 Subject: [PATCH 22/22] fix unit test (path changed on merge) --- .../Tests/Editor/Analytics/InferenceAnalyticsTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs index 90d0398a3d..f3dbb3feef 100644 --- a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs +++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs @@ -13,8 +13,8 @@ namespace Unity.MLAgents.Tests.Analytics [TestFixture] public class InferenceAnalyticsTests { - const string k_continuous2vis8vec2actionPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.nn"; - NNModel continuous2vis8vec2actionModel; + const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.onnx"; + NNModel continuousONNXModel; Test3DSensorComponent sensor_21_20_3; Test3DSensorComponent sensor_20_22_3; @@ -26,7 +26,7 @@ ActionSpec GetContinuous2vis8vec2actionActionSpec() [SetUp] public void SetUp() { - continuous2vis8vec2actionModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuous2vis8vec2actionPath, typeof(NNModel)); + continuousONNXModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuousONNXPath, typeof(NNModel)); var go = new GameObject("SensorA"); sensor_21_20_3 = go.AddComponent(); sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3); @@ -41,7 +41,7 @@ public void TestModelEvent() var behaviorName = "continuousModel"; var continuousEvent = InferenceAnalytics.GetEventForModel( - continuous2vis8vec2actionModel, behaviorName, + continuousONNXModel, behaviorName, InferenceDevice.CPU, sensors, GetContinuous2vis8vec2actionActionSpec() );