diff --git a/README.md b/README.md
index 84cb810276..c7c7e704f6 100644
--- a/README.md
+++ b/README.md
@@ -161,6 +161,12 @@ minutes to
For any other questions or feedback, connect directly with the ML-Agents team at
ml-agents@unity3d.com.
+## Privacy
+
+In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics.
+Please refer to "Information that is passively collected by Unity" in the
+[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy).
+
## License
[Apache License 2.0](LICENSE)
diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md
index 6b8ac35680..d5bf95bfd0 100755
--- a/com.unity.ml-agents/CHANGELOG.md
+++ b/com.unity.ml-agents/CHANGELOG.md
@@ -16,8 +16,12 @@ and this project adheres to
#### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
- Agents with both continuous and discrete actions are now supported. You can specify
both continuous and discrete action sizes in Behavior Parameters. (#4702, #4718)
+- In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics.
+Please refer to "Information that is passively collected by Unity" in the
+[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy). (#4677)
+
#### ml-agents / ml-agents-envs / gym-unity (Python)
-- `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`.
+- `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`. (#4691)
### Bug Fixes
#### com.unity.ml-agents (C#)
diff --git a/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md b/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md
index e5c368d284..02e6233547 100755
--- a/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md
+++ b/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md
@@ -111,6 +111,10 @@ If you are new to the Unity ML-Agents package, or have a question after reading
the documentation, you can checkout our [GitHub Repository], which also includes
a number of ways to [connect with us] including our [ML-Agents Forum].
+In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics.
+Please refer to "Information that is passively collected by Unity" in the
+[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy).
+
[unity ML-Agents Toolkit]: https://github.com/Unity-Technologies/ml-agents
[unity inference engine]: https://docs.unity3d.com/Packages/com.unity.barracuda@latest/index.html
[package manager documentation]: https://docs.unity3d.com/Manual/upm-ui-install.html
diff --git a/com.unity.ml-agents/Runtime/Analytics.meta b/com.unity.ml-agents/Runtime/Analytics.meta
new file mode 100644
index 0000000000..260b85a9b3
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 8b12ac54c5224758af88c67e2af4a01e
+timeCreated: 1604359666
\ No newline at end of file
diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs
new file mode 100644
index 0000000000..9dd5292318
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs
@@ -0,0 +1,89 @@
+using System;
+using System.Collections.Generic;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+namespace Unity.MLAgents.Analytics
+{
+ internal struct InferenceEvent
+ {
+ ///
+ /// Hash of the BehaviorName.
+ ///
+ public string BehaviorName;
+ public string BarracudaModelSource;
+ public string BarracudaModelVersion;
+ public string BarracudaModelProducer;
+ public string BarracudaPackageVersion;
+ ///
+ /// Whether inference is performed on CPU (0) or GPU (1).
+ ///
+ public int InferenceDevice;
+ public List ObservationSpecs;
+ public EventActionSpec ActionSpec;
+ public int MemorySize;
+ public long TotalWeightSizeBytes;
+ public string ModelHash;
+ }
+
+ ///
+ /// Simplified version of ActionSpec struct for use in analytics
+ ///
+ [Serializable]
+ internal struct EventActionSpec
+ {
+ public int NumContinuousActions;
+ public int NumDiscreteActions;
+ public int[] BranchSizes;
+
+ public static EventActionSpec FromActionSpec(ActionSpec actionSpec)
+ {
+ var branchSizes = actionSpec.BranchSizes ?? Array.Empty();
+ return new EventActionSpec
+ {
+ NumContinuousActions = actionSpec.NumContinuousActions,
+ NumDiscreteActions = actionSpec.NumDiscreteActions,
+ BranchSizes = branchSizes,
+ };
+ }
+ }
+
+ ///
+ /// Information about one dimension of an observation.
+ ///
+ [Serializable]
+ internal struct EventObservationDimensionInfo
+ {
+ public int Size;
+ public int Flags;
+ }
+
+ ///
+ /// Simplified summary of Agent observations for use in analytics
+ ///
+ [Serializable]
+ internal struct EventObservationSpec
+ {
+ public string SensorName;
+ public string CompressionType;
+ public EventObservationDimensionInfo[] DimensionInfos;
+
+ public static EventObservationSpec FromSensor(ISensor sensor)
+ {
+ var shape = sensor.GetObservationShape();
+ var dimInfos = new EventObservationDimensionInfo[shape.Length];
+ for (var i = 0; i < shape.Length; i++)
+ {
+ dimInfos[i].Size = shape[i];
+ // TODO copy flags when we have them
+ }
+
+ return new EventObservationSpec
+ {
+ SensorName = sensor.GetName(),
+ CompressionType = sensor.GetCompressionType().ToString(),
+ DimensionInfos = dimInfos,
+ };
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta b/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta
new file mode 100644
index 0000000000..347eebcd51
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 0a0d7cda6d74425a80775769a9283ba6
+timeCreated: 1604359798
\ No newline at end of file
diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs
new file mode 100644
index 0000000000..0b509eb491
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs
@@ -0,0 +1,263 @@
+using System;
+using System.Collections.Generic;
+using Unity.Barracuda;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Inference;
+using Unity.MLAgents.Policies;
+using Unity.MLAgents.Sensors;
+using UnityEngine;
+using UnityEngine.Analytics;
+
+#if UNITY_EDITOR
+using UnityEditor;
+using UnityEditor.Analytics;
+#endif
+
+
+namespace Unity.MLAgents.Analytics
+{
+ internal class InferenceAnalytics
+ {
+ const string k_VendorKey = "unity.ml-agents";
+ const string k_EventName = "ml_agents_inferencemodelset";
+
+ ///
+ /// Whether or not we've registered this particular event yet
+ ///
+ static bool s_EventRegistered = false;
+
+ ///
+ /// Hourly limit for this event name
+ ///
+ const int k_MaxEventsPerHour = 1000;
+
+ ///
+ /// Maximum number of items in this event.
+ ///
+ const int k_MaxNumberOfElements = 1000;
+
+ ///
+ /// Models that we've already sent events for.
+ ///
+ private static HashSet s_SentModels;
+
+ static bool EnableAnalytics()
+ {
+ if (s_EventRegistered)
+ {
+ return true;
+ }
+
+#if UNITY_EDITOR
+ AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey);
+#else
+ AnalyticsResult result = AnalyticsResult.UnsupportedPlatform;
+#endif
+ if (result == AnalyticsResult.Ok)
+ {
+ s_EventRegistered = true;
+ }
+
+ if (s_EventRegistered && s_SentModels == null)
+ {
+ s_SentModels = new HashSet();
+ }
+
+ return s_EventRegistered;
+ }
+
+ public static bool IsAnalyticsEnabled()
+ {
+#if UNITY_EDITOR
+ return EditorAnalytics.enabled;
+#else
+ return false;
+#endif
+ }
+
+ ///
+ /// Send an analytics event for the NNModel when it is set up for inference.
+ /// No events will be sent if analytics are disabled, and at most one event
+ /// will be sent per model instance.
+ ///
+ /// The NNModel being used for inference.
+ /// The BehaviorName of the Agent using the model
+ /// Whether inference is being performed on the CPU or GPU
+ /// List of ISensors for the Agent. Used to generate information about the observation space.
+ /// ActionSpec for the Agent. Used to generate information about the action space.
+ ///
+ public static void InferenceModelSet(
+ NNModel nnModel,
+ string behaviorName,
+ InferenceDevice inferenceDevice,
+ IList sensors,
+ ActionSpec actionSpec
+ )
+ {
+ // The event shouldn't be able to report if this is disabled but if we know we're not going to report
+ // Lets early out and not waste time gathering all the data
+ if (!IsAnalyticsEnabled())
+ return;
+
+ if (!EnableAnalytics())
+ return;
+
+ var added = s_SentModels.Add(nnModel);
+
+ if (!added)
+ {
+ // We previously added this model. Exit so we don't resend.
+ return;
+ }
+
+ var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec);
+ // Note - to debug, use JsonUtility.ToJson on the event.
+ // Debug.Log(JsonUtility.ToJson(data, true));
+#if UNITY_EDITOR
+ EditorAnalytics.SendEventWithLimit(k_EventName, data);
+#else
+ return;
+#endif
+ }
+
+ ///
+ /// Generate an InferenceEvent for the model.
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ internal static InferenceEvent GetEventForModel(
+ NNModel nnModel,
+ string behaviorName,
+ InferenceDevice inferenceDevice,
+ IList sensors,
+ ActionSpec actionSpec
+ )
+ {
+ var barracudaModel = ModelLoader.Load(nnModel);
+ var inferenceEvent = new InferenceEvent();
+
+ // Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
+ var behaviorNameHash = Hash128.Compute(behaviorName);
+ inferenceEvent.BehaviorName = behaviorNameHash.ToString();
+
+ inferenceEvent.BarracudaModelSource = barracudaModel.IrSource;
+ inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion;
+ inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName;
+ inferenceEvent.MemorySize = (int)barracudaModel.GetTensorByName(TensorNames.MemorySize)[0];
+ inferenceEvent.InferenceDevice = (int)inferenceDevice;
+
+ if (barracudaModel.ProducerName == "Script")
+ {
+ // .nn files don't have these fields set correctly. Assign some placeholder values.
+ inferenceEvent.BarracudaModelSource = "NN";
+ inferenceEvent.BarracudaModelProducer = "tensorflow_to_barracuda.py";
+ }
+
+#if UNITY_2019_3_OR_NEWER && UNITY_EDITOR
+ var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly);
+ inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version;
+#else
+ inferenceEvent.BarracudaPackageVersion = null;
+#endif
+
+ inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec);
+ inferenceEvent.ObservationSpecs = new List(sensors.Count);
+ foreach (var sensor in sensors)
+ {
+ inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor));
+ }
+
+ inferenceEvent.TotalWeightSizeBytes = GetModelWeightSize(barracudaModel);
+ inferenceEvent.ModelHash = GetModelHash(barracudaModel);
+ return inferenceEvent;
+ }
+
+ ///
+ /// Compute the total model weight size in bytes.
+ /// This corresponds to the "Total weight size" display in the Barracuda inspector,
+ /// and the calculations are the same.
+ ///
+ ///
+ ///
+ static long GetModelWeightSize(Model barracudaModel)
+ {
+ long totalWeightsSizeInBytes = 0;
+ for (var l = 0; l < barracudaModel.layers.Count; ++l)
+ {
+ for (var d = 0; d < barracudaModel.layers[l].datasets.Length; ++d)
+ {
+ totalWeightsSizeInBytes += barracudaModel.layers[l].datasets[d].length;
+ }
+ }
+ return totalWeightsSizeInBytes;
+ }
+
+ ///
+ /// Wrapper around Hash128 that supports Append(float[], int, int)
+ ///
+ struct MLAgentsHash128
+ {
+ private Hash128 m_Hash;
+
+ public void Append(float[] values, int count)
+ {
+ if (values == null)
+ {
+ return;
+ }
+
+ // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings and scalars)
+ // For these versions, we'll hash element by element.
+#if UNITY_2020_1_OR_NEWER
+ m_Hash.Append(values, 0, count);
+#else
+ for (var i = 0; i < count; i++)
+ {
+ var tempHash = new Hash128();
+ HashUtilities.ComputeHash128(ref values[i], ref tempHash);
+ HashUtilities.AppendHash(ref tempHash, ref m_Hash);
+ }
+#endif
+ }
+
+ public void Append(string value)
+ {
+ var tempHash = Hash128.Compute(value);
+ HashUtilities.AppendHash(ref tempHash, ref m_Hash);
+ }
+
+ public override string ToString()
+ {
+ return m_Hash.ToString();
+ }
+ }
+
+ ///
+ /// Compute a hash of the model's layer data and return it as a string.
+ /// A subset of the layer weights are used for performance.
+ /// This increases the chance of a collision, but this should still be extremely rare.
+ ///
+ ///
+ ///
+ static string GetModelHash(Model barracudaModel)
+ {
+ var hash = new MLAgentsHash128();
+
+ // Limit the max number of float bytes that we hash for performance.
+ const int kMaxFloats = 256;
+
+ foreach (var layer in barracudaModel.layers)
+ {
+ hash.Append(layer.name);
+ var numFloatsToHash = Mathf.Min(layer.weights.Length, kMaxFloats);
+ hash.Append(layer.weights, numFloatsToHash);
+ }
+
+ return hash.ToString();
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta
new file mode 100644
index 0000000000..e81b2ecbb6
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: ac4c40c2394d481ebf602caa600a32f3
+timeCreated: 1604359787
\ No newline at end of file
diff --git a/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs b/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs
index 3fa8c220a4..851341ff9c 100644
--- a/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs
+++ b/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs
@@ -86,6 +86,16 @@ public ModelRunner(
actionSpec, seed, m_TensorAllocator, m_Memories, barracudaModel);
}
+ public InferenceDevice InferenceDevice
+ {
+ get { return m_InferenceDevice; }
+ }
+
+ public NNModel Model
+ {
+ get { return m_Model; }
+ }
+
static Dictionary PrepareBarracudaInputs(IEnumerable infInputs)
{
var inputs = new Dictionary();
diff --git a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs
index ff4c995540..4d6cada589 100644
--- a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs
+++ b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs
@@ -41,20 +41,42 @@ internal class BarracudaPolicy : IPolicy
List m_SensorShapes;
ActionSpec m_ActionSpec;
+ private string m_BehaviorName;
+
+ ///
+ /// Whether or not we've tried to send analytics for this model. We only ever try to send once per policy,
+ /// and do additional deduplication in the analytics code.
+ ///
+ private bool m_AnalyticsSent;
+
///
public BarracudaPolicy(
ActionSpec actionSpec,
NNModel model,
- InferenceDevice inferenceDevice)
+ InferenceDevice inferenceDevice,
+ string behaviorName
+ )
{
var modelRunner = Academy.Instance.GetOrCreateModelRunner(model, actionSpec, inferenceDevice);
m_ModelRunner = modelRunner;
+ m_BehaviorName = behaviorName;
m_ActionSpec = actionSpec;
}
///
public void RequestDecision(AgentInfo info, List sensors)
{
+ if (!m_AnalyticsSent)
+ {
+ m_AnalyticsSent = true;
+ Analytics.InferenceAnalytics.InferenceModelSet(
+ m_ModelRunner.Model,
+ m_BehaviorName,
+ m_ModelRunner.InferenceDevice,
+ sensors,
+ m_ActionSpec
+ );
+ }
m_AgentId = info.episodeId;
m_ModelRunner?.PutObservations(info, sensors);
}
diff --git a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs
index add0dc359e..100220e320 100644
--- a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs
+++ b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs
@@ -212,7 +212,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen
"Either assign a model, or change to a different Behavior Type."
);
}
- return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice);
+ return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName);
}
case BehaviorType.Default:
if (Academy.Instance.IsCommunicatorOn)
@@ -221,7 +221,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen
}
if (m_Model != null)
{
- return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice);
+ return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName);
}
else
{
@@ -241,5 +241,6 @@ internal void UpdateAgentPolicy()
}
agent.ReloadPolicy();
}
+
}
}
diff --git a/com.unity.ml-agents/Tests/Editor/Analytics.meta b/com.unity.ml-agents/Tests/Editor/Analytics.meta
new file mode 100644
index 0000000000..473f2be08f
--- /dev/null
+++ b/com.unity.ml-agents/Tests/Editor/Analytics.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: adbf291ff40848a296523d69a5be65a5
+timeCreated: 1607379470
\ No newline at end of file
diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs
new file mode 100644
index 0000000000..f3dbb3feef
--- /dev/null
+++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs
@@ -0,0 +1,68 @@
+using System.Collections.Generic;
+using NUnit.Framework;
+using Unity.MLAgents.Sensors;
+using UnityEngine;
+using Unity.Barracuda;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Analytics;
+using Unity.MLAgents.Policies;
+using UnityEditor;
+
+namespace Unity.MLAgents.Tests.Analytics
+{
+ [TestFixture]
+ public class InferenceAnalyticsTests
+ {
+ const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.onnx";
+ NNModel continuousONNXModel;
+ Test3DSensorComponent sensor_21_20_3;
+ Test3DSensorComponent sensor_20_22_3;
+
+ ActionSpec GetContinuous2vis8vec2actionActionSpec()
+ {
+ return ActionSpec.MakeContinuous(2);
+ }
+
+ [SetUp]
+ public void SetUp()
+ {
+ continuousONNXModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuousONNXPath, typeof(NNModel));
+ var go = new GameObject("SensorA");
+ sensor_21_20_3 = go.AddComponent();
+ sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3);
+ sensor_20_22_3 = go.AddComponent();
+ sensor_20_22_3.Sensor = new Test3DSensor("SensorB", 20, 22, 3);
+ }
+
+ [Test]
+ public void TestModelEvent()
+ {
+ var sensors = new List { sensor_21_20_3.Sensor, sensor_20_22_3.Sensor };
+ var behaviorName = "continuousModel";
+
+ var continuousEvent = InferenceAnalytics.GetEventForModel(
+ continuousONNXModel, behaviorName,
+ InferenceDevice.CPU, sensors, GetContinuous2vis8vec2actionActionSpec()
+ );
+
+ // The behavior name should be hashed, not pass-through.
+ Assert.AreNotEqual(behaviorName, continuousEvent.BehaviorName);
+
+ Assert.AreEqual(2, continuousEvent.ActionSpec.NumContinuousActions);
+ Assert.AreEqual(0, continuousEvent.ActionSpec.NumDiscreteActions);
+ Assert.AreEqual(2, continuousEvent.ObservationSpecs.Count);
+ Assert.AreEqual(3, continuousEvent.ObservationSpecs[0].DimensionInfos.Length);
+ Assert.AreEqual(20, continuousEvent.ObservationSpecs[0].DimensionInfos[0].Size);
+ Assert.AreEqual("None", continuousEvent.ObservationSpecs[0].CompressionType);
+ Assert.AreNotEqual(null, continuousEvent.ModelHash);
+
+ // Make sure nested fields get serialized
+ var jsonString = JsonUtility.ToJson(continuousEvent, true);
+ Assert.IsTrue(jsonString.Contains("ObservationSpecs"));
+ Assert.IsTrue(jsonString.Contains("ActionSpec"));
+ Assert.IsTrue(jsonString.Contains("NumDiscreteActions"));
+ Assert.IsTrue(jsonString.Contains("SensorName"));
+ Assert.IsTrue(jsonString.Contains("Flags"));
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta
new file mode 100644
index 0000000000..20f024f03b
--- /dev/null
+++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 9f054f620b8b468bbd8ccf7d2cc14ccd
+timeCreated: 1607379491
\ No newline at end of file