diff --git a/README.md b/README.md
index afe11b009e..ae39639f25 100644
--- a/README.md
+++ b/README.md
@@ -161,6 +161,12 @@ minutes to
For any other questions or feedback, connect directly with the ML-Agents team at
ml-agents@unity3d.com.
+## Privacy
+
+In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics.
+Please refer to "Information that is passively collected by Unity" in the
+[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy).
+
## License
[Apache License 2.0](LICENSE)
diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md
index a5534576c9..653a10ad8e 100755
--- a/com.unity.ml-agents/CHANGELOG.md
+++ b/com.unity.ml-agents/CHANGELOG.md
@@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to
[Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+## [Unreleased]
+### Minor Changes
+#### com.unity.ml-agents (C#)
+In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics.
+Please refer to "Information that is passively collected by Unity" in the
+[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy).
+
+### Bug Fixes
+#### com.unity.ml-agents (C#)
+
## [1.0.6] - 2020-11-13
### Minor Changes
diff --git a/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md b/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md
index 179adbfc76..fb41eed439 100755
--- a/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md
+++ b/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md
@@ -113,6 +113,10 @@ If you are new to the Unity ML-Agents package, or have a question after reading
the documentation, you can checkout our [GitHUb Repository], which also includes
a number of ways to [connect with us] including our [ML-Agents Forum].
+In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics.
+Please refer to "Information that is passively collected by Unity" in the
+[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy).
+
[unity ML-Agents Toolkit]: https://github.com/Unity-Technologies/ml-agents/tree/release_2_verified_docs
[unity inference engine]: https://docs.unity3d.com/Packages/com.unity.barracuda@latest/index.html
[package manager documentation]: https://docs.unity3d.com/Manual/upm-ui-install.html
diff --git a/com.unity.ml-agents/Runtime/Analytics.meta b/com.unity.ml-agents/Runtime/Analytics.meta
new file mode 100644
index 0000000000..78a476e12c
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics.meta
@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 8b12ac54c5224758af88c67e2af4a01e
+folderAsset: yes
+DefaultImporter:
+ externalObjects: {}
+ userData:
+ assetBundleName:
+ assetBundleVariant:
diff --git a/com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs b/com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs
new file mode 100644
index 0000000000..fb480b7a11
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs
@@ -0,0 +1,40 @@
+using System;
+using UnityEngine;
+
+namespace Unity.MLAgents.Analytics
+{
+ internal static class AnalyticsUtils
+ {
+ ///
+ /// Hash a string to remove PII or secret info before sending to analytics
+ ///
+ ///
+ /// A string containing the Hash128 of the input string.
+ public static string Hash(string s)
+ {
+ var behaviorNameHash = Hash128.Compute(s);
+ return behaviorNameHash.ToString();
+ }
+
+ internal static bool s_SendEditorAnalytics = true;
+
+ ///
+ /// Helper class to temporarily disable sending analytics from unit tests.
+ ///
+ internal class DisableAnalyticsSending : IDisposable
+ {
+ private bool m_PreviousSendEditorAnalytics;
+
+ public DisableAnalyticsSending()
+ {
+ m_PreviousSendEditorAnalytics = s_SendEditorAnalytics;
+ s_SendEditorAnalytics = false;
+ }
+
+ public void Dispose()
+ {
+ s_SendEditorAnalytics = m_PreviousSendEditorAnalytics;
+ }
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs.meta b/com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs.meta
new file mode 100644
index 0000000000..b00fab1c90
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: af1ef3e70f1242938d7b39284b1a892b
+timeCreated: 1610575760
\ No newline at end of file
diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs
new file mode 100644
index 0000000000..c78a206b83
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs
@@ -0,0 +1,180 @@
+using System;
+using System.Collections.Generic;
+using Unity.MLAgents.Policies;
+using Unity.MLAgents.Sensors;
+
+namespace Unity.MLAgents.Analytics
+{
+ internal struct InferenceEvent
+ {
+ ///
+ /// Hash of the BehaviorName.
+ ///
+ public string BehaviorName;
+ public string BarracudaModelSource;
+ public string BarracudaModelVersion;
+ public string BarracudaModelProducer;
+ public string BarracudaPackageVersion;
+ ///
+ /// Whether inference is performed on CPU (0) or GPU (1).
+ ///
+ public int InferenceDevice;
+ public List ObservationSpecs;
+ public EventActionSpec ActionSpec;
+ public int MemorySize;
+ public long TotalWeightSizeBytes;
+ public string ModelHash;
+ }
+
+ ///
+ /// Simplified version of ActionSpec struct for use in analytics
+ ///
+ [Serializable]
+ internal struct EventActionSpec
+ {
+ public int NumContinuousActions;
+ public int NumDiscreteActions;
+ public int[] BranchSizes;
+
+ public static EventActionSpec FromBrainParameters(BrainParameters brainParameters)
+ {
+ if (brainParameters.VectorActionSpaceType == SpaceType.Continuous)
+ {
+ return new EventActionSpec
+ {
+ NumContinuousActions = brainParameters.NumActions,
+ NumDiscreteActions = 0,
+ BranchSizes = Array.Empty(),
+ };
+ }
+ else
+ {
+ return new EventActionSpec
+ {
+ NumContinuousActions = 0,
+ NumDiscreteActions = brainParameters.NumActions,
+ BranchSizes = brainParameters.VectorActionSize,
+ };
+ }
+ }
+ }
+
+ ///
+ /// Information about one dimension of an observation.
+ ///
+ [Serializable]
+ internal struct EventObservationDimensionInfo
+ {
+ public int Size;
+ public int Flags;
+ }
+
+ ///
+ /// Simplified summary of Agent observations for use in analytics
+ ///
+ [Serializable]
+ internal struct EventObservationSpec
+ {
+ public string SensorName;
+ public string CompressionType;
+ public int BuiltInSensorType;
+ public EventObservationDimensionInfo[] DimensionInfos;
+
+ public static EventObservationSpec FromSensor(ISensor sensor)
+ {
+ var shape = sensor.GetObservationShape();
+ var dimInfos = new EventObservationDimensionInfo[shape.Length];
+ for (var i = 0; i < shape.Length; i++)
+ {
+ dimInfos[i].Size = shape[i];
+ // TODO copy flags when we have them
+ }
+
+ var builtInSensorType = sensor.GetBuiltInSensorType();
+
+ return new EventObservationSpec
+ {
+ SensorName = sensor.GetName(),
+ CompressionType = sensor.GetCompressionType().ToString(),
+ BuiltInSensorType = (int)builtInSensorType,
+ DimensionInfos = dimInfos,
+ };
+ }
+ }
+
+ internal struct RemotePolicyInitializedEvent
+ {
+ public string TrainingSessionGuid;
+ ///
+ /// Hash of the BehaviorName.
+ ///
+ public string BehaviorName;
+ public List ObservationSpecs;
+ public EventActionSpec ActionSpec;
+
+ ///
+ /// This will be the same as TrainingEnvironmentInitializedEvent if available, but
+ /// TrainingEnvironmentInitializedEvent maybe not always be available with older trainers.
+ ///
+ public string MLAgentsEnvsVersion;
+ public string TrainerCommunicationVersion;
+ }
+
+ // These were added as part of a new interface in https://github.com/Unity-Technologies/ml-agents/pull/4871/
+ // Since we can't add a new interface in a patch release, we'll detect the type of the sensor and return
+ // the enum accordingly
+ internal enum BuiltInSensorType
+ {
+ Unknown = 0,
+ VectorSensor = 1,
+ // Note that StackingSensor actually returns the wrapped sensor's type
+ StackingSensor = 2,
+ RayPerceptionSensor = 3,
+ // ReflectionSensor = 4, // Added after 1.0.x
+ CameraSensor = 5,
+ RenderTextureSensor = 6,
+ // BufferSensor = 7, // Added after 1.0.x
+ // PhysicsBodySensor = 8, // In extensions package
+ // Match3Sensor = 9, // In extensions package
+ // GridSensor = 10 // In extensions package
+ }
+
+ ///
+ /// Helper methods to be shared by all classes that implement .
+ ///
+ internal static class BuiltInSensorExtensions
+ {
+ ///
+ /// Get the total number of elements in the ISensor's observation (i.e. the product of the
+ /// shape elements).
+ ///
+ ///
+ ///
+ public static BuiltInSensorType GetBuiltInSensorType(this ISensor sensor)
+ {
+ if (sensor as VectorSensor != null)
+ {
+ return BuiltInSensorType.VectorSensor;
+ }
+ if (sensor as RayPerceptionSensor != null)
+ {
+ return BuiltInSensorType.RayPerceptionSensor;
+ }
+ if (sensor as CameraSensor != null)
+ {
+ return BuiltInSensorType.CameraSensor;
+ }
+ if (sensor as RenderTextureSensor != null)
+ {
+ return BuiltInSensorType.RenderTextureSensor;
+ }
+ var stackingSensor = sensor as StackingSensor;
+ if (stackingSensor != null)
+ {
+ // Recurse on the wrapped sensor
+ return stackingSensor.GetWrappedSensor().GetBuiltInSensorType() ;
+ }
+ return BuiltInSensorType.Unknown;
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta b/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta
new file mode 100644
index 0000000000..347eebcd51
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 0a0d7cda6d74425a80775769a9283ba6
+timeCreated: 1604359798
\ No newline at end of file
diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs
new file mode 100644
index 0000000000..f81dc28e0b
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs
@@ -0,0 +1,266 @@
+using System;
+using System.Collections.Generic;
+using Unity.Barracuda;
+using Unity.MLAgents.Inference;
+using Unity.MLAgents.Policies;
+using Unity.MLAgents.Sensors;
+using UnityEngine;
+using UnityEngine.Analytics;
+
+#if UNITY_EDITOR
+using UnityEditor;
+using UnityEditor.Analytics;
+#endif
+
+
+namespace Unity.MLAgents.Analytics
+{
+ internal class InferenceAnalytics
+ {
+ const string k_VendorKey = "unity.ml-agents";
+ const string k_EventName = "ml_agents_inferencemodelset";
+ const int k_EventVersion = 1;
+
+ ///
+ /// Whether or not we've registered this particular event yet
+ ///
+ static bool s_EventRegistered = false;
+
+ ///
+ /// Hourly limit for this event name
+ ///
+ const int k_MaxEventsPerHour = 1000;
+
+ ///
+ /// Maximum number of items in this event.
+ ///
+ const int k_MaxNumberOfElements = 1000;
+
+
+ ///
+ /// Models that we've already sent events for.
+ ///
+ private static HashSet s_SentModels;
+
+ static bool EnableAnalytics()
+ {
+ if (s_EventRegistered)
+ {
+ return true;
+ }
+
+#if UNITY_EDITOR
+ AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey, k_EventVersion);
+#else
+ AnalyticsResult result = AnalyticsResult.UnsupportedPlatform;
+#endif
+ if (result == AnalyticsResult.Ok)
+ {
+ s_EventRegistered = true;
+ }
+
+ if (s_EventRegistered && s_SentModels == null)
+ {
+ s_SentModels = new HashSet();
+ }
+
+ return s_EventRegistered;
+ }
+
+ public static bool IsAnalyticsEnabled()
+ {
+#if UNITY_EDITOR
+ return EditorAnalytics.enabled;
+#else
+ return false;
+#endif
+ }
+
+ ///
+ /// Send an analytics event for the NNModel when it is set up for inference.
+ /// No events will be sent if analytics are disabled, and at most one event
+ /// will be sent per model instance.
+ ///
+ /// The NNModel being used for inference.
+ /// The BehaviorName of the Agent using the model
+ /// Whether inference is being performed on the CPU or GPU
+ /// List of ISensors for the Agent. Used to generate information about the observation space.
+ /// BrainParameters for the Agent. Used to generate information about the action space.
+ ///
+ public static void InferenceModelSet(
+ NNModel nnModel,
+ string behaviorName,
+ InferenceDevice inferenceDevice,
+ IList sensors,
+ BrainParameters brainParameters
+ )
+ {
+ // The event shouldn't be able to report if this is disabled but if we know we're not going to report
+ // Lets early out and not waste time gathering all the data
+ if (!IsAnalyticsEnabled())
+ return;
+
+ if (!EnableAnalytics())
+ return;
+
+ var added = s_SentModels.Add(nnModel);
+
+ if (!added)
+ {
+ // We previously added this model. Exit so we don't resend.
+ return;
+ }
+
+ var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, brainParameters);
+ // Note - to debug, use JsonUtility.ToJson on the event.
+ //Debug.Log(JsonUtility.ToJson(data, true));
+#if UNITY_EDITOR
+ if (AnalyticsUtils.s_SendEditorAnalytics)
+ {
+ EditorAnalytics.SendEventWithLimit(k_EventName, data, k_EventVersion);
+ }
+#else
+ return;
+#endif
+ }
+
+ ///
+ /// Generate an InferenceEvent for the model.
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ internal static InferenceEvent GetEventForModel(
+ NNModel nnModel,
+ string behaviorName,
+ InferenceDevice inferenceDevice,
+ IList sensors,
+ BrainParameters brainParameters
+ )
+ {
+ var barracudaModel = ModelLoader.Load(nnModel);
+ var inferenceEvent = new InferenceEvent();
+
+ // Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
+ inferenceEvent.BehaviorName = AnalyticsUtils.Hash(behaviorName);
+
+ inferenceEvent.BarracudaModelSource = barracudaModel.IrSource;
+ inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion;
+ inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName;
+ inferenceEvent.MemorySize = (int)barracudaModel.GetTensorByName(TensorNames.MemorySize)[0];
+ inferenceEvent.InferenceDevice = (int)inferenceDevice;
+
+ if (barracudaModel.ProducerName == "Script")
+ {
+ // .nn files don't have these fields set correctly. Assign some placeholder values.
+ inferenceEvent.BarracudaModelSource = "NN";
+ inferenceEvent.BarracudaModelProducer = "tensorflow_to_barracuda.py";
+ }
+
+#if UNITY_2019_3_OR_NEWER && UNITY_EDITOR
+ var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly);
+ inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version;
+#else
+ inferenceEvent.BarracudaPackageVersion = null;
+#endif
+
+ inferenceEvent.ActionSpec = EventActionSpec.FromBrainParameters(brainParameters);
+ inferenceEvent.ObservationSpecs = new List(sensors.Count);
+ foreach (var sensor in sensors)
+ {
+ inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor));
+ }
+
+ inferenceEvent.TotalWeightSizeBytes = GetModelWeightSize(barracudaModel);
+ inferenceEvent.ModelHash = GetModelHash(barracudaModel);
+ return inferenceEvent;
+ }
+
+ ///
+ /// Compute the total model weight size in bytes.
+ /// This corresponds to the "Total weight size" display in the Barracuda inspector,
+ /// and the calculations are the same.
+ ///
+ ///
+ ///
+ static long GetModelWeightSize(Model barracudaModel)
+ {
+ long totalWeightsSizeInBytes = 0;
+ for (var l = 0; l < barracudaModel.layers.Count; ++l)
+ {
+ for (var d = 0; d < barracudaModel.layers[l].datasets.Length; ++d)
+ {
+ totalWeightsSizeInBytes += barracudaModel.layers[l].datasets[d].length;
+ }
+ }
+ return totalWeightsSizeInBytes;
+ }
+
+ ///
+ /// Wrapper around Hash128 that supports Append(float[], int, int)
+ ///
+ struct MLAgentsHash128
+ {
+ private Hash128 m_Hash;
+
+ public void Append(float[] values, int count)
+ {
+ if (values == null)
+ {
+ return;
+ }
+
+ // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings and scalars)
+ // For these versions, we'll hash element by element.
+#if UNITY_2020_1_OR_NEWER
+ m_Hash.Append(values, 0, count);
+#else
+ for (var i = 0; i < count; i++)
+ {
+ var tempHash = new Hash128();
+ HashUtilities.ComputeHash128(ref values[i], ref tempHash);
+ HashUtilities.AppendHash(ref tempHash, ref m_Hash);
+ }
+#endif
+ }
+
+ public void Append(string value)
+ {
+ var tempHash = Hash128.Compute(value);
+ HashUtilities.AppendHash(ref tempHash, ref m_Hash);
+ }
+
+ public override string ToString()
+ {
+ return m_Hash.ToString();
+ }
+ }
+
+ ///
+ /// Compute a hash of the model's layer data and return it as a string.
+ /// A subset of the layer weights are used for performance.
+ /// This increases the chance of a collision, but this should still be extremely rare.
+ ///
+ ///
+ ///
+ static string GetModelHash(Model barracudaModel)
+ {
+ var hash = new MLAgentsHash128();
+
+ // Limit the max number of float bytes that we hash for performance.
+ const int kMaxFloats = 256;
+
+ foreach (var layer in barracudaModel.layers)
+ {
+ hash.Append(layer.name);
+ var numFloatsToHash = Mathf.Min(layer.weights.Length, kMaxFloats);
+ hash.Append(layer.weights, numFloatsToHash);
+ }
+
+ return hash.ToString();
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta
new file mode 100644
index 0000000000..e81b2ecbb6
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: ac4c40c2394d481ebf602caa600a32f3
+timeCreated: 1604359787
\ No newline at end of file
diff --git a/com.unity.ml-agents/Runtime/Analytics/TrainingAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/TrainingAnalytics.cs
new file mode 100644
index 0000000000..193003b0d1
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/TrainingAnalytics.cs
@@ -0,0 +1,168 @@
+using System;
+using System.Collections.Generic;
+using Unity.MLAgents.Policies;
+using Unity.MLAgents.Sensors;
+using UnityEngine;
+using UnityEngine.Analytics;
+
+#if UNITY_EDITOR
+using UnityEditor;
+using UnityEditor.Analytics;
+#endif
+
+namespace Unity.MLAgents.Analytics
+{
+ internal class TrainingAnalytics
+ {
+ const string k_VendorKey = "unity.ml-agents";
+ const string k_RemotePolicyInitializedEventName = "ml_agents_remote_policy_initialized";
+
+ ///
+ /// Whether or not we've registered this particular event yet
+ ///
+ static bool s_EventsRegistered = false;
+
+ ///
+ /// Hourly limit for this event name
+ ///
+ const int k_MaxEventsPerHour = 1000;
+
+ ///
+ /// Maximum number of items in this event.
+ ///
+ const int k_MaxNumberOfElements = 1000;
+
+ ///
+ /// Behaviors that we've already sent events for.
+ ///
+ private static HashSet s_SentRemotePolicyInitialized;
+
+ private static Guid s_TrainingSessionGuid;
+
+ // These are set when the RpcCommunicator connects
+ private static string s_TrainerPackageVersion = "";
+ private static string s_TrainerCommunicationVersion = "";
+
+ static bool EnableAnalytics()
+ {
+ if (s_EventsRegistered)
+ {
+ return true;
+ }
+
+
+#if UNITY_EDITOR
+ AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_RemotePolicyInitializedEventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey);
+#else
+ AnalyticsResult result = AnalyticsResult.UnsupportedPlatform;
+#endif
+ if (result != AnalyticsResult.Ok)
+ {
+ return false;
+ }
+
+ s_EventsRegistered = true;
+
+ if (s_SentRemotePolicyInitialized == null)
+ {
+ s_SentRemotePolicyInitialized = new HashSet();
+ s_TrainingSessionGuid = Guid.NewGuid();
+ }
+
+ return s_EventsRegistered;
+ }
+
+ ///
+ /// Cache information about the trainer when it becomes available in the RpcCommunicator.
+ ///
+ ///
+ ///
+ public static void SetTrainerInformation(string packageVersion, string communicationVersion)
+ {
+ s_TrainerPackageVersion = packageVersion;
+ s_TrainerCommunicationVersion = communicationVersion;
+ }
+
+ public static bool IsAnalyticsEnabled()
+ {
+#if UNITY_EDITOR
+ return EditorAnalytics.enabled;
+#else
+ return false;
+#endif
+ }
+
+ public static void RemotePolicyInitialized(
+ string fullyQualifiedBehaviorName,
+ IList sensors,
+ BrainParameters brainParameters
+ )
+ {
+ if (!IsAnalyticsEnabled())
+ return;
+
+ if (!EnableAnalytics())
+ return;
+
+ // Extract base behavior name (no team ID)
+ var behaviorName = ParseBehaviorName(fullyQualifiedBehaviorName);
+ var added = s_SentRemotePolicyInitialized.Add(behaviorName);
+
+ if (!added)
+ {
+ // We previously added this model. Exit so we don't resend.
+ return;
+ }
+
+ var data = GetEventForRemotePolicy(behaviorName, sensors, brainParameters);
+ // Note - to debug, use JsonUtility.ToJson on the event.
+ // Debug.Log(
+ // $"Would send event {k_RemotePolicyInitializedEventName} with body {JsonUtility.ToJson(data, true)}"
+ // );
+#if UNITY_EDITOR
+ if (AnalyticsUtils.s_SendEditorAnalytics)
+ {
+ EditorAnalytics.SendEventWithLimit(k_RemotePolicyInitializedEventName, data);
+ }
+#else
+ return;
+#endif
+ }
+
+ internal static string ParseBehaviorName(string fullyQualifiedBehaviorName)
+ {
+ var lastQuestionIndex = fullyQualifiedBehaviorName.LastIndexOf("?");
+ if (lastQuestionIndex < 0)
+ {
+ // Nothing to remove
+ return fullyQualifiedBehaviorName;
+ }
+
+ return fullyQualifiedBehaviorName.Substring(0, lastQuestionIndex);
+ }
+
+
+ static RemotePolicyInitializedEvent GetEventForRemotePolicy(
+ string behaviorName,
+ IList sensors,
+ BrainParameters brainParameters)
+ {
+ var remotePolicyEvent = new RemotePolicyInitializedEvent();
+
+ // Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
+ remotePolicyEvent.BehaviorName = AnalyticsUtils.Hash(behaviorName);
+
+ remotePolicyEvent.TrainingSessionGuid = s_TrainingSessionGuid.ToString();
+ remotePolicyEvent.ActionSpec = EventActionSpec.FromBrainParameters(brainParameters);
+ remotePolicyEvent.ObservationSpecs = new List(sensors.Count);
+ foreach (var sensor in sensors)
+ {
+ remotePolicyEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor));
+ }
+
+ remotePolicyEvent.MLAgentsEnvsVersion = s_TrainerPackageVersion;
+ remotePolicyEvent.TrainerCommunicationVersion = s_TrainerCommunicationVersion;
+ return remotePolicyEvent;
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Runtime/Analytics/TrainingAnalytics.cs.meta b/com.unity.ml-agents/Runtime/Analytics/TrainingAnalytics.cs.meta
new file mode 100644
index 0000000000..9109c265a2
--- /dev/null
+++ b/com.unity.ml-agents/Runtime/Analytics/TrainingAnalytics.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 5ad0bc6b45614bb7929d25dd59d5ac38
+timeCreated: 1608168600
\ No newline at end of file
diff --git a/com.unity.ml-agents/Runtime/Communicator/RpcCommunicator.cs b/com.unity.ml-agents/Runtime/Communicator/RpcCommunicator.cs
index 7ab00db795..0506a30246 100644
--- a/com.unity.ml-agents/Runtime/Communicator/RpcCommunicator.cs
+++ b/com.unity.ml-agents/Runtime/Communicator/RpcCommunicator.cs
@@ -8,6 +8,7 @@
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
+using Unity.MLAgents.Analytics;
using Unity.MLAgents.CommunicatorObjects;
using Unity.MLAgents.Sensors;
using Unity.MLAgents.Policies;
@@ -153,6 +154,8 @@ public UnityRLInitParameters Initialize(CommunicatorInitParameters initParameter
var pythonPackageVersion = initializationInput.RlInitializationInput.PackageVersion;
var unityCommunicationVersion = initParameters.unityCommunicationVersion;
+ TrainingAnalytics.SetTrainerInformation(pythonPackageVersion, pythonCommunicationVersion);
+
var communicationIsCompatible = CheckCommunicationVersionsAreCompatible(unityCommunicationVersion,
pythonCommunicationVersion,
pythonPackageVersion);
diff --git a/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs b/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs
index 584591803a..72fa4d7891 100644
--- a/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs
+++ b/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs
@@ -86,6 +86,16 @@ public ModelRunner(
brainParameters, seed, m_TensorAllocator, m_Memories, barracudaModel);
}
+ public InferenceDevice InferenceDevice
+ {
+ get { return m_InferenceDevice; }
+ }
+
+ public NNModel Model
+ {
+ get { return m_Model; }
+ }
+
static Dictionary PrepareBarracudaInputs(IEnumerable infInputs)
{
var inputs = new Dictionary();
diff --git a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs
index 6c6bcec4c1..74bd9f7f2b 100644
--- a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs
+++ b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs
@@ -37,19 +37,44 @@ internal class BarracudaPolicy : IPolicy
///
List m_SensorShapes;
+ private string m_BehaviorName;
+ private BrainParameters m_BrainParameters;
+
+ ///
+ /// Whether or not we've tried to send analytics for this model. We only ever try to send once per policy,
+ /// and do additional deduplication in the analytics code.
+ ///
+ private bool m_AnalyticsSent;
+
///
public BarracudaPolicy(
BrainParameters brainParameters,
NNModel model,
- InferenceDevice inferenceDevice)
+ InferenceDevice inferenceDevice,
+ string behaviorName
+ )
{
var modelRunner = Academy.Instance.GetOrCreateModelRunner(model, brainParameters, inferenceDevice);
m_ModelRunner = modelRunner;
+ m_BehaviorName = behaviorName;
+ m_BrainParameters = brainParameters;
}
///
public void RequestDecision(AgentInfo info, List sensors)
{
+
+ if (!m_AnalyticsSent)
+ {
+ m_AnalyticsSent = true;
+ Analytics.InferenceAnalytics.InferenceModelSet(
+ m_ModelRunner.Model,
+ m_BehaviorName,
+ m_ModelRunner.InferenceDevice,
+ sensors,
+ m_BrainParameters
+ );
+ }
m_AgentId = info.episodeId;
m_ModelRunner?.PutObservations(info, sensors);
}
diff --git a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs
index 013df3bad4..7e4be789c9 100644
--- a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs
+++ b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs
@@ -153,7 +153,7 @@ internal IPolicy GeneratePolicy(HeuristicPolicy.ActionGenerator heuristic)
"Either assign a model, or change to a different Behavior Type."
);
}
- return new BarracudaPolicy(m_BrainParameters, m_Model, m_InferenceDevice);
+ return new BarracudaPolicy(m_BrainParameters, m_Model, m_InferenceDevice, m_BehaviorName);
}
case BehaviorType.Default:
if (Academy.Instance.IsCommunicatorOn)
@@ -162,7 +162,7 @@ internal IPolicy GeneratePolicy(HeuristicPolicy.ActionGenerator heuristic)
}
if (m_Model != null)
{
- return new BarracudaPolicy(m_BrainParameters, m_Model, m_InferenceDevice);
+ return new BarracudaPolicy(m_BrainParameters, m_Model, m_InferenceDevice, m_BehaviorName);
}
else
{
diff --git a/com.unity.ml-agents/Runtime/Policies/RemotePolicy.cs b/com.unity.ml-agents/Runtime/Policies/RemotePolicy.cs
index 2f88d37f53..d8cde0b864 100644
--- a/com.unity.ml-agents/Runtime/Policies/RemotePolicy.cs
+++ b/com.unity.ml-agents/Runtime/Policies/RemotePolicy.cs
@@ -1,6 +1,7 @@
using UnityEngine;
using System.Collections.Generic;
using System;
+using Unity.MLAgents.Analytics;
using Unity.MLAgents.Sensors;
namespace Unity.MLAgents.Policies
@@ -14,6 +15,9 @@ internal class RemotePolicy : IPolicy
int m_AgentId;
string m_FullyQualifiedBehaviorName;
+ private bool m_AnalyticsSent = false;
+ private BrainParameters m_BrainParameters;
+
internal ICommunicator m_Communicator;
///
@@ -23,12 +27,23 @@ public RemotePolicy(
{
m_FullyQualifiedBehaviorName = fullyQualifiedBehaviorName;
m_Communicator = Academy.Instance.Communicator;
- m_Communicator.SubscribeBrain(m_FullyQualifiedBehaviorName, brainParameters);
+ m_Communicator?.SubscribeBrain(m_FullyQualifiedBehaviorName, brainParameters);
+ m_BrainParameters = brainParameters;
}
///
public void RequestDecision(AgentInfo info, List sensors)
{
+
+ if (!m_AnalyticsSent)
+ {
+ m_AnalyticsSent = true;
+ TrainingAnalytics.RemotePolicyInitialized(
+ m_FullyQualifiedBehaviorName,
+ sensors,
+ m_BrainParameters
+ );
+ }
m_AgentId = info.episodeId;
m_Communicator?.PutObservations(m_FullyQualifiedBehaviorName, info, sensors);
}
diff --git a/com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs b/com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs
index 962ffe1a9e..7ca466549c 100644
--- a/com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs
+++ b/com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs
@@ -139,6 +139,12 @@ public virtual SensorCompressionType GetCompressionType()
return SensorCompressionType.None;
}
+ internal ISensor GetWrappedSensor()
+ {
+ return m_WrappedSensor;
+ }
+
+
// TODO support stacked compressed observations (byte stream)
}
}
diff --git a/com.unity.ml-agents/Tests/Editor/Analytics.meta b/com.unity.ml-agents/Tests/Editor/Analytics.meta
new file mode 100644
index 0000000000..8f4a22cf22
--- /dev/null
+++ b/com.unity.ml-agents/Tests/Editor/Analytics.meta
@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: adbf291ff40848a296523d69a5be65a5
+folderAsset: yes
+DefaultImporter:
+ externalObjects: {}
+ userData:
+ assetBundleName:
+ assetBundleVariant:
diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs
new file mode 100644
index 0000000000..e2cabf60b7
--- /dev/null
+++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs
@@ -0,0 +1,91 @@
+using System.Collections.Generic;
+using NUnit.Framework;
+using Unity.MLAgents.Sensors;
+using UnityEngine;
+using Unity.Barracuda;
+using Unity.MLAgents.Analytics;
+using Unity.MLAgents.Policies;
+using UnityEditor;
+
+namespace Unity.MLAgents.Tests.Analytics
+{
+ [TestFixture]
+ public class InferenceAnalyticsTests
+ {
+ const string k_continuous2vis8vec2actionPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.nn";
+ NNModel continuous2vis8vec2actionModel;
+ Test3DSensorComponent sensor_21_20_3;
+ Test3DSensorComponent sensor_20_22_3;
+
+ BrainParameters GetContinuous2vis8vec2actionBrainParameters()
+ {
+ var validBrainParameters = new BrainParameters();
+ validBrainParameters.VectorObservationSize = 8;
+ validBrainParameters.VectorActionSize = new [] { 2 };
+ validBrainParameters.NumStackedVectorObservations = 1;
+ validBrainParameters.VectorActionSpaceType = SpaceType.Continuous;
+ return validBrainParameters;
+ }
+
+ [SetUp]
+ public void SetUp()
+ {
+ if (Academy.IsInitialized)
+ {
+ Academy.Instance.Dispose();
+ }
+
+ continuous2vis8vec2actionModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuous2vis8vec2actionPath, typeof(NNModel));
+ var go = new GameObject("SensorA");
+ sensor_21_20_3 = go.AddComponent();
+ sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3);
+ sensor_20_22_3 = go.AddComponent();
+ sensor_20_22_3.Sensor = new Test3DSensor("SensorB", 20, 22, 3);
+ }
+
+ [Test]
+ public void TestModelEvent()
+ {
+ var sensors = new List { sensor_21_20_3.Sensor, sensor_20_22_3.Sensor };
+ var behaviorName = "continuousModel";
+
+ var continuousEvent = InferenceAnalytics.GetEventForModel(
+ continuous2vis8vec2actionModel, behaviorName,
+ InferenceDevice.CPU, sensors, GetContinuous2vis8vec2actionBrainParameters()
+ );
+
+ // The behavior name should be hashed, not pass-through.
+ Assert.AreNotEqual(behaviorName, continuousEvent.BehaviorName);
+
+ Assert.AreEqual(2, continuousEvent.ActionSpec.NumContinuousActions);
+ Assert.AreEqual(0, continuousEvent.ActionSpec.NumDiscreteActions);
+ Assert.AreEqual(2, continuousEvent.ObservationSpecs.Count);
+ Assert.AreEqual(3, continuousEvent.ObservationSpecs[0].DimensionInfos.Length);
+ Assert.AreEqual(20, continuousEvent.ObservationSpecs[0].DimensionInfos[0].Size);
+ Assert.AreEqual("None", continuousEvent.ObservationSpecs[0].CompressionType);
+ Assert.AreEqual(0, continuousEvent.ObservationSpecs[0].BuiltInSensorType);
+ Assert.AreNotEqual(null, continuousEvent.ModelHash);
+
+ // Make sure nested fields get serialized
+ var jsonString = JsonUtility.ToJson(continuousEvent, true);
+ Assert.IsTrue(jsonString.Contains("ObservationSpecs"));
+ Assert.IsTrue(jsonString.Contains("ActionSpec"));
+ Assert.IsTrue(jsonString.Contains("NumDiscreteActions"));
+ Assert.IsTrue(jsonString.Contains("SensorName"));
+ Assert.IsTrue(jsonString.Contains("Flags"));
+ }
+
+ [Test]
+ public void TestBarracudaPolicy()
+ {
+ // Explicitly request decisions for a policy so we get code coverage on the event sending
+ using (new AnalyticsUtils.DisableAnalyticsSending())
+ {
+ var sensors = new List { sensor_21_20_3.Sensor, sensor_20_22_3.Sensor };
+ var policy = new BarracudaPolicy(GetContinuous2vis8vec2actionBrainParameters(), continuous2vis8vec2actionModel, InferenceDevice.CPU, "testBehavior");
+ policy.RequestDecision(new AgentInfo(), sensors);
+ }
+ Academy.Instance.Dispose();
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta
new file mode 100644
index 0000000000..20f024f03b
--- /dev/null
+++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 9f054f620b8b468bbd8ccf7d2cc14ccd
+timeCreated: 1607379491
\ No newline at end of file
diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/TrainingAnalyticsTest.cs b/com.unity.ml-agents/Tests/Editor/Analytics/TrainingAnalyticsTest.cs
new file mode 100644
index 0000000000..4ef1a8d55b
--- /dev/null
+++ b/com.unity.ml-agents/Tests/Editor/Analytics/TrainingAnalyticsTest.cs
@@ -0,0 +1,96 @@
+using System.Collections.Generic;
+using NUnit.Framework;
+using Unity.MLAgents.Sensors;
+using UnityEngine;
+using Unity.Barracuda;
+using Unity.MLAgents.Analytics;
+using Unity.MLAgents.Policies;
+using UnityEditor;
+
+namespace Unity.MLAgents.Tests.Analytics
+{
+ [TestFixture]
+ public class TrainingAnalyticsTests
+ {
+ [TestCase("foo?team=42", ExpectedResult = "foo")]
+ [TestCase("foo", ExpectedResult = "foo")]
+ [TestCase("foo?bar?team=1337", ExpectedResult = "foo?bar")]
+ public string TestParseBehaviorName(string fullyQualifiedBehaviorName)
+ {
+ return TrainingAnalytics.ParseBehaviorName(fullyQualifiedBehaviorName);
+ }
+
+ [Test]
+ public void TestRemotePolicy()
+ {
+ if (Academy.IsInitialized)
+ {
+ Academy.Instance.Dispose();
+ }
+
+ using (new AnalyticsUtils.DisableAnalyticsSending())
+ {
+ var brainParameters = new BrainParameters();
+ brainParameters.VectorObservationSize = 8;
+ brainParameters.VectorActionSize = new [] { 2 };
+ brainParameters.NumStackedVectorObservations = 1;
+ brainParameters.VectorActionSpaceType = SpaceType.Continuous;
+
+ var policy = new RemotePolicy(brainParameters, "TestBehavior?team=42");
+ policy.RequestDecision(new AgentInfo(), new List());
+ }
+
+ Academy.Instance.Dispose();
+ }
+
+ [Test]
+ public void TestBuiltInSensorType()
+ {
+ // Unknown
+ {
+ var sensor = new TestSensor("test");
+ Assert.AreEqual(sensor.GetBuiltInSensorType(), BuiltInSensorType.Unknown);
+
+ var stackingSensor = new StackingSensor(sensor, 2);
+ Assert.AreEqual(BuiltInSensorType.Unknown, stackingSensor.GetBuiltInSensorType());
+ }
+
+ // Vector
+ {
+ var sensor = new VectorSensor(6);
+ Assert.AreEqual(BuiltInSensorType.VectorSensor, sensor.GetBuiltInSensorType());
+
+ var stackingSensor = new StackingSensor(sensor, 2);
+ Assert.AreEqual(BuiltInSensorType.VectorSensor, stackingSensor.GetBuiltInSensorType());
+ }
+
+ var gameObject = new GameObject();
+
+ // Ray
+ {
+ var sensorComponent = gameObject.AddComponent();
+ sensorComponent.DetectableTags = new List();
+ var sensor = sensorComponent.CreateSensor();
+ Assert.AreEqual(BuiltInSensorType.RayPerceptionSensor, sensor.GetBuiltInSensorType());
+
+ var stackingSensor = new StackingSensor(sensor, 2);
+ Assert.AreEqual(BuiltInSensorType.RayPerceptionSensor, stackingSensor.GetBuiltInSensorType());
+ }
+
+ // Camera
+ {
+ var sensorComponent = gameObject.AddComponent();
+ var sensor = sensorComponent.CreateSensor();
+ Assert.AreEqual(BuiltInSensorType.CameraSensor, sensor.GetBuiltInSensorType());
+ }
+
+ // RenderTexture
+ {
+ var sensorComponent = gameObject.AddComponent();
+ var sensor = sensorComponent.CreateSensor();
+ Assert.AreEqual(BuiltInSensorType.RenderTextureSensor, sensor.GetBuiltInSensorType());
+ }
+
+ }
+ }
+}
diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/TrainingAnalyticsTest.cs.meta b/com.unity.ml-agents/Tests/Editor/Analytics/TrainingAnalyticsTest.cs.meta
new file mode 100644
index 0000000000..df394c157a
--- /dev/null
+++ b/com.unity.ml-agents/Tests/Editor/Analytics/TrainingAnalyticsTest.cs.meta
@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 70b8f1544bc34b4e8f1bc1068c64f01c
+timeCreated: 1610419546
\ No newline at end of file
diff --git a/com.unity.ml-agents/package.json b/com.unity.ml-agents/package.json
index 209cd930b2..2fd95e8178 100755
--- a/com.unity.ml-agents/package.json
+++ b/com.unity.ml-agents/package.json
@@ -9,6 +9,7 @@
"com.unity.modules.imageconversion": "1.0.0",
"com.unity.modules.jsonserialize": "1.0.0",
"com.unity.modules.physics": "1.0.0",
- "com.unity.modules.physics2d": "1.0.0"
+ "com.unity.modules.physics2d": "1.0.0",
+ "com.unity.modules.unityanalytics": "1.0.0"
}
}
diff --git a/utils/validate_meta_files.py b/utils/validate_meta_files.py
index 467d0d9b4b..6cba4227bd 100644
--- a/utils/validate_meta_files.py
+++ b/utils/validate_meta_files.py
@@ -2,34 +2,64 @@
def main():
- asset_path = "Project/Assets"
+ asset_paths = [
+ "Project/Assets",
+ "DevProject/Assets",
+ "com.unity.ml-agents",
+ "com.unity.ml-agents.extensions",
+ ]
meta_suffix = ".meta"
python_suffix = ".py"
+ allow_list = frozenset(
+ [
+ "com.unity.ml-agents/.editorconfig",
+ "com.unity.ml-agents/.gitignore",
+ "com.unity.ml-agents/.npmignore",
+ "com.unity.ml-agents/Tests/.tests.json",
+ "com.unity.ml-agents.extensions/.gitignore",
+ "com.unity.ml-agents.extensions/.npmignore",
+ "com.unity.ml-agents.extensions/Tests/.tests.json",
+ ]
+ )
+ ignored_dirs = {"Documentation~"}
num_matched = 0
unmatched = set()
- for root, dirs, files in os.walk(asset_path):
- dirs = set(dirs)
- files = set(files)
-
- combined = dirs | files
- for f in combined:
- if f.endswith(python_suffix):
- # Probably this script; skip it
- continue
-
- # We expect each non-.meta file to have a .meta file, and each .meta file to have a non-.meta file
- if f.endswith(meta_suffix):
- expected = f.replace(meta_suffix, "")
- else:
- expected = f + meta_suffix
-
- if expected not in combined:
- unmatched.add(os.path.join(root, f))
- else:
- num_matched += 1
+ for asset_path in asset_paths:
+ for root, dirs, files in os.walk(asset_path):
+ # Modifying the dirs list with topdown=True (the default) will prevent us from recursing those directories
+ for ignored in ignored_dirs:
+ try:
+ dirs.remove(ignored)
+ except ValueError:
+ pass
+
+ dirs = set(dirs)
+ files = set(files)
+
+ combined = dirs | files
+ for f in combined:
+
+ if f.endswith(python_suffix):
+ # Probably this script; skip it
+ continue
+
+ full_path = os.path.join(root, f)
+ if full_path in allow_list:
+ continue
+
+ # We expect each non-.meta file to have a .meta file, and each .meta file to have a non-.meta file
+ if f.endswith(meta_suffix):
+ expected = f.replace(meta_suffix, "")
+ else:
+ expected = f + meta_suffix
+
+ if expected not in combined:
+ unmatched.add(full_path)
+ else:
+ num_matched += 1
if unmatched:
raise Exception(