diff --git a/README.md b/README.md index 84cb810276..c7c7e704f6 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,12 @@ minutes to For any other questions or feedback, connect directly with the ML-Agents team at ml-agents@unity3d.com. +## Privacy + +In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics. +Please refer to "Information that is passively collected by Unity" in the +[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy). + ## License [Apache License 2.0](LICENSE) diff --git a/com.unity.ml-agents/CHANGELOG.md b/com.unity.ml-agents/CHANGELOG.md index 6b8ac35680..d5bf95bfd0 100755 --- a/com.unity.ml-agents/CHANGELOG.md +++ b/com.unity.ml-agents/CHANGELOG.md @@ -16,8 +16,12 @@ and this project adheres to #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#) - Agents with both continuous and discrete actions are now supported. You can specify both continuous and discrete action sizes in Behavior Parameters. (#4702, #4718) +- In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics. +Please refer to "Information that is passively collected by Unity" in the +[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy). (#4677) + #### ml-agents / ml-agents-envs / gym-unity (Python) -- `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`. +- `ActionSpec.validate_action()` now enforces that `UnityEnvironment.set_action_for_agent()` receives a 1D `np.array`. (#4691) ### Bug Fixes #### com.unity.ml-agents (C#) diff --git a/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md b/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md index e5c368d284..02e6233547 100755 --- a/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md +++ b/com.unity.ml-agents/Documentation~/com.unity.ml-agents.md @@ -111,6 +111,10 @@ If you are new to the Unity ML-Agents package, or have a question after reading the documentation, you can checkout our [GitHub Repository], which also includes a number of ways to [connect with us] including our [ML-Agents Forum]. +In order to improve the developer experience for Unity ML-Agents Toolkit, we have added in-editor analytics. +Please refer to "Information that is passively collected by Unity" in the +[Unity Privacy Policy](https://unity3d.com/legal/privacy-policy). + [unity ML-Agents Toolkit]: https://github.com/Unity-Technologies/ml-agents [unity inference engine]: https://docs.unity3d.com/Packages/com.unity.barracuda@latest/index.html [package manager documentation]: https://docs.unity3d.com/Manual/upm-ui-install.html diff --git a/com.unity.ml-agents/Runtime/Analytics.meta b/com.unity.ml-agents/Runtime/Analytics.meta new file mode 100644 index 0000000000..260b85a9b3 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 8b12ac54c5224758af88c67e2af4a01e +timeCreated: 1604359666 \ No newline at end of file diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs b/com.unity.ml-agents/Runtime/Analytics/Events.cs new file mode 100644 index 0000000000..9dd5292318 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using Unity.MLAgents.Actuators; +using Unity.MLAgents.Sensors; + +namespace Unity.MLAgents.Analytics +{ + internal struct InferenceEvent + { + /// + /// Hash of the BehaviorName. + /// + public string BehaviorName; + public string BarracudaModelSource; + public string BarracudaModelVersion; + public string BarracudaModelProducer; + public string BarracudaPackageVersion; + /// + /// Whether inference is performed on CPU (0) or GPU (1). + /// + public int InferenceDevice; + public List ObservationSpecs; + public EventActionSpec ActionSpec; + public int MemorySize; + public long TotalWeightSizeBytes; + public string ModelHash; + } + + /// + /// Simplified version of ActionSpec struct for use in analytics + /// + [Serializable] + internal struct EventActionSpec + { + public int NumContinuousActions; + public int NumDiscreteActions; + public int[] BranchSizes; + + public static EventActionSpec FromActionSpec(ActionSpec actionSpec) + { + var branchSizes = actionSpec.BranchSizes ?? Array.Empty(); + return new EventActionSpec + { + NumContinuousActions = actionSpec.NumContinuousActions, + NumDiscreteActions = actionSpec.NumDiscreteActions, + BranchSizes = branchSizes, + }; + } + } + + /// + /// Information about one dimension of an observation. + /// + [Serializable] + internal struct EventObservationDimensionInfo + { + public int Size; + public int Flags; + } + + /// + /// Simplified summary of Agent observations for use in analytics + /// + [Serializable] + internal struct EventObservationSpec + { + public string SensorName; + public string CompressionType; + public EventObservationDimensionInfo[] DimensionInfos; + + public static EventObservationSpec FromSensor(ISensor sensor) + { + var shape = sensor.GetObservationShape(); + var dimInfos = new EventObservationDimensionInfo[shape.Length]; + for (var i = 0; i < shape.Length; i++) + { + dimInfos[i].Size = shape[i]; + // TODO copy flags when we have them + } + + return new EventObservationSpec + { + SensorName = sensor.GetName(), + CompressionType = sensor.GetCompressionType().ToString(), + DimensionInfos = dimInfos, + }; + } + } +} diff --git a/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta b/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta new file mode 100644 index 0000000000..347eebcd51 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics/Events.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 0a0d7cda6d74425a80775769a9283ba6 +timeCreated: 1604359798 \ No newline at end of file diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs new file mode 100644 index 0000000000..0b509eb491 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs @@ -0,0 +1,263 @@ +using System; +using System.Collections.Generic; +using Unity.Barracuda; +using Unity.MLAgents.Actuators; +using Unity.MLAgents.Inference; +using Unity.MLAgents.Policies; +using Unity.MLAgents.Sensors; +using UnityEngine; +using UnityEngine.Analytics; + +#if UNITY_EDITOR +using UnityEditor; +using UnityEditor.Analytics; +#endif + + +namespace Unity.MLAgents.Analytics +{ + internal class InferenceAnalytics + { + const string k_VendorKey = "unity.ml-agents"; + const string k_EventName = "ml_agents_inferencemodelset"; + + /// + /// Whether or not we've registered this particular event yet + /// + static bool s_EventRegistered = false; + + /// + /// Hourly limit for this event name + /// + const int k_MaxEventsPerHour = 1000; + + /// + /// Maximum number of items in this event. + /// + const int k_MaxNumberOfElements = 1000; + + /// + /// Models that we've already sent events for. + /// + private static HashSet s_SentModels; + + static bool EnableAnalytics() + { + if (s_EventRegistered) + { + return true; + } + +#if UNITY_EDITOR + AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey); +#else + AnalyticsResult result = AnalyticsResult.UnsupportedPlatform; +#endif + if (result == AnalyticsResult.Ok) + { + s_EventRegistered = true; + } + + if (s_EventRegistered && s_SentModels == null) + { + s_SentModels = new HashSet(); + } + + return s_EventRegistered; + } + + public static bool IsAnalyticsEnabled() + { +#if UNITY_EDITOR + return EditorAnalytics.enabled; +#else + return false; +#endif + } + + /// + /// Send an analytics event for the NNModel when it is set up for inference. + /// No events will be sent if analytics are disabled, and at most one event + /// will be sent per model instance. + /// + /// The NNModel being used for inference. + /// The BehaviorName of the Agent using the model + /// Whether inference is being performed on the CPU or GPU + /// List of ISensors for the Agent. Used to generate information about the observation space. + /// ActionSpec for the Agent. Used to generate information about the action space. + /// + public static void InferenceModelSet( + NNModel nnModel, + string behaviorName, + InferenceDevice inferenceDevice, + IList sensors, + ActionSpec actionSpec + ) + { + // The event shouldn't be able to report if this is disabled but if we know we're not going to report + // Lets early out and not waste time gathering all the data + if (!IsAnalyticsEnabled()) + return; + + if (!EnableAnalytics()) + return; + + var added = s_SentModels.Add(nnModel); + + if (!added) + { + // We previously added this model. Exit so we don't resend. + return; + } + + var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec); + // Note - to debug, use JsonUtility.ToJson on the event. + // Debug.Log(JsonUtility.ToJson(data, true)); +#if UNITY_EDITOR + EditorAnalytics.SendEventWithLimit(k_EventName, data); +#else + return; +#endif + } + + /// + /// Generate an InferenceEvent for the model. + /// + /// + /// + /// + /// + /// + /// + internal static InferenceEvent GetEventForModel( + NNModel nnModel, + string behaviorName, + InferenceDevice inferenceDevice, + IList sensors, + ActionSpec actionSpec + ) + { + var barracudaModel = ModelLoader.Load(nnModel); + var inferenceEvent = new InferenceEvent(); + + // Hash the behavior name so that there's no concern about PII or "secret" data being leaked. + var behaviorNameHash = Hash128.Compute(behaviorName); + inferenceEvent.BehaviorName = behaviorNameHash.ToString(); + + inferenceEvent.BarracudaModelSource = barracudaModel.IrSource; + inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion; + inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName; + inferenceEvent.MemorySize = (int)barracudaModel.GetTensorByName(TensorNames.MemorySize)[0]; + inferenceEvent.InferenceDevice = (int)inferenceDevice; + + if (barracudaModel.ProducerName == "Script") + { + // .nn files don't have these fields set correctly. Assign some placeholder values. + inferenceEvent.BarracudaModelSource = "NN"; + inferenceEvent.BarracudaModelProducer = "tensorflow_to_barracuda.py"; + } + +#if UNITY_2019_3_OR_NEWER && UNITY_EDITOR + var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly); + inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version; +#else + inferenceEvent.BarracudaPackageVersion = null; +#endif + + inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec); + inferenceEvent.ObservationSpecs = new List(sensors.Count); + foreach (var sensor in sensors) + { + inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor)); + } + + inferenceEvent.TotalWeightSizeBytes = GetModelWeightSize(barracudaModel); + inferenceEvent.ModelHash = GetModelHash(barracudaModel); + return inferenceEvent; + } + + /// + /// Compute the total model weight size in bytes. + /// This corresponds to the "Total weight size" display in the Barracuda inspector, + /// and the calculations are the same. + /// + /// + /// + static long GetModelWeightSize(Model barracudaModel) + { + long totalWeightsSizeInBytes = 0; + for (var l = 0; l < barracudaModel.layers.Count; ++l) + { + for (var d = 0; d < barracudaModel.layers[l].datasets.Length; ++d) + { + totalWeightsSizeInBytes += barracudaModel.layers[l].datasets[d].length; + } + } + return totalWeightsSizeInBytes; + } + + /// + /// Wrapper around Hash128 that supports Append(float[], int, int) + /// + struct MLAgentsHash128 + { + private Hash128 m_Hash; + + public void Append(float[] values, int count) + { + if (values == null) + { + return; + } + + // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings and scalars) + // For these versions, we'll hash element by element. +#if UNITY_2020_1_OR_NEWER + m_Hash.Append(values, 0, count); +#else + for (var i = 0; i < count; i++) + { + var tempHash = new Hash128(); + HashUtilities.ComputeHash128(ref values[i], ref tempHash); + HashUtilities.AppendHash(ref tempHash, ref m_Hash); + } +#endif + } + + public void Append(string value) + { + var tempHash = Hash128.Compute(value); + HashUtilities.AppendHash(ref tempHash, ref m_Hash); + } + + public override string ToString() + { + return m_Hash.ToString(); + } + } + + /// + /// Compute a hash of the model's layer data and return it as a string. + /// A subset of the layer weights are used for performance. + /// This increases the chance of a collision, but this should still be extremely rare. + /// + /// + /// + static string GetModelHash(Model barracudaModel) + { + var hash = new MLAgentsHash128(); + + // Limit the max number of float bytes that we hash for performance. + const int kMaxFloats = 256; + + foreach (var layer in barracudaModel.layers) + { + hash.Append(layer.name); + var numFloatsToHash = Mathf.Min(layer.weights.Length, kMaxFloats); + hash.Append(layer.weights, numFloatsToHash); + } + + return hash.ToString(); + } + } +} diff --git a/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta new file mode 100644 index 0000000000..e81b2ecbb6 --- /dev/null +++ b/com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: ac4c40c2394d481ebf602caa600a32f3 +timeCreated: 1604359787 \ No newline at end of file diff --git a/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs b/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs index 3fa8c220a4..851341ff9c 100644 --- a/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs +++ b/com.unity.ml-agents/Runtime/Inference/ModelRunner.cs @@ -86,6 +86,16 @@ public ModelRunner( actionSpec, seed, m_TensorAllocator, m_Memories, barracudaModel); } + public InferenceDevice InferenceDevice + { + get { return m_InferenceDevice; } + } + + public NNModel Model + { + get { return m_Model; } + } + static Dictionary PrepareBarracudaInputs(IEnumerable infInputs) { var inputs = new Dictionary(); diff --git a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs index ff4c995540..4d6cada589 100644 --- a/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs +++ b/com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs @@ -41,20 +41,42 @@ internal class BarracudaPolicy : IPolicy List m_SensorShapes; ActionSpec m_ActionSpec; + private string m_BehaviorName; + + /// + /// Whether or not we've tried to send analytics for this model. We only ever try to send once per policy, + /// and do additional deduplication in the analytics code. + /// + private bool m_AnalyticsSent; + /// public BarracudaPolicy( ActionSpec actionSpec, NNModel model, - InferenceDevice inferenceDevice) + InferenceDevice inferenceDevice, + string behaviorName + ) { var modelRunner = Academy.Instance.GetOrCreateModelRunner(model, actionSpec, inferenceDevice); m_ModelRunner = modelRunner; + m_BehaviorName = behaviorName; m_ActionSpec = actionSpec; } /// public void RequestDecision(AgentInfo info, List sensors) { + if (!m_AnalyticsSent) + { + m_AnalyticsSent = true; + Analytics.InferenceAnalytics.InferenceModelSet( + m_ModelRunner.Model, + m_BehaviorName, + m_ModelRunner.InferenceDevice, + sensors, + m_ActionSpec + ); + } m_AgentId = info.episodeId; m_ModelRunner?.PutObservations(info, sensors); } diff --git a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs index add0dc359e..100220e320 100644 --- a/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs +++ b/com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs @@ -212,7 +212,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen "Either assign a model, or change to a different Behavior Type." ); } - return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice); + return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName); } case BehaviorType.Default: if (Academy.Instance.IsCommunicatorOn) @@ -221,7 +221,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen } if (m_Model != null) { - return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice); + return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName); } else { @@ -241,5 +241,6 @@ internal void UpdateAgentPolicy() } agent.ReloadPolicy(); } + } } diff --git a/com.unity.ml-agents/Tests/Editor/Analytics.meta b/com.unity.ml-agents/Tests/Editor/Analytics.meta new file mode 100644 index 0000000000..473f2be08f --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Analytics.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: adbf291ff40848a296523d69a5be65a5 +timeCreated: 1607379470 \ No newline at end of file diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs new file mode 100644 index 0000000000..f3dbb3feef --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs @@ -0,0 +1,68 @@ +using System.Collections.Generic; +using NUnit.Framework; +using Unity.MLAgents.Sensors; +using UnityEngine; +using Unity.Barracuda; +using Unity.MLAgents.Actuators; +using Unity.MLAgents.Analytics; +using Unity.MLAgents.Policies; +using UnityEditor; + +namespace Unity.MLAgents.Tests.Analytics +{ + [TestFixture] + public class InferenceAnalyticsTests + { + const string k_continuousONNXPath = "Packages/com.unity.ml-agents/Tests/Editor/TestModels/continuous2vis8vec2action.onnx"; + NNModel continuousONNXModel; + Test3DSensorComponent sensor_21_20_3; + Test3DSensorComponent sensor_20_22_3; + + ActionSpec GetContinuous2vis8vec2actionActionSpec() + { + return ActionSpec.MakeContinuous(2); + } + + [SetUp] + public void SetUp() + { + continuousONNXModel = (NNModel)AssetDatabase.LoadAssetAtPath(k_continuousONNXPath, typeof(NNModel)); + var go = new GameObject("SensorA"); + sensor_21_20_3 = go.AddComponent(); + sensor_21_20_3.Sensor = new Test3DSensor("SensorA", 21, 20, 3); + sensor_20_22_3 = go.AddComponent(); + sensor_20_22_3.Sensor = new Test3DSensor("SensorB", 20, 22, 3); + } + + [Test] + public void TestModelEvent() + { + var sensors = new List { sensor_21_20_3.Sensor, sensor_20_22_3.Sensor }; + var behaviorName = "continuousModel"; + + var continuousEvent = InferenceAnalytics.GetEventForModel( + continuousONNXModel, behaviorName, + InferenceDevice.CPU, sensors, GetContinuous2vis8vec2actionActionSpec() + ); + + // The behavior name should be hashed, not pass-through. + Assert.AreNotEqual(behaviorName, continuousEvent.BehaviorName); + + Assert.AreEqual(2, continuousEvent.ActionSpec.NumContinuousActions); + Assert.AreEqual(0, continuousEvent.ActionSpec.NumDiscreteActions); + Assert.AreEqual(2, continuousEvent.ObservationSpecs.Count); + Assert.AreEqual(3, continuousEvent.ObservationSpecs[0].DimensionInfos.Length); + Assert.AreEqual(20, continuousEvent.ObservationSpecs[0].DimensionInfos[0].Size); + Assert.AreEqual("None", continuousEvent.ObservationSpecs[0].CompressionType); + Assert.AreNotEqual(null, continuousEvent.ModelHash); + + // Make sure nested fields get serialized + var jsonString = JsonUtility.ToJson(continuousEvent, true); + Assert.IsTrue(jsonString.Contains("ObservationSpecs")); + Assert.IsTrue(jsonString.Contains("ActionSpec")); + Assert.IsTrue(jsonString.Contains("NumDiscreteActions")); + Assert.IsTrue(jsonString.Contains("SensorName")); + Assert.IsTrue(jsonString.Contains("Flags")); + } + } +} diff --git a/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta new file mode 100644 index 0000000000..20f024f03b --- /dev/null +++ b/com.unity.ml-agents/Tests/Editor/Analytics/InferenceAnalyticsTests.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 9f054f620b8b468bbd8ccf7d2cc14ccd +timeCreated: 1607379491 \ No newline at end of file