[MLA-1540] Training Analytics (#4780)

Author: Chris Elion

com.unity.ml-agents/Runtime/Academy.cs

@@ -91,9 +91,13 @@ public class Academy : IDisposable
         ///         <term>1.3.0</term>
         ///         <description>Support both continuous and discrete actions.</description>
         ///     </item>
+        ///     <item>
+        ///         <term>1.4.0</term>
+        ///         <description>Support training analytics sent from python trainer to the editor.</description>
+        ///     </item>
         /// </list>
         /// </remarks>
-        const string k_ApiVersion = "1.3.0";
+        const string k_ApiVersion = "1.4.0";
 
         /// <summary>
         /// Unity package version of com.unity.ml-agents.
@@ -406,6 +410,7 @@ void InitializeEnvironment()
             EnableAutomaticStepping();
 
             SideChannelManager.RegisterSideChannel(new EngineConfigurationChannel());
+            SideChannelManager.RegisterSideChannel(new TrainingAnalyticsSideChannel());
             m_EnvironmentParameters = new EnvironmentParameters();
             m_StatsRecorder = new StatsRecorder();
 

com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs

@@ -0,0 +1,40 @@
+using System;
+using UnityEngine;
+
+namespace Unity.MLAgents.Analytics
+{
+    internal static class AnalyticsUtils
+    {
+        /// <summary>
+        /// Hash a string to remove PII or secret info before sending to analytics
+        /// </summary>
+        /// <param name="s"></param>
+        /// <returns>A string containing the Hash128 of the input string.</returns>
+        public static string Hash(string s)
+        {
+            var behaviorNameHash = Hash128.Compute(s);
+            return behaviorNameHash.ToString();
+        }
+
+        internal static bool s_SendEditorAnalytics = true;
+
+        /// <summary>
+        /// Helper class to temporarily disable sending analytics from unit tests.
+        /// </summary>
+        internal class DisableAnalyticsSending : IDisposable
+        {
+            private bool m_PreviousSendEditorAnalytics;
+
+            public DisableAnalyticsSending()
+            {
+                m_PreviousSendEditorAnalytics = s_SendEditorAnalytics;
+                s_SendEditorAnalytics = false;
+            }
+
+            public void Dispose()
+            {
+                s_SendEditorAnalytics = m_PreviousSendEditorAnalytics;
+            }
+        }
+    }
+}

com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs.meta

@@ -91,4 +91,67 @@ public static EventObservationSpec FromSensor(ISensor sensor)
             };
         }
     }
+
+    internal struct RemotePolicyInitializedEvent
+    {
+        public string TrainingSessionGuid;
+        /// <summary>
+        /// Hash of the BehaviorName.
+        /// </summary>
+        public string BehaviorName;
+        public List<EventObservationSpec> ObservationSpecs;
+        public EventActionSpec ActionSpec;
+
+        /// <summary>
+        /// This will be the same as TrainingEnvironmentInitializedEvent if available, but
+        /// TrainingEnvironmentInitializedEvent maybe not always be available with older trainers.
+        /// </summary>
+        public string MLAgentsEnvsVersion;
+        public string TrainerCommunicationVersion;
+    }
+
+    internal struct TrainingEnvironmentInitializedEvent
+    {
+        public string TrainingSessionGuid;
+
+        public string TrainerPythonVersion;
+        public string MLAgentsVersion;
+        public string MLAgentsEnvsVersion;
+        public string TorchVersion;
+        public string TorchDeviceType;
+        public int NumEnvironments;
+        public int NumEnvironmentParameters;
+    }
+
+    [Flags]
+    internal enum RewardSignals
+    {
+        Extrinsic = 1 << 0,
+        Gail = 1 << 1,
+        Curiosity = 1 << 2,
+        Rnd = 1 << 3,
+    }
+
+    [Flags]
+    internal enum TrainingFeatures
+    {
+        BehavioralCloning = 1 << 0,
+        Recurrent = 1 << 1,
+        Threaded = 1 << 2,
+        SelfPlay = 1 << 3,
+        Curriculum = 1 << 4,
+    }
+
+    internal struct TrainingBehaviorInitializedEvent
+    {
+        public string TrainingSessionGuid;
+
+        public string BehaviorName;
+        public string TrainerType;
+        public RewardSignals RewardSignalFlags;
+        public TrainingFeatures TrainingFeatureFlags;
+        public string VisualEncoder;
+        public int NumNetworkLayers;
+        public int NumNetworkHiddenUnits;
+    }
 }

com.unity.ml-agents/Runtime/Analytics/Events.cs

@@ -116,7 +116,10 @@ ActionSpec actionSpec
             // Note - to debug, use JsonUtility.ToJson on the event.
             //Debug.Log(JsonUtility.ToJson(data, true));
 #if UNITY_EDITOR
-            EditorAnalytics.SendEventWithLimit(k_EventName, data, k_EventVersion);
+            if (AnalyticsUtils.s_SendEditorAnalytics)
+            {
+                EditorAnalytics.SendEventWithLimit(k_EventName, data, k_EventVersion);
+            }
 #else
             return;
 #endif
@@ -143,8 +146,7 @@ ActionSpec actionSpec
             var inferenceEvent = new InferenceEvent();
 
             // Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
-            var behaviorNameHash = Hash128.Compute(behaviorName);
-            inferenceEvent.BehaviorName = behaviorNameHash.ToString();
+            inferenceEvent.BehaviorName = AnalyticsUtils.Hash(behaviorName);
 
             inferenceEvent.BarracudaModelSource = barracudaModel.IrSource;
             inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion;