Added python implementation for mish (#1139)

* Added py implementation for mish

Author: gabrieldemarmiesse

tensorflow_addons/activations/mish.py

@@ -42,3 +42,7 @@ def mish(x: types.TensorLike) -> tf.Tensor:
 @tf.RegisterGradient("Addons>Mish")
 def _mish_grad(op, grad):
     return _activation_so.ops.addons_mish_grad(grad, op.inputs[0])
+
+
+def _mish_py(x):
+    return x * tf.math.tanh(tf.math.softplus(x))

tensorflow_addons/activations/mish_test.py

@@ -18,6 +18,7 @@
 import numpy as np
 import tensorflow as tf
 from tensorflow_addons.activations import mish
+from tensorflow_addons.activations.mish import _mish_py
 from tensorflow_addons.utils import test_utils
 
 
@@ -42,6 +43,28 @@ def test_theoretical_gradients(self, dtype):
         theoretical, numerical = tf.test.compute_gradient(mish, [x])
         self.assertAllCloseAccordingToType(theoretical, numerical, atol=1e-4)
 
+    @parameterized.named_parameters(("float32", np.float32), ("float64", np.float64))
+    def test_same_as_py_func(self, dtype):
+        np.random.seed(1234)
+        for _ in range(20):
+            self.verify_funcs_are_equivalent(dtype)
+
+    def verify_funcs_are_equivalent(self, dtype):
+        x_np = np.random.uniform(-10, 10, size=(4, 4)).astype(dtype)
+        x = tf.convert_to_tensor(x_np)
+
+        with tf.GradientTape(persistent=True) as t:
+            t.watch(x)
+            y_native = mish(x)
+            y_py = _mish_py(x)
+
+        self.assertAllCloseAccordingToType(y_native, y_py, atol=1e-4)
+
+        grad_native = t.gradient(y_native, x)
+        grad_py = t.gradient(y_py, x)
+
+        self.assertAllCloseAccordingToType(grad_native, grad_py, atol=1e-4)
+
 
 if __name__ == "__main__":
     tf.test.main()