Add Activations

tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_LeakyRelu.pbtxt

@@ -1,5 +1,6 @@
 op {
   graph_op_name: "LeakyRelu"
+  visibility: VISIBLE
   endpoint {
     name: "nn.LeakyRelu"
   }

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/NnOps.java

@@ -59,6 +59,7 @@
 import org.tensorflow.op.nn.FusedResizeAndPadConv2d;
 import org.tensorflow.op.nn.InTopK;
 import org.tensorflow.op.nn.L2Loss;
+import org.tensorflow.op.nn.LeakyRelu;
 import org.tensorflow.op.nn.LearnedUnigramCandidateSampler;
 import org.tensorflow.op.nn.LocalResponseNormalization;
 import org.tensorflow.op.nn.LogSoftmax;
@@ -1226,6 +1227,19 @@ public <T extends TNumber> L2Loss<T> l2Loss(Operand<T> t) {
     return L2Loss.create(scope, t);
   }
 
+  /**
+   * Computes rectified linear: `max(features, features * alpha)`.
+   *
+   * @param <T> data type for {@code activations()} output
+   * @param features
+   * @param options carries optional attributes values
+   * @return a new instance of LeakyRelu
+   */
+  public <T extends TNumber> LeakyRelu<T> leakyRelu(Operand<T> features,
+      LeakyRelu.Options... options) {
+    return LeakyRelu.create(scope, features, options);
+  }
+
   /**
    * Generates labels for candidate sampling with a learned unigram distribution.
    *  <p>

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/nn/LeakyRelu.java

@@ -33,6 +33,7 @@
  * 
  * @param <T> data type for {@code activations()} output
  */
+@Operator(group = "nn")
 public final class LeakyRelu<T extends TNumber> extends RawOp implements Operand<T> {
 
   /**

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TBfloat16.java

@@ -30,7 +30,7 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /**
  * Brain 16-bit float tensor type.
@@ -48,7 +48,7 @@
  * <p>Note that some CPUs support the bfloat16 format natively, which can result in faster
  * computation compared to {@link TFloat16} when GPUs are not used.
  */
-public interface TBfloat16 extends FloatNdArray, TNumber {
+public interface TBfloat16 extends FloatNdArray, TFloating {
   /** readable-name for the data type */
   static final String NAME = "BFLOAT16";
 

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat16.java

@@ -30,7 +30,7 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /**
  * IEEE-754 half-precision 16-bit float tensor type.
@@ -45,7 +45,7 @@
  * most CPUs do not support this format natively. For CPU computation on 16-bit floats, the {@link
  * TBfloat16} tensor type might be a better option.
  */
-public interface TFloat16 extends FloatNdArray, TNumber {
+public interface TFloat16 extends FloatNdArray, TFloating {
 
   /** readable-name for the data type */
   static final String NAME = "FLOAT16";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat32.java

@@ -29,10 +29,10 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /** IEEE-754 single-precision 32-bit float tensor type. */
-public interface TFloat32 extends FloatNdArray, TNumber {
+public interface TFloat32 extends FloatNdArray, TFloating {
 
   /** readable-name for the data type */
   static final String NAME = "FLOAT";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat64.java

@@ -29,10 +29,11 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.DoubleDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
+
 
 /** IEEE-754 double-precision 64-bit float tensor type. */
-public interface TFloat64 extends DoubleNdArray, TNumber {
+public interface TFloat64 extends DoubleNdArray, TFloating {
 
   /** readable-name for the data type */
   static final String NAME = "DOUBLE";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/family/TFloating.java

@@ -0,0 +1,19 @@
+package org.tensorflow.types.family;
+
+/**
+ * Marker interface for floating point tensor types.
+ *
+ * <p>Operations that only accepts floating point values as some of their operands enforce that the tensor
+ * types for these operands to be bound to this interface. For example:
+ *
+ * <pre>{@code
+ * TFloat32 tensor1 = TFloat32.vectorOf(1, 2, 3);
+ * TBool tensor2 = TBool.vectorOf(true, false, true);
+ *
+ * Ops tf = Ops.create();
+ * Exponential<TFloat32> exp = new Exponential<>(tf);
+ * exp.call(tf.constant(tensor1));  // OK
+ * exp.call(tf.constant(tensor2));  // Compilation failure
+ * }</pre>
+ */
+public interface TFloating extends TNumber {}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/Activation.java

@@ -0,0 +1,68 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Abstract base class for Activations
+ *
+ * <p><b>Note:</b> The {@link #tf} attribute must be set prior to invoking the call method. See
+ * {@link #setTF(Ops)} and the constructor {@link #Activation(Ops)}.
+ *
+ * @param <T> the data type of the activation
+ */
+public abstract class Activation<T extends TNumber> {
+
+  /** The TensorFlow Ops */
+  protected Ops tf;
+
+  /**
+   * Creates the abstract class for an Activation
+   *
+   * @param tf the TensorFlow Ops
+   */
+  protected Activation(Ops tf) {
+    this.tf = tf;
+  }
+
+  /**
+   * Sets the TensorFlow Ops
+   *
+   * @param tf the TensorFlow Ops
+   */
+  protected void setTF(Ops tf) {
+    this.tf = tf;
+  }
+
+  /**
+   * Gets the TensorFlow Ops
+   *
+   * @return the TensorFlow Ops
+   */
+  protected Ops getTF() {
+    return this.tf;
+  }
+
+  /**
+   * Gets the calculation operation for the activation.
+   *
+   * @param input the input tensor
+   * @return The operand for the activation
+   */
+  public abstract Operand<T> call(Operand<T> input);
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/ELU.java

@@ -0,0 +1,98 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.DataType;
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.TBool;
+import org.tensorflow.types.family.TFloating;
+
+/**
+ * Exponential linear unit.
+ *
+ * <p>The exponential linear unit (ELU) with <code>alpha &gt; 0</code> is:
+ *
+ * <p><code>x</code> if <code>x &gt; 0</code> and <code>alpha * (exp(x) -
+ * 1)</code> if <code>x &lt; 0</code>.
+ *
+ * <p>The ELU hyperparameter <code>alpha</code> controls the value to which an ELU saturates for
+ * negative net inputs. ELUs diminish the vanishing gradient effect.
+ *
+ * <p>ELUs have negative values which pushes the mean of the activations closer to zero. Mean
+ * activations that are closer to zero enable faster learning as they bring the gradient closer to
+ * the natural gradient. ELUs saturate to a negative value when the argument gets smaller.
+ * Saturation means a small derivative which decreases the variation and the information that is
+ * propagated to the next layer.
+ *
+ * <p>Example Usage:
+ *
+ * <pre>
+ *     Operand&lt;TFloat32&gt; input = &#46;&#46;&#46;;
+ *     ELU&lt;TFloat32&gt; elu = new ELU&lt;&gt;(tf, 2.0f);
+ *     Operand&lt;TFloat32&gt; result = elu.call(input);
+ * </pre>
+ *
+ * @param <T> the data type of the activation
+ * @see <a href="https://arxiv.org/abs/1511.07289">Clevert et al, 2016, Fast and Accurate Deep
+ *     Network Learning by Exponential Linear Units (ELUs)</a>
+ */
+public class ELU<T extends TFloating> extends Activation<T> {
+
+  private static final double ALPHA_DEFAULT = 1.0;
+
+  /** A scalar, slope of negative section. */
+  private final double alpha;
+
+  /**
+   * Creates a new ELU with alpha={@link #ALPHA_DEFAULT}.
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public ELU(Ops tf) {
+    this(tf, ALPHA_DEFAULT);
+  }
+
+  /**
+   * Creates a new ELU
+   *
+   * @param tf the TensorFlow Ops
+   * @param alpha A scalar, slope of negative section. It controls the value to which an ELU
+   *     saturates for negative net inputs.
+   */
+  public ELU(Ops tf, double alpha) {
+    super(tf);
+    this.alpha = alpha;
+  }
+
+  /**
+   * Gets the calculation operation for the activation.
+   *
+   * @param input the input tensor
+   * @return The operand for the activation
+   */
+  @Override
+  public Operand<T> call(Operand<T> input) {
+
+    Operand<T> result = tf.nn.elu(input);
+    if (alpha == 1.0) return result;
+    else {
+      DataType<T> dataType = input.asOutput().dataType();
+      Operand<T> y = tf.math.mul(result, tf.dtypes.cast(tf.constant(alpha), dataType));
+      Operand<TBool> cond = tf.math.greater(result, tf.dtypes.cast(tf.constant(0), dataType));
+      return tf.select(cond, result, y);
+    }
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/Exponential.java

@@ -0,0 +1,57 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TFloating;
+
+/**
+ * Exponential activation function.
+ *
+ * <p>For example:
+ *
+ * <pre>
+ *   Operand&lt;TFloat32&gt; input = tf.constant(
+ *          new float[] {-3.0f,-1.0f, 0.0f,1.0f,3.0f});
+ *   Exponential&lt;TFloat32&gt; exp = new Exponential&lt;&gt;(tf);
+ *   Operand&lt;TFloat32&gt; result = exp.call(input);
+ *   // result is [0.04978707f,  0.36787945f,  1.f,  2.7182817f, 20.085537f]
+ * </pre>
+ *
+ * @param <T> the data type of the activation
+ */
+public class Exponential<T extends TFloating> extends Activation<T> {
+
+  /**
+   * Creates an Exponential activation.
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public Exponential(Ops tf) {
+    super(tf);
+  }
+
+  /**
+   * Calculates the Exponential activation.
+   *
+   * @param input the input tensor
+   * @return an Operand for the exponential activation: <code>exp(x)</code>.
+   */
+  @Override
+  public Operand<T> call(Operand<T> input) {
+    return tf.math.exp(input);
+  }
+}