Sam 8 LeakyReLU

.github/workflows/ci.yml

@@ -8,14 +8,8 @@ jobs:
     runs-on: ${{ matrix.operating-system }}
     strategy:
       matrix:
-<<<<<<< HEAD
         operating-system: [windows-latest, ubuntu-latest, macos-latest]
         php-versions: ['8.4']
-=======
-        operating-system: [ubuntu-latest, macos-latest]
-        php-versions: ['8.0', '8.1', '8.2']
->>>>>>> master
-
     steps:
       - name: Checkout
         uses: actions/checkout@v3

docs/transformers/regex-filter.md

@@ -28,7 +28,6 @@ $transformer = new RegexFilter([
 ```
 
 ## Predefined Regex Patterns
-<<<<<<< HEAD
 | Class Constant | Description                                                                                              |
 |---|----------------------------------------------------------------------------------------------------------|
 | EMAIL | A pattern to match any email address.                                                                    |
@@ -41,20 +40,6 @@ $transformer = new RegexFilter([
 | EXTRA_WHITESPACE | Matches consecutively repeated whitespace characters.                                                    |
 | MENTION | A pattern that matches Twitter-style mentions (@example).                                                |
 | HASHTAG | Matches Twitter-style hashtags (#example).                                                               |
-=======
-| Class Constant | Description |
-|---|---|
-| EMAIL | A pattern to match any email address. |
-| URL | An alias for the default (Gruber 1) URL matching pattern. |
-| GRUBER_1 | The original Gruber URL matching pattern. |
-| GRUBER_2 | The improved Gruber URL matching pattern. |
-| EXTRA_CHARACTERS | Matches consecutively repeated non word or number characters such as punctuation and special characters. |
-| EXTRA_WORDS | Matches consecutively repeated words. |
-| EXTRA_WHITESPACE | Matches consecutively repeated whitespace characters. |
-| EMOJIS | A pattern to match unicode emojis. |
-| MENTION | A pattern that matches Twitter-style mentions (@example). |
-| HASHTAG | Matches Twitter-style hashtags (#example). |
->>>>>>> 2.4
 
 ## Additional Methods
 This transformer does not have any additional methods.

src/NeuralNet/ActivationFunctions/Base/Contracts/IBufferDerivative.php

@@ -13,14 +13,15 @@
  * @package     Rubix/ML
  * @author      Andrew DalPino
  * @author      Aleksei Nechaev <[email protected]>
+ * @author      Samuel Akopyan <[email protected]>
  */
 interface IBufferDerivative extends Derivative
 {
     /**
      * Calculate the derivative of the single parameter.
      *
-     * @param NDArray $x Input matrix
+     * @param NDArray $input Input matrix
      * @return NDArray Derivative matrix
      */
-    public function differentiate(NDArray $x) : NDArray;
+    public function differentiate(NDArray $input) : NDArray;
 }

src/NeuralNet/ActivationFunctions/Base/Contracts/IOBufferDerivative.php

@@ -2,6 +2,25 @@
 
 namespace Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts;
 
+use NDArray;
+
+/**
+ * Derivative based on input / output buffer
+ *
+ * @category    Machine Learning
+ * @package     Rubix/ML
+ * @author      Andrew DalPino
+ * @author      Aleksei Nechaev <[email protected]>
+ * @author      Samuel Akopyan <[email protected]>
+ */
 interface IOBufferDerivative
 {
+    /**
+     * Calculate the derivative of the activation.
+     *
+     * @param NDArray $input Input matrix
+     * @param NDArray $output Output matrix
+     * @return NDArray Derivative matrix
+     */
+    public function differentiate(NDArray $input, NDArray $output) : NDArray;
 }

src/NeuralNet/ActivationFunctions/Base/Contracts/OBufferDerivative.php

@@ -2,6 +2,24 @@
 
 namespace Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts;
 
+use NDArray;
+
+/**
+ * Derivative based on output buffer
+ *
+ * @category    Machine Learning
+ * @package     Rubix/ML
+ * @author      Andrew DalPino
+ * @author      Aleksei Nechaev <[email protected]>
+ * @author      Samuel Akopyan <[email protected]>
+ */
 interface OBufferDerivative
 {
+    /**
+     * Calculate the derivative of the activation.
+     *
+     * @param NDArray $output Output matrix
+     * @return NDArray Derivative matrix
+     */
+    public function differentiate(NDArray $output) : NDArray;
 }

src/NeuralNet/ActivationFunctions/ELU/ELU.php

@@ -7,7 +7,7 @@
 use NumPower;
 use NDArray;
 use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\ActivationFunction;
-use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IBufferDerivative;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IOBufferDerivative;
 use Rubix\ML\NeuralNet\ActivationFunctions\ELU\Exceptions\InvalidAlphaException;
 
 /**
@@ -26,7 +26,7 @@
  * @author      Aleksei Nechaev <[email protected]>
  * @author      Samuel Akopyan <[email protected]>
  */
-class ELU implements ActivationFunction, IBufferDerivative
+class ELU implements ActivationFunction, IOBufferDerivative
 {
     /**
      * Class constructor.
@@ -71,28 +71,29 @@ public function activate(NDArray $input) : NDArray
     }
 
     /**
-     * Calculate the derivative of the activation function.
+     * Calculate the derivative of the ELU activation function using input and output.
      *
-     * f'(x) = 1               if x > 0
-     * f'(x) = α * e^x         if x ≤ 0
+     * f'(x) = 1             if x > 0
+     * f'(x) = f(x) + α      if x ≤ 0, where f(x) is the ELU output
      *
-     * @param NDArray $x Output matrix
-     * @return NDArray Derivative matrix
+     * @param NDArray $input  Input matrix (used to determine x > 0 mask)
+     * @param NDArray $output Output from the ELU activation function
+     * @return NDArray        Derivative matrix
      */
-    public function differentiate(NDArray $x) : NDArray
+    public function differentiate(NDArray $input, NDArray $output): NDArray
     {
         // For x > 0: 1
-        $positivePart = NumPower::greater($x, 0);
+        $positiveMask = NumPower::greater($input, 0);
 
-        // For x <= 0: α * e^x
-        $negativeMask = NumPower::lessEqual($x, 0);
+        // For x <= 0: output + α
+        $negativeMask = NumPower::lessEqual($input, 0);
         $negativePart = NumPower::multiply(
-            NumPower::multiply($negativeMask, NumPower::exp($x)),
-            $this->alpha
+            NumPower::add($output, $this->alpha),
+            $negativeMask
         );
 
         // Combine both parts
-        return NumPower::add($positivePart, $negativePart);
+        return NumPower::add($positiveMask, $negativePart);
     }
 
     /**

src/NeuralNet/ActivationFunctions/GELU/GELU.php

@@ -87,19 +87,19 @@ public function activate(NDArray $input) : NDArray
      * - β = 0.044715
      * - sech^2(z) = (1/cosh(z))^2
      *
-     * @param NDArray $x Output matrix
+     * @param NDArray $input Input matrix
      * @return NDArray Derivative matrix
      */
-    public function differentiate(NDArray $x) : NDArray
+    public function differentiate(NDArray $input) : NDArray
     {
         // Calculate x^3
-        $cubed = NumPower::pow($x, 3);
+        $cubed = NumPower::pow($input, 3);
 
         // Calculate inner term: ALPHA * (x + BETA * x^3)
         $innerTerm = NumPower::multiply(
             self::ALPHA,
             NumPower::add(
-                $x,
+                $input,
                 NumPower::multiply(self::BETA, $cubed)
             )
         );
@@ -122,15 +122,15 @@ public function differentiate(NDArray $x) : NDArray
             NumPower::multiply(
                 NumPower::multiply(
                     0.5 * self::ALPHA,
-                    $x
+                    $input
                 ),
                 $sech2
             ),
             NumPower::add(
                 1.0,
                 NumPower::multiply(
                     3.0 * self::BETA,
-                    NumPower::pow($x, 2)
+                    NumPower::pow($input, 2)
                 )
             )
         );

src/NeuralNet/ActivationFunctions/HardSiLU/HardSiLU.php

@@ -63,16 +63,16 @@ public function activate(NDArray $input) : NDArray
      *
      * f'(x) = HardSigmoid(x) + x * HardSigmoid'(x)
      *
-     * @param NDArray $x Input matrix
+     * @param NDArray $output Output matrix
      * @return NDArray Derivative matrix
      */
-    public function differentiate(NDArray $x) : NDArray
+    public function differentiate(NDArray $output) : NDArray
     {
         // Calculate HardSigmoid(x)
-        $hardSigmoid = $this->hardSigmoid->activate($x);
+        $hardSigmoid = $this->hardSigmoid->activate($output);
 
         // Calculate HardSigmoid'(x)
-        $hardSigmoidDerivative = $this->hardSigmoid->differentiate($x);
+        $hardSigmoidDerivative = $this->hardSigmoid->differentiate($output);
 
         // Calculate x * HardSigmoid'(x)
         $xTimesDerivative = NumPower::multiply($x, $hardSigmoidDerivative);

src/NeuralNet/ActivationFunctions/HardSigmoid/HardSigmoid.php

@@ -76,17 +76,17 @@ public function activate(NDArray $input) : NDArray
     /**
      * Calculate the derivative of the activation function.
      *
-     * f'(x) = 0.2 if -2.5 < x < 2.5
+     * f'(x) = 0.2 if -2.5 <= x <= 2.5
      * f'(x) = 0   otherwise
      *
-     * @param NDArray $x Input matrix
+     * @param NDArray $input Input matrix
      * @return NDArray Derivative matrix
      */
-    public function differentiate(NDArray $x) : NDArray
+    public function differentiate(NDArray $input) : NDArray
     {
-        // For values in the linear region (-2.5 < x < 2.5): SLOPE
-        $inLinearRegion = NumPower::greater($x, self::LOWER_BOUND);
-        $inLinearRegion = NumPower::multiply($inLinearRegion, NumPower::less($x, self::UPPER_BOUND));
+        // For values in the linear region (-2.5 <= x <= 2.5): SLOPE
+        $inLinearRegion = NumPower::greaterEqual($input, self::LOWER_BOUND);
+        $inLinearRegion = NumPower::multiply($inLinearRegion, NumPower::lessEqual($input, self::UPPER_BOUND));
         $linearPart = NumPower::multiply($inLinearRegion, self::SLOPE);
 
         // For values outside the linear region: 0

src/NeuralNet/ActivationFunctions/HyperbolicTangent/HyperbolicTangent.php

@@ -6,6 +6,8 @@
 
 use NumPower;
 use NDArray;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\ActivationFunction;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IBufferDerivative;
 
 /**
  * Hyperbolic Tangent
@@ -16,31 +18,44 @@
  * @category    Machine Learning
  * @package     Rubix/ML
  * @author      Andrew DalPino
+ * @author      Samuel Akopyan <[email protected]>
  */
-class HyperbolicTangent implements ActivationFunction
+class HyperbolicTangent implements ActivationFunction, IBufferDerivative
 {
     /**
-     * @inheritdoc
+     * Apply the Hyperbolic Tangent activation function to the input.
+     *
+     * f(x) = tanh(x)
+     *
+     * @param NDArray $input The input values
+     * @return NDArray The activated values
      */
     public function activate(NDArray $input) : NDArray
     {
         return NumPower::tanh($input);
     }
 
     /**
-     * @inheritdoc
+     * Calculate the derivative of the activation function.
+     *
+     * f'(x) = 1 - tanh^2(x)
+     *
+     * @param NDArray $output Output matrix
+     * @return NDArray Derivative matrix
      */
     public function differentiate(NDArray $output) : NDArray
     {
-        return 1 - ($output ** 2);
+        // Calculate tanh^2(x)
+        $squared = NumPower::pow($output, 2);
+
+        // Calculate 1 - tanh^2(x)
+        return NumPower::subtract(1.0, $squared);
     }
 
     /**
      * Return the string representation of the activation function.
      *
-     * @internal
-     *
-     * @return string
+     * @return string String representation
      */
     public function __toString() : string
     {

src/NeuralNet/ActivationFunctions/LeakyReLU/Exceptions/InvalidLeakageException.php

@@ -0,0 +1,14 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rubix\ML\NeuralNet\ActivationFunctions\LeakyReLU\Exceptions;
+
+use Rubix\ML\Exceptions\InvalidArgumentException;
+
+/**
+ * Invalid `Leakage` parameter for LeakyReLU Activation function
+ */
+class InvalidLeakageException extends InvalidArgumentException
+{
+}