comment on Softmax alpha term

2019-02-03 14:31:34 +01:00 · 2019-02-03 14:31:34 +01:00 · 1e6887cdbd
parent b8c40d2e2f
commit 1e6887cdbd
1 changed files with 2 additions and 0 deletions
--- a/onn/activation.py
+++ b/onn/activation.py
@ -137,6 +137,8 @@ class Gelu(Activation):

 class Softmax(Activation):
    def forward(self, X):
+        # this alpha term is for numerical stability
+        # and is not strictly essential.
        alpha = np.max(X, axis=-1, keepdims=True)
        num = np.exp(X - alpha)
        den = np.sum(num, axis=-1, keepdims=True)