From 1e6887cdbd59d7716b13a4390c96dade3419f7c1 Mon Sep 17 00:00:00 2001 From: Connor Olding Date: Sun, 3 Feb 2019 14:31:34 +0100 Subject: [PATCH] comment on Softmax alpha term --- onn/activation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onn/activation.py b/onn/activation.py index a18a1d0..340c2f1 100644 --- a/onn/activation.py +++ b/onn/activation.py @@ -137,6 +137,8 @@ class Gelu(Activation): class Softmax(Activation): def forward(self, X): + # this alpha term is for numerical stability + # and is not strictly essential. alpha = np.max(X, axis=-1, keepdims=True) num = np.exp(X - alpha) den = np.sum(num, axis=-1, keepdims=True)