comment on Softmax alpha term

This commit is contained in:
Connor Olding 2019-02-03 14:31:34 +01:00
parent b8c40d2e2f
commit 1e6887cdbd

View file

@ -137,6 +137,8 @@ class Gelu(Activation):
class Softmax(Activation):
def forward(self, X):
# this alpha term is for numerical stability
# and is not strictly essential.
alpha = np.max(X, axis=-1, keepdims=True)
num = np.exp(X - alpha)
den = np.sum(num, axis=-1, keepdims=True)