notwa/optim
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

comment on Softmax alpha term

Browse source
This commit is contained in:
Connor Olding 2019-02-03 14:31:34 +01:00
parent b8c40d2e2f
commit 1e6887cdbd
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
onn/activation.py
View file

@ -137,6 +137,8 @@ class Gelu(Activation):
class Softmax(Activation): class Softmax(Activation):
def forward(self, X): def forward(self, X):
# this alpha term is for numerical stability
# and is not strictly essential.
alpha = np.max(X, axis=-1, keepdims=True) alpha = np.max(X, axis=-1, keepdims=True)
num = np.exp(X - alpha) num = np.exp(X - alpha)
den = np.sum(num, axis=-1, keepdims=True) den = np.sum(num, axis=-1, keepdims=True)