comment on Softmax alpha term
This commit is contained in:
parent
b8c40d2e2f
commit
1e6887cdbd
1 changed files with 2 additions and 0 deletions
|
@ -137,6 +137,8 @@ class Gelu(Activation):
|
|||
|
||||
class Softmax(Activation):
|
||||
def forward(self, X):
|
||||
# this alpha term is for numerical stability
|
||||
# and is not strictly essential.
|
||||
alpha = np.max(X, axis=-1, keepdims=True)
|
||||
num = np.exp(X - alpha)
|
||||
den = np.sum(num, axis=-1, keepdims=True)
|
||||
|
|
Loading…
Reference in a new issue