1 ! Copyright (C) 2017 Doug Coleman.
2 ! See http://factorcode.org/license.txt for BSD license.
3 USING: kernel math math.functions math.order math.vectors
5 IN: machine-learning.functions
7 : relu ( x -- x' ) 0 max ; inline
9 : relu6 ( x -- x' ) 0 6 clamp ; inline
11 : leaky-relu ( x a -- x' )
12 over 0 < [ * ] [ drop ] if ; inline
14 ! https://arxiv.org/pdf/1706.02515.pdf
16 over 0 < [ [ [ e^ ] dip * ] keep - ] [ drop ] if ; inline
18 : default-leaky-relu ( x -- x' )
19 .01 leaky-relu ; inline
21 : vexp-sum ( seq -- seq' sum )
22 [ e^ ] map dup sum ; inline
24 : softmax ( seq -- softmax )
25 vexp-sum '[ _ /f ] map ; inline
27 : log-softmax ( seq -- softmax )
28 vexp-sum '[ e^ _ * recip log ] map ;
30 : softmin ( seq -- softmin )