1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
package ml;
public class Functions {
/*--------------------------------------------------------------*/
/*---------------- Sigmoid ----------------*/
/*--------------------------------------------------------------*/
public static double sigmoid(double x) {return 1.0/(1.0+Math.exp(-x));}
static final double sigmoidDerivativeX(double x) {return sigmoidDerivativeFX(sigmoid(x));}
static final double sigmoidDerivativeFX(double fx) {return fx*(1-fx);}
static final double sigmoidDerivativeXFX(double x, double fx) {return fx*(1-fx);}
/*--------------------------------------------------------------*/
/*---------------- Extended Sigmoid ----------------*/
/*--------------------------------------------------------------*/
public static double eSigmoid(double x) {
return 2*sigmoid(x)-1;
}
static final double eSigmoidDerivativeX(double x) {
return 2*sigmoidDerivativeX(x);
}
static final double eSigmoidDerivativeFX(double fx) {
final double fx2=fx+1;
// final double sfx=fx2*0.5;
// final double d=2*sigmoidDerivativeFX(sfx); //This should be correct
final double d2=0.5*fx2*(2-fx2); //This now matches the correct one and should be faster.
// assert(d==d2) : fx+", "+fx2+", "+sfx+", "+d+", "+d2;
return d2;
}
static final double eSigmoidDerivativeXFX(double x, double fx) {
return eSigmoidDerivativeFX(fx);//TODO: Change to x if x is faster
}
/*--------------------------------------------------------------*/
/*---------------- TanH ----------------*/
/*--------------------------------------------------------------*/
public static float tanh(double x) {
// return (float)Math.tanh(x); //This is slower and gives the same results
if(x<-20) {return -1;}
if(x>20) {return 1;}
double ex=Math.exp(x);
double emx=Math.exp(-x);
return (float)((ex-emx)/(ex+emx));
// float ex=(float)Math.exp(x);This gives totally different results
// float emx=(float)Math.exp(-x);
// return ((ex-emx)/(ex+emx));
}
static final double tanhDerivativeX(double x) {
return tanhDerivativeFX(tanh(x));
}
static final double tanhDerivativeFX(double fx) {
return 1-fx*fx;
}
static final double tanhDerivativeXFX(double x, double fx) {
return 1-fx*fx;
}
/*--------------------------------------------------------------*/
/*---------------- Swish ----------------*/
/*--------------------------------------------------------------*/
public static double swish(double x) {
// double b=1;
// return x*sigmoidD(b*x);
// return x*sigmoidD(x);
final double y=x/(1+Math.exp(-x)); //Maybe faster
// return y<100000 ? y : Math.min(y, 100000+Math.log(y));
return y;
}
public static double swishDerivativeX(double x) {
double sigx=sigmoid(x);
double fx=x*sigx;
//double fx=swish(x);
return swishDerivativeFXSIGX(fx, sigx);
}
static final double swishDerivativeFX(double fx) {
throw new RuntimeException("Unimplemented.");
}
public static double swishDerivativeXFX(double x, double fx) {
double sigx=sigmoid(x);
return swishDerivativeFXSIGX(fx, sigx);
}
public static double swishDerivativeFXSIGX(double fx, double sigx) {
return fx+sigx*(1-fx);
}
/*--------------------------------------------------------------*/
/*---------------- RSLog ----------------*/
/*--------------------------------------------------------------*/
public static double rslog(double x) {
if(x<0) {return -rslog(-x);}
return Math.log(x+1);
}
static final double rslogDerivativeX(double x) {
if(x<0) {x=-x;}
return 1/(x+1);
}
static final double rslogDerivativeFX(double fx) {
assert(false);
if(fx<0) {fx=-fx;}
//if fx=10 then log(x+1)=10 so e^10=x+1
double x=Math.exp(fx)-1;
return rslogDerivativeX(x);
}
static final double rslogDerivativeXFX(double x, double fx) {
return rslogDerivativeX(x);
}
/*--------------------------------------------------------------*/
/*---------------- Mirrored Sigmoid ----------------*/
/*--------------------------------------------------------------*/
private static final double MSIG_X_OFFSET=5; //Bigger makes the peak higher, wider, and less sharp
private static final double MSIG_X_MULT=2; //Useful if you design around a 0-1 range
private static final double MSIG_Y_MULT=1.0/sigmoid(MSIG_X_OFFSET);//Should be very slightly higher than 1.
public static double mSig(double x) {
final double offset=MSIG_X_OFFSET;
final double xmult=MSIG_X_MULT;
final double ymult=MSIG_Y_MULT;
//sigmoid: 1.0/(1.0+Math.exp(-x));
if(x<0) {
//=1/(1+EXP(-(2*C7+$E$5)))
double y=1.0/(1.0+Math.exp(-(xmult*x+offset)));
// double y2=sigmoid(mult*x+offset);
// assert(y2==y) : y+", "+y2+", "+x;
return ymult*y;
}else {
double y=1.0/(1.0+Math.exp(xmult*x-offset));
// double y2=sigmoid(-(mult*x-offset));
// assert(y2==y) : y+", "+y2+", "+x;
return ymult*y;
}
}
static final double mSigDerivativeX(double x) {
double fx=mSig(x);
return mSigDerivativeXFX(x, fx);
}
static final double mSigDerivativeFX(double fx) {
throw new RuntimeException("Cannot be calculated.");
}
static final double mSigDerivativeXFX(double x, double fx) {
final double xmult=MSIG_X_MULT;
double d=sigmoidDerivativeFX(fx);
if(x<0){return xmult*d;}
return -xmult*d;
}
/*--------------------------------------------------------------*/
/*---------------- Extended Mirrored Sigmoid ----------------*/
/*--------------------------------------------------------------*/
public static double emSig(double x) {
return 2*mSig(x)-1;
}
static final double emSigDerivativeX(double x) {
return 2*mSigDerivativeX(x);
}
static final double emSigDerivativeFX(double fx) {
throw new RuntimeException("Cannot be calculated.");
}
static final double emSigDerivativeXFX(double x, double fx) {
return 2*mSigDerivativeX(x); //Possibly slow, but simple.
}
/*--------------------------------------------------------------*/
/*---------------- Gaussian ----------------*/
/*--------------------------------------------------------------*/
public static double bell(double x) {
return Math.exp(-(x*x));
}
static final double bellDerivativeX(double x) {
return -2*x*Math.exp(-x*x); //i.e. -2*x*fx
}
static final double bellDerivativeFX(double fx) {
throw new RuntimeException("Cannot be calculated.");
}
static final double bellDerivativeXFX(double x, double fx) {
return -2*x*fx;
}
/*--------------------------------------------------------------*/
/*---------------- Other ----------------*/
/*--------------------------------------------------------------*/
//For multiple tests of a single output neuron
public static double mse(float[] target, float[] actual) {
assert(target.length==actual.length);
double sum=0;
final int len=target.length;
for(int i=0; i<len; i++) {
float t=target[i], a=actual[i];
float dif=t-a;
sum+=(dif*dif);
}
return sum/len;
}
}
|
