1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
|
#include <math.h>
#include "caffe2/operators/glu_op.h"
namespace caffe2 {
namespace {
float sigmoid(const float x) {
if (x >= 0) {
return 1. / (1. + exp(-x));
} else {
const float exp_x = exp(x);
return exp_x / (1 + exp_x);
}
}
} // namespace
template <>
void GluOp<float, CPUContext>::ComputeGlu(
const int M,
const int split_dim,
const int N,
const float* Xdata,
float* Ydata) {
const int xStride = 2 * split_dim * N;
const int yStride = split_dim * N;
for (int i = 0; i < M; ++i) {
const int idx = i * xStride;
const int idy = i * yStride;
for (int j = 0; j < split_dim; ++j) {
const int jN = j * N;
const int jdx1 = idx + jN;
const int jdx2 = idx + (j + split_dim) * N;
const int jdy = idy + jN;
for (int k = 0; k < N; ++k) {
const float x1 = Xdata[jdx1 + k];
const float x2 = Xdata[jdx2 + k];
Ydata[jdy + k] = x1 * sigmoid(x2);
}
}
}
}
OPERATOR_SCHEMA(Glu)
.NumInputs(1)
.NumOutputs(1)
.SetDoc(R"DOC(
Applies gated linear unit to the input Tensor X. The output Y is half the size
of the input X, so if the shape of X is [d1, d2, ..., N] shape of Y will be
[d1, d2, ..., dn/2] and Y(:dn-1, i) = GLU(X(:dn-1, i), X(:dn-1, i+N/2)) =
X(dn-1, i) * sigmoid(X(dn-1, i+N/2))
)DOC")
.Input(0, "X", "1D input tensor")
.Output(0, "Y", "1D output tensor");
REGISTER_CPU_OPERATOR(Glu, GluOp<float, CPUContext>);
} // namespace caffe2
|
