1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
#include "caffe2/operators/swish_op.h"
#include <string>
#include <vector>
#include "caffe2/core/types.h"
#include "caffe2/utils/eigen_utils.h"
#include "caffe2/utils/math.h"
namespace caffe2 {
template <>
template <typename T>
bool SwishFunctor<CPUContext>::
operator()(const int N, const T* X, T* Y, CPUContext* /* context */) const {
ConstEigenVectorArrayMap<T> X_arr(X, N);
EigenVectorArrayMap<T>(Y, N) = X_arr / (T(1) + (-X_arr).exp());
return true;
}
template <>
template <typename T>
bool SwishGradientOp<CPUContext>::DoRunWithType() {
auto& Xin = Input(X);
auto& Yin = Input(Y);
auto& DYin = Input(DY);
CAFFE_ENFORCE_EQ(Xin.numel(), Yin.numel());
CAFFE_ENFORCE_EQ(DYin.numel(), Yin.numel());
auto* DXout = Output(DX, Yin.sizes(), at::dtype<float>());
const float* Xdata = Xin.template data<float>();
const float* Ydata = Yin.template data<float>();
const float* dYdata = DYin.template data<float>();
float* dXdata = DXout->template mutable_data<float>();
EigenVectorArrayMap<float> dXvec(dXdata, DXout->numel());
ConstEigenVectorArrayMap<float> Xvec(Xdata, Xin.numel());
ConstEigenVectorArrayMap<float> Yvec(Ydata, Yin.numel());
ConstEigenVectorArrayMap<float> dYvec(dYdata, DYin.numel());
// dx = dy * (y + sigmoid(x)*(1-y))
dXvec = dYvec * (Yvec + (T(1) / (T(1) + (-Xvec).exp())) * (T(1) - Yvec));
return true;
}
REGISTER_CPU_OPERATOR(
Swish,
UnaryElementwiseOp<
TensorTypes<float>,
CPUContext,
SwishFunctor<CPUContext>>);
REGISTER_CPU_OPERATOR(SwishGradient, SwishGradientOp<CPUContext>);
// Input: X, output: Y
OPERATOR_SCHEMA(Swish)
.NumInputs(1)
.NumOutputs(1)
.IdenticalTypeAndShape()
.SetDoc(R"DOC(
Swish takes one input data (Tensor) and produces one output data
(Tensor) where the swish function, y = x / (1 + exp(-x)), is applied to the
tensor elementwise.
)DOC")
.Input(0, "X", "1D input tensor")
.Output(0, "Y", "1D output tensor");
// Input: X, Y, dY, output: dX
OPERATOR_SCHEMA(SwishGradient)
.NumInputs(3)
.NumOutputs(1)
.AllowInplace({{2, 0}})
.SetDoc(R"DOC(
SwishGradient takes X, Y and dY and uses this to update dX according to the
chain rule and derivatives of the swish function.
)DOC");
namespace {
class GetSwishGradient : public GradientMakerBase {
using GradientMakerBase::GradientMakerBase;
std::vector<OperatorDef> GetGradientDefs() override {
return SingleGradientDef(
"SwishGradient",
"",
std::vector<std::string>{I(0), O(0), GO(0)},
std::vector<std::string>{GI(0)});
}
};
} // namespace
REGISTER_GRADIENT(Swish, GetSwishGradient);
} // namespace caffe2
|
