1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
# @package adaptive_weight
# Module caffe2.fb.python.layers.adaptive_weight
import numpy as np
from caffe2.python import core, schema
from caffe2.python.layers.layers import ModelLayer
from caffe2.python.regularizer import BoundedGradientProjection, LogBarrier
"""
Implementation of adaptive weighting: https://arxiv.org/pdf/1705.07115.pdf
"""
class AdaptiveWeight(ModelLayer):
def __init__(
self,
model,
input_record,
name="adaptive_weight",
optimizer=None,
weights=None,
enable_diagnose=False,
estimation_method="log_std",
pos_optim_method="log_barrier",
reg_lambda=0.1,
**kwargs
):
super(AdaptiveWeight, self).__init__(model, name, input_record, **kwargs)
self.output_schema = schema.Scalar(
np.float32, self.get_next_blob_reference("adaptive_weight")
)
self.data = self.input_record.field_blobs()
self.num = len(self.data)
self.optimizer = optimizer
if weights is not None:
assert len(weights) == self.num
else:
weights = [1. / self.num for _ in range(self.num)]
assert min(weights) > 0, "initial weights must be positive"
self.weights = np.array(weights).astype(np.float32)
self.estimation_method = str(estimation_method).lower()
# used in positivity-constrained parameterization as when the estimation method
# is inv_var, with optimization method being either log barrier, or grad proj
self.pos_optim_method = str(pos_optim_method).lower()
self.reg_lambda = float(reg_lambda)
self.enable_diagnose = enable_diagnose
self.init_func = getattr(self, self.estimation_method + "_init")
self.weight_func = getattr(self, self.estimation_method + "_weight")
self.reg_func = getattr(self, self.estimation_method + "_reg")
self.init_func()
if self.enable_diagnose:
self.weight_i = [
self.get_next_blob_reference("adaptive_weight_%d" % i)
for i in range(self.num)
]
for i in range(self.num):
self.model.add_ad_hoc_plot_blob(self.weight_i[i])
def concat_data(self, net):
reshaped = [net.NextScopedBlob("reshaped_data_%d" % i) for i in range(self.num)]
# coerce shape for single real values
for i in range(self.num):
net.Reshape(
[self.data[i]],
[reshaped[i], net.NextScopedBlob("new_shape_%d" % i)],
shape=[1],
)
concated = net.NextScopedBlob("concated_data")
net.Concat(
reshaped, [concated, net.NextScopedBlob("concated_new_shape")], axis=0
)
return concated
def log_std_init(self):
"""
mu = 2 log sigma, sigma = standard variance
per task objective:
min 1 / 2 / e^mu X + mu / 2
"""
values = np.log(1. / 2. / self.weights)
initializer = (
"GivenTensorFill",
{"values": values, "dtype": core.DataType.FLOAT},
)
self.mu = self.create_param(
param_name="mu",
shape=[self.num],
initializer=initializer,
optimizer=self.optimizer,
)
def log_std_weight(self, x, net, weight):
"""
min 1 / 2 / e^mu X + mu / 2
"""
mu_neg = net.NextScopedBlob("mu_neg")
net.Negative(self.mu, mu_neg)
mu_neg_exp = net.NextScopedBlob("mu_neg_exp")
net.Exp(mu_neg, mu_neg_exp)
net.Scale(mu_neg_exp, weight, scale=0.5)
def log_std_reg(self, net, reg):
net.Scale(self.mu, reg, scale=0.5)
def inv_var_init(self):
"""
k = 1 / variance
per task objective:
min 1 / 2 * k X - 1 / 2 * log k
"""
values = 2. * self.weights
initializer = (
"GivenTensorFill",
{"values": values, "dtype": core.DataType.FLOAT},
)
if self.pos_optim_method == "log_barrier":
regularizer = LogBarrier(reg_lambda=self.reg_lambda)
elif self.pos_optim_method == "pos_grad_proj":
regularizer = BoundedGradientProjection(lb=0, left_open=True)
else:
raise TypeError(
"unknown positivity optimization method: {}".format(
self.pos_optim_method
)
)
self.k = self.create_param(
param_name="k",
shape=[self.num],
initializer=initializer,
optimizer=self.optimizer,
regularizer=regularizer,
)
def inv_var_weight(self, x, net, weight):
net.Scale(self.k, weight, scale=0.5)
def inv_var_reg(self, net, reg):
log_k = net.NextScopedBlob("log_k")
net.Log(self.k, log_k)
net.Scale(log_k, reg, scale=-0.5)
def _add_ops_impl(self, net, enable_diagnose):
x = self.concat_data(net)
weight = net.NextScopedBlob("weight")
reg = net.NextScopedBlob("reg")
weighted_x = net.NextScopedBlob("weighted_x")
weighted_x_add_reg = net.NextScopedBlob("weighted_x_add_reg")
self.weight_func(x, net, weight)
self.reg_func(net, reg)
net.Mul([weight, x], weighted_x)
net.Add([weighted_x, reg], weighted_x_add_reg)
net.SumElements(weighted_x_add_reg, self.output_schema())
if enable_diagnose:
for i in range(self.num):
net.Slice(weight, self.weight_i[i], starts=[i], ends=[i + 1])
def add_ops(self, net):
self._add_ops_impl(net, self.enable_diagnose)
|
