1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
|
"""Script to generate baseline values from PyTorch initialization algorithms"""
import sys
import torch
HEADER = """
#include <torch/types.h>
#include <vector>
namespace expected_parameters {
"""
FOOTER = "} // namespace expected_parameters"
PARAMETERS = "inline std::vector<std::vector<torch::Tensor>> {}() {{"
INITIALIZERS = {
"Xavier_Uniform": lambda w: torch.nn.init.xavier_uniform(w),
"Xavier_Normal": lambda w: torch.nn.init.xavier_normal(w),
"Kaiming_Normal": lambda w: torch.nn.init.kaiming_normal(w),
"Kaiming_Uniform": lambda w: torch.nn.init.kaiming_uniform(w),
}
def emit(initializer_parameter_map):
# Don't write generated with an @ in front, else this file is recognized as generated.
print("// @{} from {}".format("generated", __file__))
print(HEADER)
for initializer_name, weights in initializer_parameter_map.items():
print(PARAMETERS.format(initializer_name))
print(" return {")
for sample in weights:
print(" {")
for parameter in sample:
parameter_values = "{{{}}}".format(", ".join(map(str, parameter)))
print(f" torch::tensor({parameter_values}),")
print(" },")
print(" };")
print("}\n")
print(FOOTER)
def run(initializer):
torch.manual_seed(0)
layer1 = torch.nn.Linear(7, 15)
INITIALIZERS[initializer](layer1.weight)
layer2 = torch.nn.Linear(15, 15)
INITIALIZERS[initializer](layer2.weight)
layer3 = torch.nn.Linear(15, 2)
INITIALIZERS[initializer](layer3.weight)
weight1 = layer1.weight.data.numpy()
weight2 = layer2.weight.data.numpy()
weight3 = layer3.weight.data.numpy()
return [weight1, weight2, weight3]
def main():
initializer_parameter_map = {}
for initializer in INITIALIZERS.keys():
sys.stderr.write(f"Evaluating {initializer} ...\n")
initializer_parameter_map[initializer] = run(initializer)
emit(initializer_parameter_map)
if __name__ == "__main__":
main()
|
