1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
from typing import Callable, cast
import numpy
from .backends import Ops
from .config import registry
from .types import FloatsXd, Shape
from .util import partial
# TODO: Harmonize naming with Keras, and fill in missing entries
# https://keras.io/initializers/ We should also have He normal/uniform
# and probably lecun normal/uniform.
# Initialize via numpy, before copying to ops. This makes it easier to work with
# the different backends, because the backend won't affect the randomization.
def lecun_normal_init(ops: Ops, shape: Shape) -> FloatsXd:
scale = numpy.sqrt(1.0 / shape[1])
return ops.asarray_f(cast(FloatsXd, numpy.random.normal(0, scale, shape)))
@registry.initializers("lecun_normal_init.v1")
def configure_lecun_normal_init() -> Callable[[Shape], FloatsXd]:
return partial(lecun_normal_init)
def he_normal_init(ops: Ops, shape: Shape) -> FloatsXd:
scale = numpy.sqrt(2.0 / shape[1])
return ops.asarray_f(cast(FloatsXd, numpy.random.normal(0, scale, shape)))
@registry.initializers("he_normal_init.v1")
def configure_he_normal_init() -> Callable[[Shape], FloatsXd]:
return partial(he_normal_init)
def glorot_normal_init(ops: Ops, shape: Shape) -> FloatsXd:
scale = numpy.sqrt(2.0 / (shape[1] + shape[0]))
return ops.asarray_f(cast(FloatsXd, numpy.random.normal(0, scale, shape)))
@registry.initializers("glorot_normal_init.v1")
def configure_glorot_normal_init() -> Callable[[Shape], FloatsXd]:
return partial(glorot_normal_init)
def he_uniform_init(ops: Ops, shape: Shape) -> FloatsXd:
scale = numpy.sqrt(6.0 / shape[1])
return ops.asarray_f(cast(FloatsXd, numpy.random.uniform(-scale, scale, shape)))
@registry.initializers("he_uniform_init.v1")
def configure_he_uniform_init() -> Callable[[Shape], FloatsXd]:
return partial(he_uniform_init)
def lecun_uniform_init(ops: Ops, shape: Shape) -> FloatsXd:
scale = numpy.sqrt(3.0 / shape[1])
return ops.asarray_f(cast(FloatsXd, numpy.random.uniform(-scale, scale, shape)))
@registry.initializers("lecun_uniform_init.v1")
def configure_lecun_uniform_init() -> Callable[[Shape], FloatsXd]:
return partial(lecun_uniform_init)
def glorot_uniform_init(ops: Ops, shape: Shape) -> FloatsXd:
scale = numpy.sqrt(6.0 / (shape[0] + shape[1]))
return ops.asarray_f(cast(FloatsXd, numpy.random.uniform(-scale, scale, shape)))
@registry.initializers("glorot_uniform_init.v1")
def configure_glorot_uniform_init() -> Callable[[Shape], FloatsXd]:
return partial(glorot_uniform_init)
def zero_init(ops: Ops, shape: Shape) -> FloatsXd:
return ops.alloc_f(shape)
@registry.initializers("zero_init.v1")
def configure_zero_init() -> Callable[[FloatsXd], FloatsXd]:
return partial(zero_init)
def uniform_init(
ops: Ops, shape: Shape, *, lo: float = -0.1, hi: float = 0.1
) -> FloatsXd:
values = numpy.random.uniform(lo, hi, shape)
return ops.asarray_f(cast(FloatsXd, values.astype("float32")))
@registry.initializers("uniform_init.v1")
def configure_uniform_init(
*, lo: float = -0.1, hi: float = 0.1
) -> Callable[[FloatsXd], FloatsXd]:
return partial(uniform_init, lo=lo, hi=hi)
def normal_init(ops: Ops, shape: Shape, *, mean: float = 0) -> FloatsXd:
size = int(ops.xp.prod(ops.xp.asarray(shape)))
inits = cast(FloatsXd, numpy.random.normal(scale=mean, size=size).astype("float32"))
inits = ops.reshape_f(inits, shape)
return ops.asarray_f(inits)
@registry.initializers("normal_init.v1")
def configure_normal_init(*, mean: float = 0) -> Callable[[FloatsXd], FloatsXd]:
return partial(normal_init, mean=mean)
__all__ = [
"normal_init",
"uniform_init",
"glorot_uniform_init",
"zero_init",
"lecun_uniform_init",
"he_uniform_init",
"glorot_normal_init",
"he_normal_init",
"lecun_normal_init",
]
|
