1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
from caffe2.proto import caffe2_pb2
from caffe2.python import core
from hypothesis import assume, given, settings, HealthCheck
import caffe2.python.hypothesis_test_util as hu
import caffe2.python.serialized_test.serialized_test_util as serial
import hypothesis.strategies as st
import numpy as np
import unittest
class TestFcOperator(serial.SerializedTestCase):
def _run_test(self, n, m, k, transposed, multi_dim, dtype, engine, gc, dc):
if dtype == np.float16:
# fp16 only supported with CUDA/HIP
assume(core.IsGPUDeviceType(gc.device_type))
dc = [d for d in dc if core.IsGPUDeviceType(d.device_type)]
if engine == 'TENSORCORE':
# TensorCore only makes sense with CUDA
assume(gc.device_type == caffe2_pb2.CUDA)
# ensures TensorCore kernels can be called
m *= 8
k *= 8
n *= 8
X = np.random.rand(m, k).astype(dtype) - 0.5
if multi_dim:
if transposed:
W = np.random.rand(k, n, 1, 1).astype(dtype) - 0.5
else:
W = np.random.rand(n, k, 1, 1).astype(dtype) - 0.5
else:
if transposed:
W = np.random.rand(k, n).astype(dtype) - 0.5
else:
W = np.random.rand(n, k).astype(dtype) - 0.5
b = np.random.rand(n).astype(dtype) - 0.5
def fc_op(X, W, b):
return [np.dot(X, W.reshape(n, k).transpose()) + b.reshape(n)]
def fc_transposed_op(X, W, b):
return [np.dot(X, W.reshape(k, n)) + b.reshape(n)]
op = core.CreateOperator(
'FCTransposed' if transposed else 'FC',
['X', 'W', 'b'],
'out',
engine=engine,
)
if dtype == np.float16 and core.IsGPUDeviceType(gc.device_type):
a = caffe2_pb2.Argument()
a.i = 1
a.name = "float16_compute"
op.arg.extend([a])
# Check against numpy reference
# ReferenceChecks is flaky, Relaxing to 1e-3.
threshold = 1e-3
self.assertReferenceChecks(
device_option=gc,
op=op,
inputs=[X, W, b],
reference=fc_transposed_op if transposed else fc_op,
threshold=threshold
)
# Check over multiple devices
self.assertDeviceChecks(dc, op, [X, W, b], [0])
# Gradient checks
threshold = 0.5 if dtype == np.float16 else 0.005
stepsize = 0.5 if dtype == np.float16 else 0.05
for i in range(3):
self.assertGradientChecks(gc, op, [X, W, b], i, [0],
threshold=threshold, stepsize=stepsize)
@settings(max_examples=50, suppress_health_check=[HealthCheck.filter_too_much])
@serial.given(n=st.integers(1, 5),
m=st.integers(0, 5),
k=st.integers(1, 5),
multi_dim=st.sampled_from([True, False]),
dtype=st.sampled_from([np.float32, np.float16]),
engine=st.sampled_from(['', 'TENSORCORE']),
**hu.gcs)
def test_fc(self, **kwargs):
self._run_test(transposed=False, **kwargs)
@settings(max_examples=50, suppress_health_check=[HealthCheck.filter_too_much])
@given(n=st.integers(1, 5),
m=st.integers(0, 5),
k=st.integers(1, 5),
multi_dim=st.sampled_from([True, False]),
dtype=st.sampled_from([np.float32, np.float16]),
engine=st.sampled_from(['', 'TENSORCORE']),
**hu.gcs)
def test_fc_transposed(self, **kwargs):
self._run_test(transposed=True, **kwargs)
if __name__ == "__main__":
import unittest
unittest.main()
|
