|
|
|
|
|
|
|
|
|
|
| import unittest |
| from functools import reduce |
| import hypothesis.strategies as st |
| from hypothesis import given, settings |
| import numpy as np |
| from caffe2.proto import caffe2_pb2 |
| from caffe2.python import core, workspace |
| import caffe2.python.hypothesis_test_util as hu |
| import caffe2.python.ideep_test_util as mu |
|
|
|
|
| @unittest.skipIf(not workspace.C.use_mkldnn, "No MKLDNN support.") |
| class FcTest(hu.HypothesisTestCase): |
| @given(n=st.integers(1, 5), m=st.integers(1, 5), |
| k=st.integers(1, 5), **mu.gcs) |
| @settings(deadline=1000) |
| def test_fc_2_dims(self, n, m, k, gc, dc): |
| X = np.random.rand(m, k).astype(np.float32) - 0.5 |
| W = np.random.rand(n, k).astype(np.float32) - 0.5 |
| b = np.random.rand(n).astype(np.float32) - 0.5 |
|
|
| op = core.CreateOperator( |
| 'FC', |
| ['X', 'W', 'b'], |
| ["Y"] |
| ) |
|
|
| self.assertDeviceChecks(dc, op, [X, W, b], [0]) |
|
|
| for i in range(3): |
| self.assertGradientChecks(gc, op, [X, W, b], i, [0]) |
|
|
| @given(n=st.integers(1, 5), |
| m=st.integers(1, 5), |
| c=st.integers(1, 5), |
| h=st.integers(1, 5), |
| w=st.integers(1, 5), |
| axis=st.integers(1, 3), |
| **mu.gcs) |
| def test_fc_with_axis(self, n, m, c, h, w, axis, gc, dc): |
| X = np.random.rand(n, c, h, w).astype(np.float32) - 0.5 |
| k = reduce((lambda x, y: x * y), [n, c, h, w][axis - 4:]) |
| nn = reduce((lambda x, y: x * y), [n, c, h, w][:axis]) |
| W = np.random.rand(m, k).astype(np.float32) - 0.5 |
| b = np.random.rand(m).astype(np.float32) - 0.5 |
| dY = np.random.rand(nn, m).astype(np.float32) - 0.5 |
|
|
| op0 = core.CreateOperator( |
| 'FC', |
| ['X', 'W', 'b'], |
| ["Y"], |
| axis=axis, |
| device_option=dc[0] |
| ) |
|
|
| op0_bw = core.CreateOperator( |
| 'FCGradient', |
| ['X', 'W', 'dY'], |
| ["dW", "db"], |
| axis=axis, |
| device_option=dc[0] |
| ) |
|
|
| workspace.ResetWorkspace() |
| workspace.FeedBlob('X', X, dc[0]) |
| workspace.FeedBlob('W', W, dc[0]) |
| workspace.FeedBlob('b', b, dc[0]) |
| workspace.RunOperatorOnce(op0) |
| Y0 = workspace.FetchBlob('Y') |
|
|
| workspace.FeedBlob('dY', dY, dc[0]) |
| workspace.RunOperatorOnce(op0_bw) |
| dW0 = workspace.FetchBlob('dW') |
| db0 = workspace.FetchBlob('db') |
|
|
| op1 = core.CreateOperator( |
| 'FC', |
| ['X', 'W', 'b'], |
| ["Y"], |
| axis=axis, |
| device_option=dc[1] |
| ) |
|
|
| op1_bw = core.CreateOperator( |
| 'FCGradient', |
| ['X', 'W', 'dY'], |
| ["dW", "db"], |
| axis=axis, |
| device_option=dc[1] |
| ) |
|
|
| workspace.SwitchWorkspace("_device_check_", True) |
| workspace.FeedBlob('X', X, dc[1]) |
| workspace.FeedBlob('W', W, dc[1]) |
| workspace.FeedBlob('b', b, dc[1]) |
| workspace.RunOperatorOnce(op1) |
| Y1 = workspace.FetchBlob('Y') |
|
|
| workspace.FeedBlob('dY', dY, dc[1]) |
| workspace.RunOperatorOnce(op1_bw) |
| dW1 = workspace.FetchBlob('dW') |
| db1 = workspace.FetchBlob('db') |
|
|
| Y0 = Y0.flatten() |
| Y1 = Y1.flatten() |
| if not np.allclose(Y0, Y1, atol=0.01, rtol=0.01): |
| print(Y1) |
| print(Y0) |
| print(np.max(np.abs(Y1 - Y0))) |
| self.assertTrue(False) |
|
|
| dW0 = dW0.flatten() |
| dW1 = dW1.flatten() |
| if not np.allclose(dW0, dW1, atol=0.01, rtol=0.01): |
| print(dW1) |
| print(dW0) |
| print(np.max(np.abs(dW1 - dW0))) |
| self.assertTrue(False) |
|
|
| db0 = db0.flatten() |
| db1 = db1.flatten() |
| if not np.allclose(db0, db1, atol=0.01, rtol=0.01): |
| print(db1) |
| print(db0) |
| print(np.max(np.abs(db1 - db0))) |
| self.assertTrue(False) |
|
|
| @given(n=st.integers(1, 5), |
| o=st.integers(1, 5), |
| i=st.integers(1, 5), |
| h=st.integers(1, 5), |
| w=st.integers(1, 5), |
| axis_w=st.integers(1, 3), |
| **mu.gcs) |
| @settings(deadline=1000) |
| def test_fc_with_axis_w(self, n, o, i, h, w, axis_w, gc, dc): |
| W = np.random.rand(o, i, h, w).astype(np.float32) - 0.5 |
| k = reduce((lambda x, y: x * y), [o, i, h, w][axis_w - 4:]) |
| m = reduce((lambda x, y: x * y), [o, i, h, w][:axis_w]) |
| X = np.random.rand(n, k).astype(np.float32) - 0.5 |
| b = np.random.rand(m).astype(np.float32) - 0.5 |
| dY = np.random.rand(n, m).astype(np.float32) - 0.5 |
|
|
| op0 = core.CreateOperator( |
| 'FC', |
| ['X', 'W', 'b'], |
| ["Y"], |
| axis_w=axis_w, |
| device_option=dc[0] |
| ) |
|
|
| op0_bw = core.CreateOperator( |
| 'FCGradient', |
| ['X', 'W', 'dY'], |
| ["dW", "db"], |
| axis_w=axis_w, |
| device_option=dc[0] |
| ) |
|
|
| workspace.ResetWorkspace() |
| workspace.FeedBlob('X', X, dc[0]) |
| workspace.FeedBlob('W', W, dc[0]) |
| workspace.FeedBlob('b', b, dc[0]) |
| workspace.RunOperatorOnce(op0) |
| Y0 = workspace.FetchBlob('Y') |
|
|
| workspace.FeedBlob('dY', dY, dc[0]) |
| workspace.RunOperatorOnce(op0_bw) |
| dW0 = workspace.FetchBlob('dW') |
| db0 = workspace.FetchBlob('db') |
|
|
| op1 = core.CreateOperator( |
| 'FC', |
| ['X', 'W', 'b'], |
| ["Y"], |
| axis_w=axis_w, |
| device_option=dc[1] |
| ) |
|
|
| op1_bw = core.CreateOperator( |
| 'FCGradient', |
| ['X', 'W', 'dY'], |
| ["dW", "db"], |
| axis_w=axis_w, |
| device_option=dc[1] |
| ) |
|
|
| workspace.SwitchWorkspace("_device_check_", True) |
| workspace.FeedBlob('X', X, dc[1]) |
| workspace.FeedBlob('W', W, dc[1]) |
| workspace.FeedBlob('b', b, dc[1]) |
| workspace.RunOperatorOnce(op1) |
| Y1 = workspace.FetchBlob('Y') |
|
|
| workspace.FeedBlob('dY', dY, dc[1]) |
| workspace.RunOperatorOnce(op1_bw) |
| dW1 = workspace.FetchBlob('dW') |
| db1 = workspace.FetchBlob('db') |
|
|
| Y0 = Y0.flatten() |
| Y1 = Y1.flatten() |
| if not np.allclose(Y0, Y1, atol=0.01, rtol=0.01): |
| print(Y1) |
| print(Y0) |
| print(np.max(np.abs(Y1 - Y0))) |
| self.assertTrue(False) |
|
|
| dW0 = dW0.flatten() |
| dW1 = dW1.flatten() |
| if not np.allclose(dW0, dW1, atol=0.01, rtol=0.01): |
| print(dW1) |
| print(dW0) |
| print(np.max(np.abs(dW1 - dW0))) |
| self.assertTrue(False) |
|
|
| db0 = db0.flatten() |
| db1 = db1.flatten() |
| if not np.allclose(db0, db1, atol=0.01, rtol=0.01): |
| print(db1) |
| print(db0) |
| print(np.max(np.abs(db1 - db0))) |
| self.assertTrue(False) |
|
|
| @given(n=st.integers(1, 5), m=st.integers(1, 5), |
| k=st.integers(1, 5), **mu.gcs) |
| @settings(deadline=10000) |
| def test_fc_4_dims_src(self, n, m, k, gc, dc): |
| X = np.random.rand(m, k, m, m).astype(np.float32) - 0.5 |
| W = np.random.rand(n, k * m * m).astype(np.float32) - 0.5 |
| b = np.random.rand(n).astype(np.float32) - 0.5 |
|
|
| op = core.CreateOperator( |
| 'FC', |
| ['X', 'W', 'b'], |
| ["Y"] |
| ) |
|
|
| self.assertDeviceChecks(dc, op, [X, W, b], [0]) |
|
|
| for i in range(3): |
| self.assertGradientChecks(gc, op, [X, W, b], i, [0]) |
|
|
| @given(n=st.integers(1, 5), m=st.integers(1, 5), |
| k=st.integers(1, 5), **mu.gcs) |
| @settings(deadline=10000) |
| def test_fc_4_dims(self, n, m, k, gc, dc): |
| X = np.random.rand(m, k, m, m).astype(np.float32) - 0.5 |
| W = np.random.rand(n, k, m, m).astype(np.float32) - 0.5 |
| b = np.random.rand(n).astype(np.float32) - 0.5 |
|
|
| op = core.CreateOperator( |
| 'FC', |
| ['X', 'W', 'b'], |
| ["Y"] |
| ) |
|
|
| self.assertDeviceChecks(dc, op, [X, W, b], [0]) |
|
|
| for i in range(3): |
| self.assertGradientChecks(gc, op, [X, W, b], i, [0]) |
|
|
| @given(n=st.integers(2, 5), m=st.integers(2, 5), |
| k=st.integers(2, 5), **mu.gcs_cpu_ideep) |
| def test_int8_fc_4_dims(self, n, m, k, gc, dc): |
| X = np.random.rand(m, k, m, m).astype(np.float32) - 0.5 |
| w = np.random.rand(n, k, m, m).astype(np.float32) - 0.5 |
| b = np.random.rand(n).astype(np.float32) - 0.5 |
|
|
| fc_fp32 = core.CreateOperator( |
| 'FC', |
| ['X', 'w', 'b'], |
| ["Y"] |
| ) |
|
|
| old_ws_name = workspace.CurrentWorkspace() |
| workspace.SwitchWorkspace("_device_check_", True) |
|
|
| workspace.FeedBlob('X', X, dc[0]) |
| workspace.FeedBlob('w', w, dc[0]) |
| workspace.FeedBlob('b', b, dc[0]) |
| workspace.RunOperatorOnce(fc_fp32) |
| Y = workspace.FetchBlob('Y') |
|
|
| workspace.ResetWorkspace() |
|
|
| Y_absmax = np.array([np.absolute(Y).max()]).astype(np.float32) |
| if Y.min() >= 0: |
| Y_scale = Y_absmax / 0xFF |
| Y_zero_point = 0 |
| else: |
| Y_scale = Y_absmax / 0x7F |
| Y_zero_point = 128 |
|
|
| X_absmax = np.array([np.absolute(X).max()]).astype(np.float32) |
| if X.min() >= 0: |
| X_scale = X_absmax / 0xFF |
| X_zero_point = 0 |
| else: |
| X_scale = X_absmax / 0x7F |
| X_zero_point = 128 |
|
|
| w_absmax = np.array([np.absolute(w[i, ...]).max() for i in range(w.shape[0])]).astype(np.float32) |
| w_scale = w_absmax / 0x7F |
| w_zero_point = 128 |
| w = np.transpose(w, (0, 2, 3, 1)).astype(np.float32) |
| w_bytes = np.rint([w[i, ...] / w_scale[i] for i in range(w.shape[0])]).astype(np.int8) + w_zero_point |
|
|
| w_filler = core.CreateOperator( |
| "Int8GivenTensorFill", |
| [], ["wi"], |
| shape=w.shape, |
| values=w_bytes.astype(np.uint8).tobytes(), |
| Y_zero_point=w_zero_point, |
| Y_scales=w_scale, |
| device_option=dc[1], |
| ) |
|
|
| b_scale = w_scale * X_scale |
| b_zero_point = 0 |
| b_bytes = np.rint([b[i] / b_scale[i] for i in range(b.shape[0])]).astype(np.int32) |
| b_filler = core.CreateOperator( |
| "Int8GivenIntTensorFill", |
| [], ["bi"], |
| shape=b.shape, |
| values=b_bytes, |
| Y_zero_point=b_zero_point, |
| Y_scales=b_scale, |
| device_option=dc[1], |
| ) |
|
|
| sw2nhwc = core.CreateOperator( |
| "NCHW2NHWC", |
| ["Xi"], |
| ["Xi_nhwc"], |
| device_option=dc[1] |
| ) |
|
|
| quantize_X = core.CreateOperator( |
| "Int8Quantize", |
| ["Xi_nhwc"], |
| ["Xi_quantized"], |
| engine="DNNLOWP", |
| device_option=dc[1], |
| Y_zero_point=X_zero_point, |
| Y_scale=X_scale[0], |
| ) |
|
|
| fc = core.CreateOperator( |
| 'Int8FC', |
| ['Xi_quantized', 'wi', 'bi'], |
| ["Y_out"], |
| engine="DNNLOWP", |
| device_option=dc[1], |
| Y_zero_point=Y_zero_point, |
| Y_scale=Y_scale[0], |
| ) |
|
|
| net = caffe2_pb2.NetDef() |
| net.op.extend([w_filler, b_filler, sw2nhwc, quantize_X, fc]) |
|
|
| workspace.FeedBlob("Xi", X, dc[1]) |
| workspace.RunNetOnce(net) |
| Y_out = workspace.FetchBlob("Y_out") |
|
|
| MSE = np.square(np.subtract(Y, Y_out)).mean() |
| if MSE > 0.005: |
| print(Y.flatten()) |
| print(Y_out.flatten()) |
| print(np.max(np.abs(Y_out - Y))) |
| print("MSE", MSE) |
| self.assertTrue(False) |
|
|
| workspace.SwitchWorkspace(old_ws_name) |
|
|
| if __name__ == "__main__": |
| unittest.main() |
|
|
