|
|
|
|
|
|
|
|
|
|
| from caffe2.python import schema |
| from caffe2.python.layers.layers import ModelLayer |
| import numpy as np |
|
|
|
|
| class ArcCosineFeatureMap(ModelLayer): |
| """ |
| A general version of the arc-cosine kernel feature map (s = 1 restores |
| the original arc-cosine kernel feature map). |
| |
| Applies H(x) * x^s, where H is the Heaviside step function and x is the |
| input after applying FC (such that x = w * x_orig + b). |
| |
| For more information, see the original paper: |
| http://cseweb.ucsd.edu/~saul/papers/nips09_kernel.pdf |
| |
| Inputs : |
| output_dims -- dimensions of the output vector |
| s -- degree to raise transformed features |
| scale -- amount to scale the standard deviation |
| weight_init -- initialization distribution for weight parameter |
| bias_init -- initialization distribution for bias pararmeter |
| weight_optim -- optimizer for weight params; None for random features |
| bias_optim -- optimizer for bias param; None for random features |
| set_weight_as_global_constant -- if True, initialized random parameters |
| will be constant across all distributed |
| instances of the layer |
| initialize_output_schema -- if True, initialize output schema as Scalar |
| from Arc Cosine; else output schema is None |
| """ |
| def __init__( |
| self, |
| model, |
| input_record, |
| output_dims, |
| s=1, |
| scale=1.0, |
| weight_init=None, |
| bias_init=None, |
| weight_optim=None, |
| bias_optim=None, |
| set_weight_as_global_constant=False, |
| initialize_output_schema=True, |
| name='arc_cosine_feature_map', |
| **kwargs): |
|
|
| super(ArcCosineFeatureMap, self).__init__(model, name, input_record, |
| **kwargs) |
| assert isinstance(input_record, schema.Scalar), "Incorrect input type" |
| self.params = [] |
| self.model = model |
| self.set_weight_as_global_constant = set_weight_as_global_constant |
|
|
| self.input_dims = input_record.field_type().shape[0] |
| assert self.input_dims >= 1, "Expected input dimensions >= 1, got %s" \ |
| % self.input_dims |
|
|
| if initialize_output_schema: |
| self.output_schema = schema.Scalar( |
| (np.float32, (output_dims, )), |
| model.net.NextScopedBlob(name + '_output') |
| ) |
|
|
| self.output_dims = output_dims |
| assert self.output_dims >= 1, "Expected output dimensions >= 1, got %s" \ |
| % self.output_dims |
| self.s = s |
| assert (self.s >= 0), "Expected s >= 0, got %s" % self.s |
| assert isinstance(self.s, int), "Expected s to be type int, got type %s" \ |
| % type(self.s) |
|
|
| assert (scale > 0.0), "Expected scale > 0, got %s" % scale |
| self.stddev = scale * np.sqrt(1.0 / self.input_dims) |
|
|
| |
| |
| if set_weight_as_global_constant: |
| w_init = np.random.normal(scale=self.stddev, |
| size=(self.output_dims, self.input_dims)) |
| b_init = np.random.uniform(low=-0.5 * self.stddev, |
| high=0.5 * self.stddev, |
| size=self.output_dims) |
| self.random_w = self.model.add_global_constant( |
| name=self.name + "_fixed_rand_W", |
| array=w_init |
| ) |
| self.random_b = self.model.add_global_constant( |
| name=self.name + "_fixed_rand_b", |
| array=b_init |
| ) |
| else: |
| (self.random_w, self.random_b) = self._initialize_params( |
| 'random_w', |
| 'random_b', |
| w_init=weight_init, |
| b_init=bias_init, |
| w_optim=weight_optim, |
| b_optim=bias_optim |
| ) |
|
|
| def _initialize_params(self, w_name, b_name, w_init=None, b_init=None, |
| w_optim=None, b_optim=None): |
| """ |
| Initializes the Layer Parameters for weight and bias terms for features |
| |
| Inputs : |
| w_blob -- blob to contain w values |
| b_blob -- blob to contain b values |
| w_init -- initialization distribution for weight parameter |
| b_init -- initialization distribution for bias parameter |
| w_optim -- optimizer to use for w; if None, then will use no optimizer |
| b_optim -- optimizer to user for b; if None, then will use no optimizer |
| """ |
|
|
| w_init = w_init if w_init else ( |
| 'GaussianFill', {'mean': 0.0, 'std': self.stddev} |
| ) |
| w_optim = w_optim if w_optim else self.model.NoOptim |
|
|
| b_init = b_init if b_init else ( |
| 'UniformFill', {'min': -0.5 * self.stddev, 'max': 0.5 * self.stddev} |
| ) |
| b_optim = b_optim if b_optim else self.model.NoOptim |
|
|
| w_param = self.create_param(param_name=w_name, |
| shape=(self.output_dims, self.input_dims), |
| initializer=w_init, |
| optimizer=w_optim) |
|
|
| b_param = self.create_param(param_name=b_name, |
| shape=[self.output_dims], |
| initializer=b_init, |
| optimizer=b_optim) |
|
|
| return [w_param, b_param] |
|
|
| def _heaviside_with_power(self, net, input_features, output_blob, s): |
| """ |
| Applies Heaviside step function and Relu / exponentiation to features |
| depending on the value of s. |
| |
| Inputs: |
| net -- net with operators |
| input_features -- features to processes |
| output_blob -- output blob reference |
| s -- degree to raise the transformed features |
| """ |
| if s == 0: |
| softsign_features = net.Softsign([input_features], |
| net.NextScopedBlob('softsign')) |
| return net.Relu(softsign_features, output_blob) |
| elif s == 1: |
| return net.Relu([input_features], |
| output_blob) |
| else: |
| relu_features = net.Relu([input_features], |
| net.NextScopedBlob('relu_rand')) |
| pow_features = net.Pow([input_features], |
| net.NextScopedBlob('pow_rand'), |
| exponent=float(s - 1)) |
| return net.Mul([relu_features, pow_features], |
| output_blob) |
|
|
| def add_ops(self, net): |
| input_blob = self.input_record.field_blobs() |
|
|
| |
| random_features = net.FC(input_blob + [self.random_w, self.random_b], |
| net.NextScopedBlob('random_features')) |
| |
| self._heaviside_with_power(net, |
| random_features, |
| self.output_schema.field_blobs(), |
| self.s) |
|
|
