From 341cf1c22527f880d3efdabe5762069b63b222c5 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sun, 12 May 2019 22:24:05 +0800 Subject: [PATCH 01/19] (non)trainable weights, layer all_layers --- tensorlayer/files/utils.py | 10 +-- .../layers/convolution/separable_conv.py | 4 +- .../layers/convolution/simplified_deconv.py | 4 +- tensorlayer/layers/core.py | 61 +++++++++++++------ tensorlayer/layers/normalization.py | 12 ++-- tensorlayer/models/core.py | 44 ++++++++++--- tensorlayer/models/mobilenetv1.py | 2 +- tensorlayer/models/squeezenetv1.py | 2 +- tensorlayer/models/vgg.py | 4 +- tests/layers/test_layers_convolution.py | 42 ++++++------- .../test_layers_core_basedense_dropout.py | 14 ++--- .../test_layers_deformable_convolution.py | 4 +- tests/models/test_model_core.py | 2 +- tests/models/test_model_save.py | 8 +-- tests/test_initializers.py | 14 ++--- 15 files changed, 144 insertions(+), 83 deletions(-) diff --git a/tensorlayer/files/utils.py b/tensorlayer/files/utils.py index 8cc718f5b..e83a246bd 100644 --- a/tensorlayer/files/utils.py +++ b/tensorlayer/files/utils.py @@ -2549,9 +2549,9 @@ def _save_weights_to_hdf5_group(f, layers): elif isinstance(layer, tl.layers.LayerList): _save_weights_to_hdf5_group(g, layer.layers) elif isinstance(layer, tl.layers.Layer): - if layer.weights is not None: - weight_values = tf_variables_to_numpy(layer.weights) - weight_names = [w.name.encode('utf8') for w in layer.weights] + if layer.all_weights is not None: + weight_values = tf_variables_to_numpy(layer.all_weights) + weight_names = [w.name.encode('utf8') for w in layer.all_weights] else: weight_values = [] weight_names = [] @@ -2593,7 +2593,7 @@ def _load_weights_from_hdf5_group_in_order(f, layers): elif isinstance(layer, tl.layers.Layer): weight_names = [n.decode('utf8') for n in g.attrs['weight_names']] for iid, w_name in enumerate(weight_names): - assign_tf_variable(layer.weights[iid], np.asarray(g[w_name])) + assign_tf_variable(layer.all_weights[iid], np.asarray(g[w_name])) else: raise Exception("Only layer or model can be saved into hdf5.") if idx == len(layers) - 1: @@ -2639,7 +2639,7 @@ def _load_weights_from_hdf5_group(f, layers, skip=False): elif isinstance(layer, tl.layers.Layer): weight_names = [n.decode('utf8') for n in g.attrs['weight_names']] for iid, w_name in enumerate(weight_names): - assign_tf_variable(layer.weights[iid], np.asarray(g[w_name])) + assign_tf_variable(layer.all_weights[iid], np.asarray(g[w_name])) else: raise Exception("Only layer or model can be saved into hdf5.") diff --git a/tensorlayer/layers/convolution/separable_conv.py b/tensorlayer/layers/convolution/separable_conv.py index a19cabbcc..d2fbf3515 100644 --- a/tensorlayer/layers/convolution/separable_conv.py +++ b/tensorlayer/layers/convolution/separable_conv.py @@ -156,7 +156,7 @@ def build(self, inputs_shape): ) # initialize weights outputs_shape = _out.shape # self._add_weights(self.layer.weights) - self._weights = self.layer.weights + self._weights = self.layer.all_weights def forward(self, inputs): outputs = self.layer(inputs) @@ -302,7 +302,7 @@ def build(self, inputs_shape): tf.convert_to_tensor(np.random.uniform(size=list(inputs_shape)), dtype=np.float) ) # initialize weights outputs_shape = _out.shape - self._weights = self.layer.weights + self._weights = self.layer.all_weights def forward(self, inputs): outputs = self.layer(inputs) diff --git a/tensorlayer/layers/convolution/simplified_deconv.py b/tensorlayer/layers/convolution/simplified_deconv.py index 0dc204caf..7c147b729 100644 --- a/tensorlayer/layers/convolution/simplified_deconv.py +++ b/tensorlayer/layers/convolution/simplified_deconv.py @@ -141,7 +141,7 @@ def build(self, inputs_shape): tf.convert_to_tensor(np.random.uniform(size=inputs_shape), dtype=np.float32) ) #np.random.uniform([1] + list(inputs_shape))) # initialize weights outputs_shape = _out.shape - self._weights = self.layer.weights + self._weights = self.layer.all_weights def forward(self, inputs): outputs = self.layer(inputs) @@ -264,7 +264,7 @@ def build(self, inputs_shape): ) #self.layer(np.random.uniform([1] + list(inputs_shape))) # initialize weights outputs_shape = _out.shape # self._add_weights(self.layer.weights) - self._weights = self.layer.weights + self._weights = self.layer.all_weights def forward(self, inputs): outputs = self.layer(inputs) diff --git a/tensorlayer/layers/core.py b/tensorlayer/layers/core.py index c5753c9bc..ce98f156c 100644 --- a/tensorlayer/layers/core.py +++ b/tensorlayer/layers/core.py @@ -34,8 +34,12 @@ class Layer(object): Initializing the Layer. __call__() (1) Building the Layer if necessary. (2) Forwarding the computation. - weights() + all_weights() + Return a list of Tensor which are all weights of this Layer. + trainable_weights() Return a list of Tensor which are all trainable weights of this Layer. + nontrainable_weights() + Return a list of Tensor which are all nontrainable weights of this Layer. build() Abstract method. Build the Layer. All trainable weights should be defined in this function. forward() @@ -89,7 +93,9 @@ def __init__(self, name=None, *args, **kwargs): self._nodes_fixed = False # Layer weight state - self._weights = None + self._all_weights = None + self._trainable_weights = None + self._nontrainable_weights = None # Layer training state self.is_train = True @@ -136,8 +142,24 @@ def config(self): return _config @property - def weights(self): - return self._weights + def all_weights(self): + if self._all_weights is not None and len(self._all_weights) > 0: + pass + else: + self._all_weights = list() + if self._trainable_weights is not None: + self._all_weights.extend(self._trainable_weights) + if self._nontrainable_weights is not None: + self._all_weights.extend(self._nontrainable_weights) + return self._all_weights + + @property + def trainable_weights(self): + return self._trainable_weights + + @property + def nontrainable_weights(self): + return self._nontrainable_weights def __call__(self, inputs, *args, **kwargs): """ @@ -218,12 +240,17 @@ def _fix_nodes_for_layers(self): """ fix LayerNodes to stop growing for this layer""" self._nodes_fixed = True - def _get_weights(self, var_name, shape, init=tl.initializers.random_normal()): + def _get_weights(self, var_name, shape, init=tl.initializers.random_normal(), trainable=True): """ Get trainable variables. """ weight = get_variable_with_initializer(scope_name=self.name, var_name=var_name, shape=shape, init=init) - if self._weights is None: - self._weights = list() - self._weights.append(weight) # Add into the weight collection + if trainable is True: + if self._trainable_weights is None: + self._trainable_weights = list() + self._trainable_weights.append(weight) + else: + if self._nontrainable_weights is None: + self._nontrainable_weights = list() + self._nontrainable_weights.append(weight) return weight @abstractmethod @@ -407,7 +434,7 @@ def __init__(self, model, name=None): self._built = True # Layer weight state - self._weights = model.weights + self._all_weights = model.all_weights # Layer training state self.is_train = True @@ -497,12 +524,12 @@ def __init__(self, layers, name=None): for layer in self.layers: if layer._built is False: is_built = False - if layer._built and layer.weights is not None: + if layer._built and layer.all_weights is not None: # some layers in the list passed in have already been built # e.g. using input shape to construct layers in dynamic eager - if self._weights is None: - self._weights = list() - self._weights.extend(layer.weights) + if self._all_weights is None: + self._all_weights = list() + self._all_weights.extend(layer.all_weights) if is_built: self._built = True @@ -550,10 +577,10 @@ def build(self, inputs_shape): is_build = layer._built out_tensor = layer(in_tensor) # nlayer = layer(in_layer) - if is_build == False and layer.weights is not None: - if self._weights is None: - self._weights = list() - self._weights.extend(layer.weights) + if is_build is False and layer.all_weights is not None: + if self._all_weights is None: + self._all_weights = list() + self._all_weights.extend(layer.all_weights) layer._built = True in_tensor = out_tensor # in_layer = nlayer diff --git a/tensorlayer/layers/normalization.py b/tensorlayer/layers/normalization.py index 572936228..5b03f5735 100644 --- a/tensorlayer/layers/normalization.py +++ b/tensorlayer/layers/normalization.py @@ -258,8 +258,8 @@ def build(self, inputs_shape): if self.gamma_init: self.gamma = self._get_weights("gamma", shape=params_shape, init=self.gamma_init) - self.moving_mean = self._get_weights("moving_mean", shape=params_shape, init=self.moving_mean_init) - self.moving_var = self._get_weights("moving_var", shape=params_shape, init=self.moving_var_init) + self.moving_mean = self._get_weights("moving_mean", shape=params_shape, init=self.moving_mean_init, trainable=False) + self.moving_var = self._get_weights("moving_var", shape=params_shape, init=self.moving_var_init, trainable=False) def forward(self, inputs): mean, var = tf.nn.moments(inputs, self.axes, keepdims=True) @@ -268,8 +268,12 @@ def forward(self, inputs): self.moving_mean = moving_averages.assign_moving_average( self.moving_mean, mean, self.decay, zero_debias=False ) - self.moving_var = moving_averages.assign_moving_average(self.moving_var, var, self.decay, zero_debias=False) - outputs = batch_normalization(inputs, mean, var, self.beta, self.gamma, self.epsilon, self.data_format) + self.moving_var = moving_averages.assign_moving_average( + self.moving_var, var, self.decay, zero_debias=False + ) + outputs = batch_normalization( + inputs, mean, var, self.beta, self.gamma, self.epsilon, self.data_format + ) else: outputs = batch_normalization( inputs, self.moving_mean, self.moving_var, self.beta, self.gamma, self.epsilon, self.data_format diff --git a/tensorlayer/models/core.py b/tensorlayer/models/core.py index b257c2c26..1de703b7c 100644 --- a/tensorlayer/models/core.py +++ b/tensorlayer/models/core.py @@ -176,7 +176,9 @@ def __init__(self, inputs=None, outputs=None, name=None): self.is_train = None # Model weights - self._weights = None + self._all_weights = None + self._trainable_weights = None + self._nontrainable_weights = None # Model args of all layers, ordered by all_layers self._config = None @@ -380,19 +382,47 @@ def all_layers(self): local_layer_name_dict.add(layer.name) return self._all_layers + @property + def trainable_weights(self): + """Return trainable weights of this network in a list.""" + if self._trainable_weights is not None and len(self._trainable_weights) > 0: + # self._trainable_weights already extracted, so do nothing + pass + else: + self._trainable_weights = [] + for layer in self.all_layers: + if layer.trainable_weights is not None: + self._trainable_weights.extend(layer.trainable_weights) + + return self._trainable_weights + + @property + def nontrainable_weights(self): + """Return nontrainable weights of this network in a list.""" + if self._nontrainable_weights is not None and len(self._nontrainable_weights) > 0: + # self._nontrainable_weights already extracted, so do nothing + pass + else: + self._nontrainable_weights = [] + for layer in self.all_layers: + if layer.nontrainable_weights is not None: + self._nontrainable_weights.extend(layer.nontrainable_weights) + + return self._nontrainable_weights + @property def weights(self): """Return all weights of this network in a list.""" - if self._weights is not None and len(self._weights) > 0: - # self._weights already extracted, so do nothing + if self._all_weights is not None and len(self._all_weights) > 0: + # self._all_weights already extracted, so do nothing pass else: - self._weights = [] + self._all_weights = [] for layer in self.all_layers: - if layer.weights is not None: - self._weights.extend(layer.weights) + if layer.all_weights is not None: + self._all_weights.extend(layer.all_weights) - return self._weights + return self._all_weights @property def config(self): diff --git a/tensorlayer/models/mobilenetv1.py b/tensorlayer/models/mobilenetv1.py index 9c8f7ac51..0454826a2 100644 --- a/tensorlayer/models/mobilenetv1.py +++ b/tensorlayer/models/mobilenetv1.py @@ -84,7 +84,7 @@ def MobileNetV1(pretrained=False, end_with='out', name=None): >>> nn = Flatten(name='flatten')(nn) >>> model = tl.models.Model(inputs=ni, outputs=nn) >>> # train your own classifier (only update the last layer) - >>> train_params = model.get_layer('out').weights + >>> train_params = model.get_layer('out').trainable_weights Returns ------- diff --git a/tensorlayer/models/squeezenetv1.py b/tensorlayer/models/squeezenetv1.py index b1a5cecc0..e94aeab90 100644 --- a/tensorlayer/models/squeezenetv1.py +++ b/tensorlayer/models/squeezenetv1.py @@ -75,7 +75,7 @@ def SqueezeNetV1(pretrained=False, end_with='out', name=None): >>> nn = GlobalMeanPool2d(name='globalmeanpool')(nn) >>> model = tl.models.Model(inputs=ni, outputs=nn) >>> # train your own classifier (only update the last layer) - >>> train_params = model.get_layer('conv10').weights + >>> train_params = model.get_layer('conv10').trainable_weights Returns ------- diff --git a/tensorlayer/models/vgg.py b/tensorlayer/models/vgg.py index 7c4bf201e..858784ec3 100644 --- a/tensorlayer/models/vgg.py +++ b/tensorlayer/models/vgg.py @@ -231,7 +231,7 @@ def vgg16(pretrained=False, end_with='outputs', mode='dynamic', name=None): >>> nn = tl.layers.Dense(n_units=100, name='out')(nn) >>> model = tl.models.Model(inputs=ni, outputs=nn) >>> # train your own classifier (only update the last layer) - >>> train_params = model.get_layer('out').weights + >>> train_params = model.get_layer('out').trainable_weights Reuse model @@ -293,7 +293,7 @@ def vgg19(pretrained=False, end_with='outputs', mode='dynamic', name=None): >>> nn = tl.layers.Dense(n_units=100, name='out')(nn) >>> model = tl.models.Model(inputs=ni, outputs=nn) >>> # train your own classifier (only update the last layer) - >>> train_params = model.get_layer('out').weights + >>> train_params = model.get_layer('out').trainable_weights Reuse model diff --git a/tests/layers/test_layers_convolution.py b/tests/layers/test_layers_convolution.py index 61beb733c..997e8cf57 100644 --- a/tests/layers/test_layers_convolution.py +++ b/tests/layers/test_layers_convolution.py @@ -51,7 +51,7 @@ def test_layer_n1(self): # self.assertEqual(len(self.n1.all_layers), 2) # self.assertEqual(len(self.n1.all_params), 2) # self.assertEqual(self.n1.count_params(), 192) - self.assertEqual(len(self.n1._info[0].layer.weights), 2) + self.assertEqual(len(self.n1._info[0].layer.all_weights), 2) self.assertEqual(self.n1.get_shape().as_list()[1:], [50, 32]) def test_layer_n2(self): @@ -59,7 +59,7 @@ def test_layer_n2(self): # self.assertEqual(len(self.n2.all_layers), 3) # self.assertEqual(len(self.n2.all_params), 4) # self.assertEqual(self.n2.count_params(), 5344) - self.assertEqual(len(self.n2._info[0].layer.weights), 2) + self.assertEqual(len(self.n2._info[0].layer.all_weights), 2) self.assertEqual(self.n2.get_shape().as_list()[1:], [25, 32]) def test_layer_n3(self): @@ -67,7 +67,7 @@ def test_layer_n3(self): # self.assertEqual(len(self.n2.all_layers), 3) # self.assertEqual(len(self.n2.all_params), 4) # self.assertEqual(self.n2.count_params(), 5344) - self.assertEqual(len(self.n3._info[0].layer.weights), 2) + self.assertEqual(len(self.n3._info[0].layer.all_weights), 2) self.assertEqual(self.n3.get_shape().as_list()[1:], [50, 64]) def test_layer_n4(self): @@ -75,7 +75,7 @@ def test_layer_n4(self): # self.assertEqual(len(self.n2.all_layers), 3) # self.assertEqual(len(self.n2.all_params), 4) # self.assertEqual(self.n2.count_params(), 5344) - self.assertEqual(len(self.n4._info[0].layer.weights), 3) + self.assertEqual(len(self.n4._info[0].layer.all_weights), 3) self.assertEqual(self.n4.get_shape().as_list()[1:], [25, 32]) def test_layer_n5(self): @@ -223,7 +223,7 @@ def test_layer_n1(self): # self.assertEqual(len(self.n1.all_layers), 2) # self.assertEqual(len(self.n1.all_params), 2) # self.assertEqual(self.n1.count_params(), 2432) - self.assertEqual(len(self.n1._info[0].layer.weights), 2) + self.assertEqual(len(self.n1._info[0].layer.all_weights), 2) self.assertEqual(self.n1.get_shape().as_list()[1:], [200, 200, 32]) def test_layer_n2(self): @@ -231,7 +231,7 @@ def test_layer_n2(self): # self.assertEqual(len(self.n2.all_layers), 3) # self.assertEqual(len(self.n2.all_params), 4) # self.assertEqual(self.n2.count_params(), 11680) - self.assertEqual(len(self.n2._info[0].layer.weights), 2) + self.assertEqual(len(self.n2._info[0].layer.all_weights), 2) self.assertEqual(self.n2.get_shape().as_list()[1:], [100, 100, 32]) def test_layer_n3(self): @@ -239,7 +239,7 @@ def test_layer_n3(self): # self.assertEqual(len(self.n3.all_layers), 4) # self.assertEqual(len(self.n3.all_params), 5) # self.assertEqual(self.n3.count_params(), 20896) - self.assertEqual(len(self.n3._info[0].layer.weights), 1) # b_init is None + self.assertEqual(len(self.n3._info[0].layer.all_weights), 1) # b_init is None self.assertEqual(self.n3.get_shape().as_list()[1:], [50, 50, 32]) def test_layer_n4(self): @@ -247,7 +247,7 @@ def test_layer_n4(self): # self.assertEqual(len(self.n4.all_layers), 5) # self.assertEqual(len(self.n4.all_params), 7) # self.assertEqual(self.n4.count_params(), 46528) - self.assertEqual(len(self.n4._info[0].layer.weights), 2) + self.assertEqual(len(self.n4._info[0].layer.all_weights), 2) self.assertEqual(self.n4.get_shape().as_list()[1:], [100, 100, 32]) def test_layer_n5(self): @@ -255,7 +255,7 @@ def test_layer_n5(self): # self.assertEqual(len(self.n5.all_layers), 6) # self.assertEqual(len(self.n5.all_params), 9) # self.assertEqual(self.n5.count_params(), 55776) - self.assertEqual(len(self.n5._info[0].layer.weights), 2) + self.assertEqual(len(self.n5._info[0].layer.all_weights), 2) self.assertEqual(self.n5.get_shape().as_list()[1:], [200, 200, 32]) def test_layer_n6(self): @@ -263,7 +263,7 @@ def test_layer_n6(self): # self.assertEqual(len(self.n6.all_layers), 7) # self.assertEqual(len(self.n6.all_params), 11) # self.assertEqual(self.n6.count_params(), 56416) - self.assertEqual(len(self.n6._info[0].layer.weights), 2) + self.assertEqual(len(self.n6._info[0].layer.all_weights), 2) self.assertEqual(self.n6.get_shape().as_list()[1:], [200, 200, 64]) def test_layer_n7(self): @@ -271,7 +271,7 @@ def test_layer_n7(self): # self.assertEqual(len(self.n7.all_layers), 8) # self.assertEqual(len(self.n7.all_params), 13) # self.assertEqual(self.n7.count_params(), 74880) - self.assertEqual(len(self.n7._info[0].layer.weights), 2) + self.assertEqual(len(self.n7._info[0].layer.all_weights), 2) self.assertEqual(self.n7.get_shape().as_list()[1:], [100, 100, 32]) def test_layer_n8(self): @@ -279,7 +279,7 @@ def test_layer_n8(self): # self.assertEqual(len(self.n7.all_layers), 8) # self.assertEqual(len(self.n7.all_params), 13) # self.assertEqual(self.n7.count_params(), 74880) - self.assertEqual(len(self.n8._info[0].layer.weights), 2) + self.assertEqual(len(self.n8._info[0].layer.all_weights), 2) self.assertEqual(self.n8.get_shape().as_list()[1:], [50, 50, 64]) def test_layer_n9(self): @@ -287,35 +287,35 @@ def test_layer_n9(self): # self.assertEqual(len(self.n7.all_layers), 8) # self.assertEqual(len(self.n7.all_params), 13) # self.assertEqual(self.n7.count_params(), 74880) - self.assertEqual(len(self.n9._info[0].layer.weights), 3) + self.assertEqual(len(self.n9._info[0].layer.all_weights), 3) self.assertEqual(self.n9.get_shape().as_list()[1:], [24, 24, 32]) def test_layer_n10(self): # self.assertEqual(len(self.n7.all_layers), 8) # self.assertEqual(len(self.n7.all_params), 13) # self.assertEqual(self.n7.count_params(), 74880) - self.assertEqual(len(self.n10._info[0].layer.weights), 2) + self.assertEqual(len(self.n10._info[0].layer.all_weights), 2) self.assertEqual(self.n10.get_shape().as_list()[1:], [12, 12, 64]) def test_layer_n11(self): # self.assertEqual(len(self.n7.all_layers), 8) # self.assertEqual(len(self.n7.all_params), 13) # self.assertEqual(self.n7.count_params(), 74880) - self.assertEqual(len(self.n11._info[0].layer.weights), 2) + self.assertEqual(len(self.n11._info[0].layer.all_weights), 2) self.assertEqual(self.n11.get_shape().as_list()[1:], [12, 12, 32]) def test_layer_n12(self): # self.assertEqual(len(self.n7.all_layers), 8) # self.assertEqual(len(self.n7.all_params), 13) # self.assertEqual(self.n7.count_params(), 74880) - self.assertEqual(len(self.n12._info[0].layer.weights), 2) + self.assertEqual(len(self.n12._info[0].layer.all_weights), 2) self.assertEqual(self.n12.get_shape().as_list()[1:], [12, 12, 64]) def test_layer_n13(self): # self.assertEqual(len(self.n7.all_layers), 8) # self.assertEqual(len(self.n7.all_params), 13) # self.assertEqual(self.n7.count_params(), 74880) - self.assertEqual(len(self.n13._info[0].layer.weights), 2) + self.assertEqual(len(self.n13._info[0].layer.all_weights), 2) self.assertEqual(self.n13.get_shape().as_list()[1:], [12, 12, 32]) def test_layer_n14(self): @@ -393,7 +393,7 @@ def test_layer_n1(self): # self.assertEqual(len(self.n1.all_layers), 2) # self.assertEqual(len(self.n1.all_params), 2) # self.assertEqual(self.n1.count_params(), 800) - self.assertEqual(len(self.n1._info[0].layer.weights), 2) + self.assertEqual(len(self.n1._info[0].layer.all_weights), 2) self.assertEqual(self.n1.get_shape().as_list()[1:], [10, 10, 10, 32]) def test_layer_n2(self): @@ -401,7 +401,7 @@ def test_layer_n2(self): # self.assertEqual(len(self.n2.all_layers), 3) # self.assertEqual(len(self.n2.all_params), 4) # self.assertEqual(self.n2.count_params(), 33696) - self.assertEqual(len(self.n2._info[0].layer.weights), 2) + self.assertEqual(len(self.n2._info[0].layer.all_weights), 2) self.assertEqual(self.n2.get_shape().as_list()[1:], [20, 20, 20, 128]) def test_layer_n3(self): @@ -409,7 +409,7 @@ def test_layer_n3(self): # self.assertEqual(len(self.n3.all_layers), 4) # self.assertEqual(len(self.n3.all_params), 6) # self.assertEqual(self.n3.count_params(), 144320) - self.assertEqual(len(self.n3._info[0].layer.weights), 1) # b_init is None + self.assertEqual(len(self.n3._info[0].layer.all_weights), 1) # b_init is None self.assertEqual(self.n3.get_shape().as_list()[1:], [7, 7, 7, 64]) def test_layer_n4(self): @@ -417,7 +417,7 @@ def test_layer_n4(self): # self.assertEqual(len(self.n3.all_layers), 4) # self.assertEqual(len(self.n3.all_params), 6) # self.assertEqual(self.n3.count_params(), 144320) - self.assertEqual(len(self.n4._info[0].layer.weights), 2) + self.assertEqual(len(self.n4._info[0].layer.all_weights), 2) self.assertEqual(self.n4.get_shape().as_list()[1:], [14, 14, 14, 32]) diff --git a/tests/layers/test_layers_core_basedense_dropout.py b/tests/layers/test_layers_core_basedense_dropout.py index 2ec0019f3..7968822a7 100644 --- a/tests/layers/test_layers_core_basedense_dropout.py +++ b/tests/layers/test_layers_core_basedense_dropout.py @@ -73,15 +73,15 @@ def test_net1(self): def test_net2(self): # test weights - self.assertEqual(self.innet._info[0].layer.weights, None) - self.assertEqual(self.dropout1._info[0].layer.weights, None) - self.assertEqual(self.dense1._info[0].layer.weights[0].get_shape().as_list(), [784, 800]) - self.assertEqual(self.dense1._info[0].layer.weights[1].get_shape().as_list(), [ + self.assertEqual(self.innet._info[0].layer.all_weights, None) + self.assertEqual(self.dropout1._info[0].layer.all_weights, None) + self.assertEqual(self.dense1._info[0].layer.all_weights[0].get_shape().as_list(), [784, 800]) + self.assertEqual(self.dense1._info[0].layer.all_weights[1].get_shape().as_list(), [ 800, ]) - self.assertEqual(self.dense2._info[0].layer.weights[0].get_shape().as_list(), [800, 10]) - self.assertEqual(len(self.dense1._info[0].layer.weights), 2) - self.assertEqual(len(self.dense2._info[0].layer.weights), 1) + self.assertEqual(self.dense2._info[0].layer.all_weights[0].get_shape().as_list(), [800, 10]) + self.assertEqual(len(self.dense1._info[0].layer.all_weights), 2) + self.assertEqual(len(self.dense2._info[0].layer.all_weights), 1) self.assertEqual(len(self.model.weights), 3) diff --git a/tests/layers/test_layers_deformable_convolution.py b/tests/layers/test_layers_deformable_convolution.py index 41b0ad533..b31d5ce98 100644 --- a/tests/layers/test_layers_deformable_convolution.py +++ b/tests/layers/test_layers_deformable_convolution.py @@ -44,12 +44,12 @@ def tearDownClass(cls): def test_layer_n1(self): - self.assertEqual(len(self.deformconv1._info[0].layer.weights), 2) + self.assertEqual(len(self.deformconv1._info[0].layer.all_weights), 2) self.assertEqual(self.deformconv1.get_shape().as_list()[1:], [10, 10, 32]) def test_layer_n2(self): - self.assertEqual(len(self.deformconv2._info[0].layer.weights), 2) + self.assertEqual(len(self.deformconv2._info[0].layer.all_weights), 2) self.assertEqual(self.deformconv2.get_shape().as_list()[1:], [10, 10, 64]) diff --git a/tests/models/test_model_core.py b/tests/models/test_model_core.py index 1c3732819..4fab4110c 100644 --- a/tests/models/test_model_core.py +++ b/tests/models/test_model_core.py @@ -381,7 +381,7 @@ def test_get_layer(self): model_basic = basic_static_model() self.assertIsInstance(model_basic.get_layer('conv2'), tl.layers.Conv2d) self.assertIsInstance(model_basic.get_layer(index=2), tl.layers.MaxPool2d) - print([w.name for w in model_basic.get_layer(index=-1).weights]) + print([w.name for w in model_basic.get_layer(index=-1).all_weights]) try: model_basic.get_layer('abc') except Exception as e: diff --git a/tests/models/test_model_save.py b/tests/models/test_model_save.py index 1029a5622..b646b28bb 100644 --- a/tests/models/test_model_save.py +++ b/tests/models/test_model_save.py @@ -196,13 +196,13 @@ def test_nested_vgg(self): nested_vgg.save_weights("nested_vgg.h5") # modify vgg1 weight val - tar_weight1 = nested_vgg.vgg1.layers[0].weights[0] + tar_weight1 = nested_vgg.vgg1.layers[0].all_weights[0] print(tar_weight1.name) ori_val1 = tar_weight1.numpy() modify_val1 = np.zeros_like(ori_val1) tar_weight1.assign(modify_val1) # modify vgg2 weight val - tar_weight2 = nested_vgg.vgg2.layers[1].weights[0] + tar_weight2 = nested_vgg.vgg2.layers[1].all_weights[0] print(tar_weight2.name) ori_val2 = tar_weight2.numpy() modify_val2 = np.zeros_like(ori_val2) @@ -236,12 +236,12 @@ def forward(self, *inputs, **kwargs): print([x.name for x in net.all_layers]) # modify vgg1 weight val - tar_weight1 = net.inner.vgg1.layers[0].weights[0] + tar_weight1 = net.inner.vgg1.layers[0].all_weights[0] ori_val1 = tar_weight1.numpy() modify_val1 = np.zeros_like(ori_val1) tar_weight1.assign(modify_val1) # modify vgg2 weight val - tar_weight2 = net.inner.vgg2.layers[1].weights[0] + tar_weight2 = net.inner.vgg2.layers[1].all_weights[0] ori_val2 = tar_weight2.numpy() modify_val2 = np.zeros_like(ori_val2) tar_weight2.assign(modify_val2) diff --git a/tests/test_initializers.py b/tests/test_initializers.py index 7998c98ac..df86fd834 100644 --- a/tests/test_initializers.py +++ b/tests/test_initializers.py @@ -30,38 +30,38 @@ def init_dense(self, w_init): def test_zeros(self): dense = self.init_dense(tl.initializers.zeros()) - self.assertEqual(np.sum(dense.weights[0].numpy() - np.zeros(shape=self.w_shape)), self.eps) + self.assertEqual(np.sum(dense.all_weights[0].numpy() - np.zeros(shape=self.w_shape)), self.eps) nn = dense(self.ni) def test_ones(self): dense = self.init_dense(tl.initializers.ones()) - self.assertEqual(np.sum(dense.weights[0].numpy() - np.ones(shape=self.w_shape)), self.eps) + self.assertEqual(np.sum(dense.all_weights[0].numpy() - np.ones(shape=self.w_shape)), self.eps) nn = dense(self.ni) def test_constant(self): dense = self.init_dense(tl.initializers.constant(value=5.0)) - self.assertEqual(np.sum(dense.weights[0].numpy() - np.ones(shape=self.w_shape) * 5.0), self.eps) + self.assertEqual(np.sum(dense.all_weights[0].numpy() - np.ones(shape=self.w_shape) * 5.0), self.eps) nn = dense(self.ni) # test with numpy arr arr = np.random.uniform(size=self.w_shape).astype(np.float32) dense = self.init_dense(tl.initializers.constant(value=arr)) - self.assertEqual(np.sum(dense.weights[0].numpy() - arr), self.eps) + self.assertEqual(np.sum(dense.all_weights[0].numpy() - arr), self.eps) nn = dense(self.ni) def test_RandomUniform(self): dense = self.init_dense(tl.initializers.random_uniform(minval=-0.1, maxval=0.1, seed=1234)) - print(dense.weights[0].numpy()) + print(dense.all_weights[0].numpy()) nn = dense(self.ni) def test_RandomNormal(self): dense = self.init_dense(tl.initializers.random_normal(mean=0.0, stddev=0.1)) - print(dense.weights[0].numpy()) + print(dense.all_weights[0].numpy()) nn = dense(self.ni) def test_TruncatedNormal(self): dense = self.init_dense(tl.initializers.truncated_normal(mean=0.0, stddev=0.1)) - print(dense.weights[0].numpy()) + print(dense.all_weights[0].numpy()) nn = dense(self.ni) def test_deconv2d_bilinear_upsampling_initializer(self): From 58edaed9ea0295efb5043a411956bbb03aa856c0 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Mon, 13 May 2019 00:13:18 +0800 Subject: [PATCH 02/19] weights -> all_weights --- docs/user/faq.rst | 4 +- .../tutorial_cifar10_cnn_static.py | 2 +- .../tutorial_mnist_mlp_dynamic.py | 2 +- .../tutorial_mnist_mlp_dynamic_2.py | 2 +- .../tutorial_mnist_mlp_static.py | 2 +- .../tutorial_mnist_mlp_static_2.py | 2 +- .../basic_tutorials/tutorial_mnist_siamese.py | 2 +- examples/keras_tfslim/tutorial_keras.py | 2 +- .../tutorial_atari_pong.py | 2 +- .../tutorial_cartpole_ac.py | 8 ++-- .../tutorial_frozenlake_dqn.py | 2 +- ...ial_spatial_transformer_network_dynamic.py | 2 +- ...rial_spatial_transformer_network_static.py | 2 +- .../tutorial_imdb_fasttext.py | 4 +- .../text_generation/tutorial_generate_text.py | 2 +- .../tutorial_word2vec_basic.py | 4 +- tensorlayer/db.py | 2 +- tensorlayer/files/utils.py | 8 ++-- tensorlayer/layers/lambda_layers.py | 4 +- tensorlayer/models/core.py | 8 ++-- tensorlayer/models/mobilenetv1.py | 4 +- tensorlayer/models/squeezenetv1.py | 2 +- tensorlayer/models/vgg.py | 4 +- tensorlayer/utils.py | 4 +- tests/files/test_utils_saveload.py | 40 +++++++++---------- .../test_layers_core_basedense_dropout.py | 2 +- tests/layers/test_layers_lambda.py | 4 +- tests/models/test_auto_naming.py | 6 +-- tests/models/test_keras_save.py | 4 +- tests/models/test_model_core.py | 12 +++--- tests/models/test_model_save.py | 40 +++++++++---------- tests/models/test_model_save_graph.py | 16 ++++---- tests/pending/test_mnist_simple.py | 2 +- tests/performance_test/vgg/tl2-autograph.py | 2 +- tests/performance_test/vgg/tl2-eager.py | 2 +- 35 files changed, 105 insertions(+), 105 deletions(-) diff --git a/docs/user/faq.rst b/docs/user/faq.rst index dc94586ff..4988a2c1d 100644 --- a/docs/user/faq.rst +++ b/docs/user/faq.rst @@ -46,13 +46,13 @@ To choose which variables to update, you can do as below. .. code-block:: python - train_params = network.weights[3:] + train_params = network.trainable_weights[3:] The second way is to get the variables by a given name. For example, if you want to get all variables which the layer name contain ``dense``, you can do as below. .. code-block:: python - train_params = network.get_layer('dense').weights + train_params = network.get_layer('dense').trainable_weights After you get the variable list, you can define your optimizer like that so as to update only a part of the variables. diff --git a/examples/basic_tutorials/tutorial_cifar10_cnn_static.py b/examples/basic_tutorials/tutorial_cifar10_cnn_static.py index 896b15c86..c12c791a1 100644 --- a/examples/basic_tutorials/tutorial_cifar10_cnn_static.py +++ b/examples/basic_tutorials/tutorial_cifar10_cnn_static.py @@ -87,7 +87,7 @@ def get_model_batchnorm(inputs_shape): # learning_rate_decay_factor = 0.1 # num_epoch_decay = 350 -train_weights = net.weights +train_weights = net.trainable_weights # learning_rate = tf.Variable(init_learning_rate) optimizer = tf.optimizers.Adam(learning_rate) diff --git a/examples/basic_tutorials/tutorial_mnist_mlp_dynamic.py b/examples/basic_tutorials/tutorial_mnist_mlp_dynamic.py index b1b5909d6..1ffa7fbe0 100644 --- a/examples/basic_tutorials/tutorial_mnist_mlp_dynamic.py +++ b/examples/basic_tutorials/tutorial_mnist_mlp_dynamic.py @@ -46,7 +46,7 @@ def forward(self, x, foo=None): n_epoch = 500 batch_size = 500 print_freq = 5 -train_weights = MLP.weights +train_weights = MLP.trainable_weights optimizer = tf.optimizers.Adam(learning_rate=0.0001) ## the following code can help you understand SGD deeply diff --git a/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_2.py b/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_2.py index b369a0b26..b752012b0 100644 --- a/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_2.py +++ b/examples/basic_tutorials/tutorial_mnist_mlp_dynamic_2.py @@ -65,7 +65,7 @@ def forward(self, x, foo=None): n_epoch = 500 batch_size = 500 print_freq = 5 -train_weights = MLP1.weights + MLP2.weights +train_weights = MLP1.trainable_weights + MLP2.trainable_weights optimizer = tf.optimizers.Adam(learning_rate=0.0001) ## the following code can help you understand SGD deeply diff --git a/examples/basic_tutorials/tutorial_mnist_mlp_static.py b/examples/basic_tutorials/tutorial_mnist_mlp_static.py index 1d99a3a86..c9c15f911 100644 --- a/examples/basic_tutorials/tutorial_mnist_mlp_static.py +++ b/examples/basic_tutorials/tutorial_mnist_mlp_static.py @@ -37,7 +37,7 @@ def get_model(inputs_shape): n_epoch = 500 batch_size = 500 print_freq = 5 -train_weights = MLP.weights +train_weights = MLP.trainable_weights optimizer = tf.optimizers.Adam(lr=0.0001) ## the following code can help you understand SGD deeply diff --git a/examples/basic_tutorials/tutorial_mnist_mlp_static_2.py b/examples/basic_tutorials/tutorial_mnist_mlp_static_2.py index 52568bb88..f0836c528 100644 --- a/examples/basic_tutorials/tutorial_mnist_mlp_static_2.py +++ b/examples/basic_tutorials/tutorial_mnist_mlp_static_2.py @@ -46,7 +46,7 @@ def get_model(inputs_shape, hmodel): n_epoch = 500 batch_size = 500 print_freq = 5 -train_weights = MLP.weights +train_weights = MLP.trainable_weights optimizer = tf.optimizers.Adam(lr=0.0001) ## the following code can help you understand SGD deeply diff --git a/examples/basic_tutorials/tutorial_mnist_siamese.py b/examples/basic_tutorials/tutorial_mnist_siamese.py index 6b744367d..db43f1163 100644 --- a/examples/basic_tutorials/tutorial_mnist_siamese.py +++ b/examples/basic_tutorials/tutorial_mnist_siamese.py @@ -96,7 +96,7 @@ def create_pairs(x, digit_indices): # training settings print_freq = 5 -train_weights = model.weights +train_weights = model.trainable_weights optimizer = tf.optimizers.RMSprop() diff --git a/examples/keras_tfslim/tutorial_keras.py b/examples/keras_tfslim/tutorial_keras.py index ba2c4b831..0622bc745 100644 --- a/examples/keras_tfslim/tutorial_keras.py +++ b/examples/keras_tfslim/tutorial_keras.py @@ -38,7 +38,7 @@ n_epoch = 200 learning_rate = 0.0001 -train_params = network.weights +train_params = network.trainable_weights optimizer = tf.optimizers.Adam(learning_rate) for epoch in range(n_epoch): diff --git a/examples/reinforcement_learning/tutorial_atari_pong.py b/examples/reinforcement_learning/tutorial_atari_pong.py index 0e18f93c4..ad8e264df 100644 --- a/examples/reinforcement_learning/tutorial_atari_pong.py +++ b/examples/reinforcement_learning/tutorial_atari_pong.py @@ -84,7 +84,7 @@ def get_model(inputs_shape): M = tl.models.Model(inputs=ni, outputs=nn, name="mlp") return M model = get_model([None, D]) -train_weights = model.weights +train_weights = model.trainable_weights # probs = model(t_states, is_train=True).outputs # sampling_prob = tf.nn.softmax(probs) diff --git a/examples/reinforcement_learning/tutorial_cartpole_ac.py b/examples/reinforcement_learning/tutorial_cartpole_ac.py index e525a3bdb..4d8b6f8ea 100644 --- a/examples/reinforcement_learning/tutorial_cartpole_ac.py +++ b/examples/reinforcement_learning/tutorial_cartpole_ac.py @@ -122,8 +122,8 @@ def learn(self, s, a, td): _logits = self.model([s]).outputs # _probs = tf.nn.softmax(_logits) _exp_v = tl.rein.cross_entropy_reward_loss(logits=_logits, actions=[a], rewards=td[0]) - grad = tape.gradient(_exp_v, self.model.weights) - self.optimizer.apply_gradients(zip(grad, self.model.weights)) + grad = tape.gradient(_exp_v, self.model.trainable_weights) + self.optimizer.apply_gradients(zip(grad, self.model.trainable_weights)) return _exp_v def choose_action(self, s): @@ -178,8 +178,8 @@ def learn(self, s, r, s_): # TD_error = r + lambd * V(newS) - V(S) td_error = r + LAMBDA * v_ - v loss = tf.square(td_error) - grad = tape.gradient(loss, self.model.weights) - self.optimizer.apply_gradients(zip(grad, self.model.weights)) + grad = tape.gradient(loss, self.model.trainable_weights) + self.optimizer.apply_gradients(zip(grad, self.model.trainable_weights)) return td_error diff --git a/examples/reinforcement_learning/tutorial_frozenlake_dqn.py b/examples/reinforcement_learning/tutorial_frozenlake_dqn.py index c905dee4c..9411da423 100644 --- a/examples/reinforcement_learning/tutorial_frozenlake_dqn.py +++ b/examples/reinforcement_learning/tutorial_frozenlake_dqn.py @@ -63,7 +63,7 @@ def get_model(inputs_shape): return tl.models.Model(inputs=ni, outputs=nn, name="Q-Network") qnetwork = get_model([1, 16]) qnetwork.train() -train_weights = qnetwork.weights +train_weights = qnetwork.trainable_weights # chose action greedily with reward. in Q-Learning, policy is greedy, so we use "max" to select the next action. # predict = tf.argmax(y, 1) diff --git a/examples/spatial_transformer_network/tutorial_spatial_transformer_network_dynamic.py b/examples/spatial_transformer_network/tutorial_spatial_transformer_network_dynamic.py index e0db623fb..aecc69f61 100644 --- a/examples/spatial_transformer_network/tutorial_spatial_transformer_network_dynamic.py +++ b/examples/spatial_transformer_network/tutorial_spatial_transformer_network_dynamic.py @@ -87,7 +87,7 @@ def forward(self, inputs): learning_rate = 0.0001 print_freq = 10 batch_size = 64 -train_weights = net.weights +train_weights = net.trainable_weights optimizer = tf.optimizers.Adam(lr=learning_rate) ##================== TRAINING ================================================## diff --git a/examples/spatial_transformer_network/tutorial_spatial_transformer_network_static.py b/examples/spatial_transformer_network/tutorial_spatial_transformer_network_static.py index dfc615fc8..c9a93629f 100644 --- a/examples/spatial_transformer_network/tutorial_spatial_transformer_network_static.py +++ b/examples/spatial_transformer_network/tutorial_spatial_transformer_network_static.py @@ -84,7 +84,7 @@ def get_model(inputs_shape): learning_rate = 0.0001 print_freq = 10 batch_size = 64 -train_weights = net.weights +train_weights = net.trainable_weights optimizer = tf.optimizers.Adam(lr=learning_rate) ##================== TRAINING ================================================## diff --git a/examples/text_classification/tutorial_imdb_fasttext.py b/examples/text_classification/tutorial_imdb_fasttext.py index 6d785f402..2c2c7aed0 100644 --- a/examples/text_classification/tutorial_imdb_fasttext.py +++ b/examples/text_classification/tutorial_imdb_fasttext.py @@ -138,8 +138,8 @@ def train_test_and_save_model(): cost = tl.cost.cross_entropy(y_pred, y_batch, name='cost') # backward, calculate gradients and update the weights - grad = tape.gradient(cost, model.weights) - optimizer.apply_gradients(zip(grad, model.weights)) + grad = tape.gradient(cost, model.trainable_weights) + optimizer.apply_gradients(zip(grad, model.trainable_weights)) # calculate the accuracy predictions = tf.argmax(y_pred, axis=1, output_type=tf.int32) diff --git a/examples/text_generation/tutorial_generate_text.py b/examples/text_generation/tutorial_generate_text.py index ef2b2e6c3..22a17ea37 100644 --- a/examples/text_generation/tutorial_generate_text.py +++ b/examples/text_generation/tutorial_generate_text.py @@ -289,7 +289,7 @@ def loss_fn(outputs, targets, batch_size, sequence_length): # tvars = network.all_params $ all parameters # tvars = network.all_params[1:] $ parameters except embedding matrix # Train the whole network. - tvars = rnn_model.weights + tvars = rnn_model.trainable_weights # grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), max_grad_norm) # optimizer = tf.train.GradientDescentOptimizer(lr) train_op = tf.train.GradientDescentOptimizer(lr).minimize(cost, var_list=tvars) diff --git a/examples/text_word_embedding/tutorial_word2vec_basic.py b/examples/text_word_embedding/tutorial_word2vec_basic.py index 4285ee992..6310699ad 100644 --- a/examples/text_word_embedding/tutorial_word2vec_basic.py +++ b/examples/text_word_embedding/tutorial_word2vec_basic.py @@ -240,8 +240,8 @@ def main_word2vec_basic(): with tf.GradientTape() as tape: outputs, nce_cost = model([batch_inputs, batch_labels]) - grad = tape.gradient(nce_cost, model.weights) - optimizer.apply_gradients(zip(grad, model.weights)) + grad = tape.gradient(nce_cost, model.trainable_weights) + optimizer.apply_gradients(zip(grad, model.trainable_weights)) average_loss += nce_cost diff --git a/tensorlayer/db.py b/tensorlayer/db.py index 025566dc2..cb8db8e10 100644 --- a/tensorlayer/db.py +++ b/tensorlayer/db.py @@ -148,7 +148,7 @@ def save_model(self, network=None, model_name='model', **kwargs): self._fill_project_info(kwargs) # put project_name into kwargs # params = network.get_all_params() - params = network.weights + params = network.all_weights s = time.time() diff --git a/tensorlayer/files/utils.py b/tensorlayer/files/utils.py index e83a246bd..72fcb1824 100644 --- a/tensorlayer/files/utils.py +++ b/tensorlayer/files/utils.py @@ -1907,7 +1907,7 @@ def save_npz(save_list=None, name='model.npz'): -------- Save model to npz - >>> tl.files.save_npz(network.weights, name='model.npz') + >>> tl.files.save_npz(network.all_weights, name='model.npz') Load model from npz (Method 1) @@ -1993,7 +1993,7 @@ def assign_weights(weights, network): """ ops = [] for idx, param in enumerate(weights): - ops.append(network.weights[idx].assign(param)) + ops.append(network.all_weights[idx].assign(param)) return ops @@ -2073,7 +2073,7 @@ def load_and_assign_npz_dict(name='model.npz', network=None, skip=False): if len(weights.keys()) != len(set(weights.keys())): raise Exception("Duplication in model npz_dict %s" % name) - net_weights_name = [w.name for w in network.weights] + net_weights_name = [w.name for w in network.all_weights] for key in weights.keys(): if key not in net_weights_name: @@ -2085,7 +2085,7 @@ def load_and_assign_npz_dict(name='model.npz', network=None, skip=False): "if you want to skip redundant or mismatch weights." % key ) else: - assign_tf_variable(network.weights[net_weights_name.index(key)], weights[key]) + assign_tf_variable(network.all_weights[net_weights_name.index(key)], weights[key]) logging.info("[*] Model restored from npz_dict %s" % name) diff --git a/tensorlayer/layers/lambda_layers.py b/tensorlayer/layers/lambda_layers.py index 997721e7c..8bb035c9f 100644 --- a/tensorlayer/layers/lambda_layers.py +++ b/tensorlayer/layers/lambda_layers.py @@ -96,8 +96,8 @@ class Lambda(Layer): >>> pred_y = model(data_x) >>> loss = tl.cost.mean_squared_error(pred_y, data_y) - >>> gradients = tape.gradient(loss, model.weights) - >>> optimizer.apply_gradients(zip(gradients, model.weights)) + >>> gradients = tape.gradient(loss, model.trainable_weights) + >>> optimizer.apply_gradients(zip(gradients, model.trainable_weights)) """ diff --git a/tensorlayer/models/core.py b/tensorlayer/models/core.py index 1de703b7c..9015f3ca9 100644 --- a/tensorlayer/models/core.py +++ b/tensorlayer/models/core.py @@ -411,7 +411,7 @@ def nontrainable_weights(self): return self._nontrainable_weights @property - def weights(self): + def all_weights(self): """Return all weights of this network in a list.""" if self._all_weights is not None and len(self._all_weights) > 0: # self._all_weights already extracted, so do nothing @@ -793,7 +793,7 @@ def save_weights(self, filepath, format=None): >>> net.save_weights('./model.npz', format='npz_dict') """ - if self.weights is None or len(self.weights) == 0: + if self.all_weights is None or len(self.all_weights) == 0: logging.warning("Model contains no weights or layers haven't been built, nothing will be saved") return @@ -807,9 +807,9 @@ def save_weights(self, filepath, format=None): if format == 'hdf5' or format == 'h5': utils.save_weights_to_hdf5(filepath, self) elif format == 'npz': - utils.save_npz(self.weights, filepath) + utils.save_npz(self.all_weights, filepath) elif format == 'npz_dict': - utils.save_npz_dict(self.weights, filepath) + utils.save_npz_dict(self.all_weights, filepath) elif format == 'ckpt': # TODO: enable this when tf save ckpt is enabled raise NotImplementedError("ckpt load/save is not supported now.") diff --git a/tensorlayer/models/mobilenetv1.py b/tensorlayer/models/mobilenetv1.py index 0454826a2..8065eeef3 100644 --- a/tensorlayer/models/mobilenetv1.py +++ b/tensorlayer/models/mobilenetv1.py @@ -44,10 +44,10 @@ def restore_params(network, path='models'): expected_bytes=25600116 ) # ls -al params = load_npz(name=os.path.join(path, 'mobilenet.npz')) - for idx, net_weight in enumerate(network.weights): + for idx, net_weight in enumerate(network.all_weights): if 'batchnorm' in net_weight.name: params[idx] = params[idx].reshape(1, 1, 1, -1) - assign_weights(params[:len(network.weights)], network) + assign_weights(params[:len(network.all_weights)], network) del params diff --git a/tensorlayer/models/squeezenetv1.py b/tensorlayer/models/squeezenetv1.py index e94aeab90..6d6a70535 100644 --- a/tensorlayer/models/squeezenetv1.py +++ b/tensorlayer/models/squeezenetv1.py @@ -38,7 +38,7 @@ def restore_params(network, path='models'): expected_bytes=7405613 ) # ls -al params = load_npz(name=os.path.join(path, 'squeezenet.npz')) - assign_weights(params[:len(network.weights)], network) + assign_weights(params[:len(network.all_weights)], network) del params diff --git a/tensorlayer/models/vgg.py b/tensorlayer/models/vgg.py index 858784ec3..391878c61 100644 --- a/tensorlayer/models/vgg.py +++ b/tensorlayer/models/vgg.py @@ -163,7 +163,7 @@ def restore_model(model, layer_type): for val in sorted(npz.items()): logging.info(" Loading weights %s in %s" % (str(val[1].shape), val[0])) weights.append(val[1]) - if len(model.weights) == len(weights): + if len(model.all_weights) == len(weights): break elif layer_type == 'vgg19': npz = np.load(os.path.join('models', model_saved_name[layer_type]), encoding='latin1').item() @@ -172,7 +172,7 @@ def restore_model(model, layer_type): logging.info(" Loading %s in %s" % (str(val[1][0].shape), val[0])) logging.info(" Loading %s in %s" % (str(val[1][1].shape), val[0])) weights.extend(val[1]) - if len(model.weights) == len(weights): + if len(model.all_weights) == len(weights): break # assign weight values assign_weights(weights, model) diff --git a/tensorlayer/utils.py b/tensorlayer/utils.py index 0d555eae3..cf56f49c2 100644 --- a/tensorlayer/utils.py +++ b/tensorlayer/utils.py @@ -661,8 +661,8 @@ def _train_step(network, X_batch, y_batch, cost, train_op=tf.optimizers.Adam(lea y_pred = network(X_batch) _loss = cost(y_pred, y_batch) - grad = tape.gradient(_loss, network.weights) - train_op.apply_gradients(zip(grad, network.weights)) + grad = tape.gradient(_loss, network.trainable_weights) + train_op.apply_gradients(zip(grad, network.trainable_weights)) if acc is not None: _acc = acc(y_pred, y_batch) diff --git a/tests/files/test_utils_saveload.py b/tests/files/test_utils_saveload.py index b3954f5b8..2cd329735 100644 --- a/tests/files/test_utils_saveload.py +++ b/tests/files/test_utils_saveload.py @@ -67,48 +67,48 @@ def tearDownClass(cls): pass def test_hdf5(self): - modify_val = np.zeros_like(self.static_model.weights[-2].numpy()) - ori_val = self.static_model.weights[-2].numpy() + modify_val = np.zeros_like(self.static_model.all_weights[-2].numpy()) + ori_val = self.static_model.all_weights[-2].numpy() tl.files.save_weights_to_hdf5("./model_basic.h5", self.static_model) - self.static_model.weights[-2].assign(modify_val) + self.static_model.all_weights[-2].assign(modify_val) tl.files.load_hdf5_to_weights_in_order("./model_basic.h5", self.static_model) - self.assertLess(np.max(np.abs(ori_val - self.static_model.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - self.static_model.all_weights[-2].numpy())), 1e-7) - self.static_model.weights[-2].assign(modify_val) + self.static_model.all_weights[-2].assign(modify_val) tl.files.load_hdf5_to_weights("./model_basic.h5", self.static_model) - self.assertLess(np.max(np.abs(ori_val - self.static_model.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - self.static_model.all_weights[-2].numpy())), 1e-7) ori_weights = self.static_model._weights self.static_model._weights = self.static_model._weights[1:] - self.static_model.weights[-2].assign(modify_val) + self.static_model.all_weights[-2].assign(modify_val) tl.files.load_hdf5_to_weights("./model_basic.h5", self.static_model, skip=True) - self.assertLess(np.max(np.abs(ori_val - self.static_model.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - self.static_model.all_weights[-2].numpy())), 1e-7) self.static_model._weights = ori_weights def test_npz(self): - modify_val = np.zeros_like(self.dynamic_model.weights[-2].numpy()) - ori_val = self.dynamic_model.weights[-2].numpy() - tl.files.save_npz(self.dynamic_model.weights, "./model_basic.npz") + modify_val = np.zeros_like(self.dynamic_model.all_weights[-2].numpy()) + ori_val = self.dynamic_model.all_weights[-2].numpy() + tl.files.save_npz(self.dynamic_model.all_weights, "./model_basic.npz") - self.dynamic_model.weights[-2].assign(modify_val) + self.dynamic_model.all_weights[-2].assign(modify_val) tl.files.load_and_assign_npz("./model_basic.npz", self.dynamic_model) - self.assertLess(np.max(np.abs(ori_val - self.dynamic_model.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - self.dynamic_model.all_weights[-2].numpy())), 1e-7) def test_npz_dict(self): - modify_val = np.zeros_like(self.dynamic_model.weights[-2].numpy()) - ori_val = self.dynamic_model.weights[-2].numpy() - tl.files.save_npz_dict(self.dynamic_model.weights, "./model_basic.npz") + modify_val = np.zeros_like(self.dynamic_model.all_weights[-2].numpy()) + ori_val = self.dynamic_model.all_weights[-2].numpy() + tl.files.save_npz_dict(self.dynamic_model.all_weights, "./model_basic.npz") - self.dynamic_model.weights[-2].assign(modify_val) + self.dynamic_model.all_weights[-2].assign(modify_val) tl.files.load_and_assign_npz_dict("./model_basic.npz", self.dynamic_model) - self.assertLess(np.max(np.abs(ori_val - self.dynamic_model.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - self.dynamic_model.all_weights[-2].numpy())), 1e-7) ori_weights = self.dynamic_model._weights self.dynamic_model._weights = self.static_model._weights[1:] - self.dynamic_model.weights[-2].assign(modify_val) + self.dynamic_model.all_weights[-2].assign(modify_val) tl.files.load_and_assign_npz_dict("./model_basic.npz", self.dynamic_model, skip=True) - self.assertLess(np.max(np.abs(ori_val - self.dynamic_model.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - self.dynamic_model.all_weights[-2].numpy())), 1e-7) self.dynamic_model._weights = ori_weights diff --git a/tests/layers/test_layers_core_basedense_dropout.py b/tests/layers/test_layers_core_basedense_dropout.py index 7968822a7..f5a048fbf 100644 --- a/tests/layers/test_layers_core_basedense_dropout.py +++ b/tests/layers/test_layers_core_basedense_dropout.py @@ -83,7 +83,7 @@ def test_net2(self): self.assertEqual(len(self.dense1._info[0].layer.all_weights), 2) self.assertEqual(len(self.dense2._info[0].layer.all_weights), 1) - self.assertEqual(len(self.model.weights), 3) + self.assertEqual(len(self.model.all_weights), 3) # a special case self.model.release_memory() diff --git a/tests/layers/test_layers_lambda.py b/tests/layers/test_layers_lambda.py index 163339ca5..e7c0bc713 100644 --- a/tests/layers/test_layers_lambda.py +++ b/tests/layers/test_layers_lambda.py @@ -58,8 +58,8 @@ def forward(self, x): pred_y = model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, model.weights) - optimizer.apply_gradients(zip(gradients, model.weights)) + gradients = tape.gradient(loss, model.trainable_weights) + optimizer.apply_gradients(zip(gradients, model.trainable_weights)) print("epoch %d, loss %f" % (epoch, loss)) diff --git a/tests/models/test_auto_naming.py b/tests/models/test_auto_naming.py index 143dba963..fb8f03720 100644 --- a/tests/models/test_auto_naming.py +++ b/tests/models/test_auto_naming.py @@ -231,7 +231,7 @@ def test_layerlist(self): tl.layers.Dense(n_units=3, name='dense1')] )(inputs) model = tl.models.Model(inputs=inputs, outputs=layer1, name='layerlistmodel') - print([w.name for w in model.weights]) + print([w.name for w in model.all_weights]) test_flag = False except Exception as e: print(e) @@ -258,7 +258,7 @@ def forward(self, x): model_layer = tl.layers.ModelLayer(inner_model())(inputs) model = tl.models.Model(inputs=inputs, outputs=model_layer, name='modellayermodel') print(model) - print([w.name for w in model.weights]) + print([w.name for w in model.all_weights]) test_flag = False except Exception as e: print(e) @@ -273,7 +273,7 @@ def test_layerlist(self): tl.layers.Dense(n_units=3, name='dense1')] )(inputs) model = tl.models.Model(inputs=inputs, outputs=layer1, name='layerlistmodel') - print([w.name for w in model.weights]) + print([w.name for w in model.all_weights]) self.fail("Fail to detect duplicate name in layerlist") except Exception as e: print(e) diff --git a/tests/models/test_keras_save.py b/tests/models/test_keras_save.py index 2d40b31ef..211ef07ed 100644 --- a/tests/models/test_keras_save.py +++ b/tests/models/test_keras_save.py @@ -36,11 +36,11 @@ def call(self, inputs, training=None, mask=None): print([x.name for x in model.layers]) # print([x.name for x in model.inner.weights]) print('vgg1:') -print([x.name for x in model.inner.vgg1.weights]) +print([x.name for x in model.inner.vgg1.all_weights]) print([x.name for x in model.inner.vgg1.layers]) print('vgg2') print(model.inner.vgg2.get_layer('block1_conv1').kernel.name) -print([x.name for x in model.inner.vgg2.weights]) +print([x.name for x in model.inner.vgg2.all_weights]) print([x.name for x in model.inner.vgg2.layers]) model.save_weights('./keras_model.h5') diff --git a/tests/models/test_model_core.py b/tests/models/test_model_core.py index 4fab4110c..ac61a2097 100644 --- a/tests/models/test_model_core.py +++ b/tests/models/test_model_core.py @@ -82,8 +82,8 @@ def test_dynamic_basic(self): self.assertEqual(len(model_basic.all_layers), 7) self.assertEqual(model_basic._weights, None) - self.assertIsNotNone(model_basic.weights) - print([w.name for w in model_basic.weights]) + self.assertIsNotNone(model_basic.all_weights) + print([w.name for w in model_basic.all_weights]) # test model mode model_basic.train() @@ -151,8 +151,8 @@ def test_static_basic(self): self.assertEqual(len(model_basic.all_layers), 8) self.assertEqual(model_basic._weights, None) - self.assertIsNotNone(model_basic.weights) - print([w.name for w in model_basic.weights]) + self.assertIsNotNone(model_basic.all_weights) + print([w.name for w in model_basic.all_weights]) # test model mode model_basic.train() @@ -283,7 +283,7 @@ def forward(self, x): return x model = ill_model() - weights = model.weights + weights = model.all_weights except Exception as e: self.assertIsInstance(e, AttributeError) print(e) @@ -360,7 +360,7 @@ def forward(self, x): return x model = my_model() - weights = model.weights + weights = model.all_weights self.assertGreater(len(weights), 2) print(len(weights)) diff --git a/tests/models/test_model_save.py b/tests/models/test_model_save.py index b646b28bb..ba224ee25 100644 --- a/tests/models/test_model_save.py +++ b/tests/models/test_model_save.py @@ -92,38 +92,38 @@ def normal_save(self, model_basic): # hdf5 print('testing hdf5 saving...') - modify_val = np.zeros_like(model_basic.weights[-2].numpy()) - ori_val = model_basic.weights[-2].numpy() + modify_val = np.zeros_like(model_basic.all_weights[-2].numpy()) + ori_val = model_basic.all_weights[-2].numpy() model_basic.save_weights("./model_basic.h5") - model_basic.weights[-2].assign(modify_val) + model_basic.all_weights[-2].assign(modify_val) model_basic.load_weights("./model_basic.h5") - self.assertLess(np.max(np.abs(ori_val - model_basic.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - model_basic.all_weights[-2].numpy())), 1e-7) - model_basic.weights[-2].assign(modify_val) + model_basic.all_weights[-2].assign(modify_val) model_basic.load_weights("./model_basic.h5", format="hdf5") - self.assertLess(np.max(np.abs(ori_val - model_basic.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - model_basic.all_weights[-2].numpy())), 1e-7) - model_basic.weights[-2].assign(modify_val) + model_basic.all_weights[-2].assign(modify_val) model_basic.load_weights("./model_basic.h5", format="hdf5", in_order=False) - self.assertLess(np.max(np.abs(ori_val - model_basic.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - model_basic.all_weights[-2].numpy())), 1e-7) # npz print('testing npz saving...') model_basic.save_weights("./model_basic.npz", format='npz') - model_basic.weights[-2].assign(modify_val) + model_basic.all_weights[-2].assign(modify_val) model_basic.load_weights("./model_basic.npz") - model_basic.weights[-2].assign(modify_val) + model_basic.all_weights[-2].assign(modify_val) model_basic.load_weights("./model_basic.npz", format='npz') model_basic.save_weights("./model_basic.npz") - self.assertLess(np.max(np.abs(ori_val - model_basic.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - model_basic.all_weights[-2].numpy())), 1e-7) # npz_dict print('testing npz_dict saving...') model_basic.save_weights("./model_basic.npz", format='npz_dict') - model_basic.weights[-2].assign(modify_val) + model_basic.all_weights[-2].assign(modify_val) model_basic.load_weights("./model_basic.npz", format='npz_dict') - self.assertLess(np.max(np.abs(ori_val - model_basic.weights[-2].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - model_basic.all_weights[-2].numpy())), 1e-7) # ckpt try: @@ -163,12 +163,12 @@ def test_skip(self): print("testing dynamic skip load...") self.dynamic_basic.save_weights("./model_basic.h5") - ori_weights = self.dynamic_basic_skip.weights + ori_weights = self.dynamic_basic_skip.all_weights ori_val = ori_weights[1].numpy() modify_val = np.zeros_like(ori_val) - self.dynamic_basic_skip.weights[1].assign(modify_val) + self.dynamic_basic_skip.all_weights[1].assign(modify_val) self.dynamic_basic_skip.load_weights("./model_basic.h5", skip=True) - self.assertLess(np.max(np.abs(ori_val - self.dynamic_basic_skip.weights[1].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - self.dynamic_basic_skip.all_weights[1].numpy())), 1e-7) try: self.dynamic_basic_skip.load_weights("./model_basic.h5", in_order=False, skip=False) @@ -177,12 +177,12 @@ def test_skip(self): print("testing static skip load...") self.static_basic.save_weights("./model_basic.h5") - ori_weights = self.static_basic_skip.weights + ori_weights = self.static_basic_skip.all_weights ori_val = ori_weights[1].numpy() modify_val = np.zeros_like(ori_val) - self.static_basic_skip.weights[1].assign(modify_val) + self.static_basic_skip.all_weights[1].assign(modify_val) self.static_basic_skip.load_weights("./model_basic.h5", skip=True) - self.assertLess(np.max(np.abs(ori_val - self.static_basic_skip.weights[1].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - self.static_basic_skip.all_weights[1].numpy())), 1e-7) try: self.static_basic_skip.load_weights("./model_basic.h5", in_order=False, skip=False) @@ -264,7 +264,7 @@ def test_layerlist(self): model = tl.models.Model(inputs=inputs, outputs=layer1, name='layerlistmodel') model.save_weights("layerlist.h5") - tar_weight = model.get_layer(index=-1)[0].weights[0] + tar_weight = model.get_layer(index=-1)[0].all_weights[0] print(tar_weight.name) ori_val = tar_weight.numpy() modify_val = np.zeros_like(ori_val) diff --git a/tests/models/test_model_save_graph.py b/tests/models/test_model_save_graph.py index adbed8c83..95229938b 100644 --- a/tests/models/test_model_save_graph.py +++ b/tests/models/test_model_save_graph.py @@ -73,7 +73,7 @@ def test_save(self): print(MLP) n_epoch = 3 batch_size = 500 - train_weights = MLP.weights + train_weights = MLP.trainable_weights optimizer = tf.optimizers.Adam(lr=0.0001) for epoch in range(n_epoch): ## iterate the dataset n_epoch times @@ -118,7 +118,7 @@ def test_save(self): n_epoch = 3 batch_size = 500 - train_weights = MLP.weights + train_weights = MLP.trainable_weights optimizer = tf.optimizers.Adam(lr=0.0001) X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 784)) val_loss, val_acc, n_iter = 0, 0, 0 @@ -302,13 +302,13 @@ def test_lambda_layer_keras_model(self): self.assertEqual((output2 == output4).all(), True) self.assertEqual(M2.config, M4.config) - ori_weights = M4.weights + ori_weights = M4.all_weights ori_val = ori_weights[1].numpy() modify_val = np.zeros_like(ori_val) + 10 - M4.weights[1].assign(modify_val) + M4.all_weights[1].assign(modify_val) M4 = Model.load('M2_keras.hdf5') - self.assertLess(np.max(np.abs(ori_val - M4.weights[1].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - M4.all_weights[1].numpy())), 1e-7) def test_lambda_layer_keras_layer(self): input_shape = [100, 5] @@ -331,13 +331,13 @@ def test_lambda_layer_keras_layer(self): self.assertEqual((output1 == output3).all(), True) self.assertEqual(M1.config, M3.config) - ori_weights = M3.weights + ori_weights = M3.all_weights ori_val = ori_weights[1].numpy() modify_val = np.zeros_like(ori_val) + 10 - M3.weights[1].assign(modify_val) + M3.all_weights[1].assign(modify_val) M3 = Model.load('M1_keras.hdf5') - self.assertLess(np.max(np.abs(ori_val - M3.weights[1].numpy())), 1e-7) + self.assertLess(np.max(np.abs(ori_val - M3.all_weights[1].numpy())), 1e-7) class ElementWise_lambda_test(CustomTestCase): diff --git a/tests/pending/test_mnist_simple.py b/tests/pending/test_mnist_simple.py index 79d3dc8dd..5fe68c97b 100644 --- a/tests/pending/test_mnist_simple.py +++ b/tests/pending/test_mnist_simple.py @@ -44,7 +44,7 @@ def setUpClass(cls): # y_op = tf.argmax(tf.nn.softmax(y), 1) # define the optimizer - train_params = cls.network.all_params + train_params = cls.network.trainable_weights cls.train_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(cls.cost, var_list=train_params) @classmethod diff --git a/tests/performance_test/vgg/tl2-autograph.py b/tests/performance_test/vgg/tl2-autograph.py index a08688843..bd84bde8c 100644 --- a/tests/performance_test/vgg/tl2-autograph.py +++ b/tests/performance_test/vgg/tl2-autograph.py @@ -23,7 +23,7 @@ # training setting num_iter = NUM_ITERS batch_size = BATCH_SIZE -train_weights = vgg.weights +train_weights = vgg.trainable_weights optimizer = tf.optimizers.Adam(learning_rate=LERANING_RATE) loss_object = tl.cost.cross_entropy diff --git a/tests/performance_test/vgg/tl2-eager.py b/tests/performance_test/vgg/tl2-eager.py index f44a54ae2..401297a8e 100644 --- a/tests/performance_test/vgg/tl2-eager.py +++ b/tests/performance_test/vgg/tl2-eager.py @@ -23,7 +23,7 @@ # training setting num_iter = NUM_ITERS batch_size = BATCH_SIZE -train_weights = vgg.weights +train_weights = vgg.trainable_weights optimizer = tf.optimizers.Adam(learning_rate=LERANING_RATE) loss_object = tl.cost.cross_entropy From ad973fdf57ba511df2ed85ac6d206865f87919fd Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Mon, 13 May 2019 07:55:22 +0800 Subject: [PATCH 03/19] weights -> all_weights, trainable weights, nontrainable_weights --- docs/modules/files.rst | 8 ++-- docs/user/faq.rst | 2 +- docs/user/get_start_advance.rst | 2 +- docs/user/get_start_model.rst | 6 +-- tensorlayer/models/core.py | 8 ++-- tensorlayer/utils.py | 4 +- tests/layers/test_layers_convolution.py | 4 +- .../test_layers_core_basedense_dropout.py | 2 +- tests/layers/test_layers_recurrent.py | 44 +++++++++---------- 9 files changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/modules/files.rst b/docs/modules/files.rst index 833bcf5ef..c823b4db9 100644 --- a/docs/modules/files.rst +++ b/docs/modules/files.rst @@ -142,14 +142,14 @@ sake of cross-platform. Other file formats such as ``.npz`` are also available. .. code-block:: python ## save model as .h5 - tl.files.save_weights_to_hdf5('model.h5', network.weights) + tl.files.save_weights_to_hdf5('model.h5', network.all_weights) # restore model from .h5 (in order) - tl.files.load_hdf5_to_weights_in_order('model.h5', network.weights) + tl.files.load_hdf5_to_weights_in_order('model.h5', network.all_weights) # restore model from .h5 (by name) - tl.files.load_hdf5_to_weights('model.h5', network.weights) + tl.files.load_hdf5_to_weights('model.h5', network.all_weights) ## save model as .npz - tl.files.save_npz(network.weights , name='model.npz') + tl.files.save_npz(network.all_weights , name='model.npz') # restore model from .npz (method 1) load_params = tl.files.load_npz(name='model.npz') tl.files.assign_weights(sess, load_params, network) diff --git a/docs/user/faq.rst b/docs/user/faq.rst index 4988a2c1d..f1c98a075 100644 --- a/docs/user/faq.rst +++ b/docs/user/faq.rst @@ -58,7 +58,7 @@ After you get the variable list, you can define your optimizer like that so as t .. code-block:: python - train_weights = network.weights + train_weights = network.trainable_weights optimizer.apply_gradients(zip(grad, train_weights)) Logging diff --git a/docs/user/get_start_advance.rst b/docs/user/get_start_advance.rst index 1b2f5b125..20cdaa871 100644 --- a/docs/user/get_start_advance.rst +++ b/docs/user/get_start_advance.rst @@ -36,7 +36,7 @@ Get a part of CNN nn = tl.layers.Dense(n_units=100, name='out')(nn) model = tl.models.Model(inputs=ni, outputs=nn) # train your own classifier (only update the last layer) - train_params = model.get_layer('out').weights + train_params = model.get_layer('out').all_weights Reuse CNN ------------------ diff --git a/docs/user/get_start_model.rst b/docs/user/get_start_model.rst index 807baf112..670d325ef 100644 --- a/docs/user/get_start_model.rst +++ b/docs/user/get_start_model.rst @@ -149,11 +149,11 @@ We can get the specific weights by indexing or naming. .. code-block:: python # indexing - all_weights = MLP.weights - some_weights = MLP.weights[1:3] + all_weights = MLP.all_weights + some_weights = MLP.all_weights[1:3] # naming - some_weights = MLP.get_layer('dense1').weights + some_weights = MLP.get_layer('dense1').all_weights Save and restore model diff --git a/tensorlayer/models/core.py b/tensorlayer/models/core.py index 6189f653a..ae2a2641c 100644 --- a/tensorlayer/models/core.py +++ b/tensorlayer/models/core.py @@ -114,7 +114,7 @@ class Model(object): >>> outputs_s = M_static(data) Save and load weights - + >>> M_static.save_weights('./model_weights.h5') >>> M_static.load_weights('./model_weights.h5') @@ -460,7 +460,7 @@ def train(self): >>> net.train() """ - if self.is_train !=True: + if self.is_train != True: self.is_train = True self._set_mode_for_layers(True) @@ -847,7 +847,7 @@ def load_weights(self, filepath, format=None, in_order=True, skip=False): skip : bool Allow skipping weights whose name is mismatched between the file and model. Only useful when 'format' is 'hdf5' or 'npz_dict'. If 'skip' is True, 'in_order' argument will be ignored and those loaded weights - whose name is not found in model weights (self.weights) will be skipped. If 'skip' is False, error will + whose name is not found in model weights (self.all_weights) will be skipped. If 'skip' is False, error will occur when mismatch is found. Default is False. @@ -879,7 +879,7 @@ def load_weights(self, filepath, format=None, in_order=True, skip=False): format = filepath.split('.')[-1] if format == 'hdf5' or format == 'h5': - if skip ==True or in_order == False: + if skip == True or in_order == False: # load by weights name utils.load_hdf5_to_weights(filepath, self, skip) else: diff --git a/tensorlayer/utils.py b/tensorlayer/utils.py index cf56f49c2..d6b8e6d78 100644 --- a/tensorlayer/utils.py +++ b/tensorlayer/utils.py @@ -127,7 +127,7 @@ def fit( tf.summary.scalar('acc', train_acc, step=epoch) # FIXME : there seems to be an internal error in Tensorboard (misuse of tf.name_scope) # if tensorboard_weight_histograms is not None: - # for param in network.weights: + # for param in network.all_weights: # tf.summary.histogram(param.name, param, step=epoch) if (X_val is not None) and (y_val is not None): @@ -138,7 +138,7 @@ def fit( tf.summary.scalar('acc', val_acc, step=epoch) # FIXME : there seems to be an internal error in Tensorboard (misuse of tf.name_scope) # if tensorboard_weight_histograms is not None: - # for param in network.weights: + # for param in network.all_weights: # tf.summary.histogram(param.name, param, step=epoch) if epoch + 1 == 1 or (epoch + 1) % print_freq == 0: diff --git a/tests/layers/test_layers_convolution.py b/tests/layers/test_layers_convolution.py index 997e8cf57..0f5979d5b 100644 --- a/tests/layers/test_layers_convolution.py +++ b/tests/layers/test_layers_convolution.py @@ -127,7 +127,7 @@ def test_layer_n5(self): # # self.assertEqual(len(self.n1.all_layers), 2) # # self.assertEqual(len(self.n1.all_params), 2) # # self.assertEqual(self.n1.count_params(), 192) -# self.assertEqual(len(self.n1._info[0].layer.weights), 2) +# self.assertEqual(len(self.n1._info[0].layer.all_weights), 2) # self.assertEqual(self.n1.get_shape().as_list()[1:], [50, 32]) # # def test_layer_n2(self): @@ -135,7 +135,7 @@ def test_layer_n5(self): # # self.assertEqual(len(self.n2.all_layers), 3) # # self.assertEqual(len(self.n2.all_params), 4) # # self.assertEqual(self.n2.count_params(), 5344) -# self.assertEqual(len(self.n2._info[0].layer.weights), 2) +# self.assertEqual(len(self.n2._info[0].layer.all_weights), 2) # self.assertEqual(self.n2.get_shape().as_list()[1:], [25, 32]) # # # def test_layer_n3(self): diff --git a/tests/layers/test_layers_core_basedense_dropout.py b/tests/layers/test_layers_core_basedense_dropout.py index f5a048fbf..06f129956 100644 --- a/tests/layers/test_layers_core_basedense_dropout.py +++ b/tests/layers/test_layers_core_basedense_dropout.py @@ -134,7 +134,7 @@ def test_layerlist(self): )(innet) model = Model(inputs=innet, outputs=hlayer) - # for w in model.weights: + # for w in model.all_weights: # print(w.name) data = np.random.normal(size=[self.batch_size, self.inputs_shape[1]]).astype(np.float32) diff --git a/tests/layers/test_layers_recurrent.py b/tests/layers/test_layers_recurrent.py index fee1acb27..2e4dbab39 100644 --- a/tests/layers/test_layers_recurrent.py +++ b/tests/layers/test_layers_recurrent.py @@ -62,8 +62,8 @@ def test_basic_simplernn(self): pred_y, final_state = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -131,8 +131,8 @@ def forward(self, x): pred_y = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -165,8 +165,8 @@ def forward(self, x): pred_y = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -205,8 +205,8 @@ def forward(self, x): pred_y = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -232,8 +232,8 @@ def test_basic_lstmrnn(self): pred_y, final_h, final_c = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -259,8 +259,8 @@ def test_basic_grurnn(self): pred_y, final_h = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -292,8 +292,8 @@ def test_basic_birnn_simplernncell(self): self.assertEqual( r.get_shape().as_list(), [self.batch_size * self.num_steps, self.hidden_size + self.hidden_size + 1] ) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -326,8 +326,8 @@ def test_basic_birnn_lstmcell(self): self.assertEqual( r.get_shape().as_list(), [self.batch_size, self.num_steps, self.hidden_size + self.hidden_size + 1] ) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -362,8 +362,8 @@ def forward(self, x): pred_y = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -396,8 +396,8 @@ def test_stack_simplernn(self): pred_y = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) @@ -433,8 +433,8 @@ def test_stack_birnn_simplernncell(self): pred_y = rnn_model(self.data_x) loss = tl.cost.mean_squared_error(pred_y, self.data_y2) - gradients = tape.gradient(loss, rnn_model.weights) - optimizer.apply_gradients(zip(gradients, rnn_model.weights)) + gradients = tape.gradient(loss, rnn_model.trainable_weights) + optimizer.apply_gradients(zip(gradients, rnn_model.trainable_weights)) if (epoch + 1) % 10 == 0: print("epoch %d, loss %f" % (epoch, loss)) From 0af6056259e7b2ba664523cc3f8bfd5797bfeb63 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Mon, 13 May 2019 19:26:10 +0800 Subject: [PATCH 04/19] fix bugs, yapf --- tensorlayer/layers/normalization.py | 16 ++++++++-------- tensorlayer/models/core.py | 4 ++-- tests/models/test_keras_save.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tensorlayer/layers/normalization.py b/tensorlayer/layers/normalization.py index 5b03f5735..d8cec274c 100644 --- a/tensorlayer/layers/normalization.py +++ b/tensorlayer/layers/normalization.py @@ -258,8 +258,12 @@ def build(self, inputs_shape): if self.gamma_init: self.gamma = self._get_weights("gamma", shape=params_shape, init=self.gamma_init) - self.moving_mean = self._get_weights("moving_mean", shape=params_shape, init=self.moving_mean_init, trainable=False) - self.moving_var = self._get_weights("moving_var", shape=params_shape, init=self.moving_var_init, trainable=False) + self.moving_mean = self._get_weights( + "moving_mean", shape=params_shape, init=self.moving_mean_init, trainable=False + ) + self.moving_var = self._get_weights( + "moving_var", shape=params_shape, init=self.moving_var_init, trainable=False + ) def forward(self, inputs): mean, var = tf.nn.moments(inputs, self.axes, keepdims=True) @@ -268,12 +272,8 @@ def forward(self, inputs): self.moving_mean = moving_averages.assign_moving_average( self.moving_mean, mean, self.decay, zero_debias=False ) - self.moving_var = moving_averages.assign_moving_average( - self.moving_var, var, self.decay, zero_debias=False - ) - outputs = batch_normalization( - inputs, mean, var, self.beta, self.gamma, self.epsilon, self.data_format - ) + self.moving_var = moving_averages.assign_moving_average(self.moving_var, var, self.decay, zero_debias=False) + outputs = batch_normalization(inputs, mean, var, self.beta, self.gamma, self.epsilon, self.data_format) else: outputs = batch_normalization( inputs, self.moving_mean, self.moving_var, self.beta, self.gamma, self.epsilon, self.data_format diff --git a/tensorlayer/models/core.py b/tensorlayer/models/core.py index ae2a2641c..bb1035feb 100644 --- a/tensorlayer/models/core.py +++ b/tensorlayer/models/core.py @@ -460,7 +460,7 @@ def train(self): >>> net.train() """ - if self.is_train != True: + if self.is_train !=True: self.is_train = True self._set_mode_for_layers(True) @@ -879,7 +879,7 @@ def load_weights(self, filepath, format=None, in_order=True, skip=False): format = filepath.split('.')[-1] if format == 'hdf5' or format == 'h5': - if skip == True or in_order == False: + if skip ==True or in_order == False: # load by weights name utils.load_hdf5_to_weights(filepath, self, skip) else: diff --git a/tests/models/test_keras_save.py b/tests/models/test_keras_save.py index 211ef07ed..2d40b31ef 100644 --- a/tests/models/test_keras_save.py +++ b/tests/models/test_keras_save.py @@ -36,11 +36,11 @@ def call(self, inputs, training=None, mask=None): print([x.name for x in model.layers]) # print([x.name for x in model.inner.weights]) print('vgg1:') -print([x.name for x in model.inner.vgg1.all_weights]) +print([x.name for x in model.inner.vgg1.weights]) print([x.name for x in model.inner.vgg1.layers]) print('vgg2') print(model.inner.vgg2.get_layer('block1_conv1').kernel.name) -print([x.name for x in model.inner.vgg2.all_weights]) +print([x.name for x in model.inner.vgg2.weights]) print([x.name for x in model.inner.vgg2.layers]) model.save_weights('./keras_model.h5') From 37bb705a5bf5315768788bc40256fcf6cf5ae650 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Mon, 13 May 2019 19:59:37 +0800 Subject: [PATCH 05/19] fix bugs --- CHANGELOG.md | 2 ++ tensorlayer/layers/convolution/separable_conv.py | 4 ++-- .../layers/convolution/simplified_deconv.py | 4 ++-- tensorlayer/layers/lambda_layers.py | 12 ++++++------ tensorlayer/layers/recurrent.py | 14 +++++++------- tests/files/test_utils_saveload.py | 12 ++++++------ tests/models/test_model_core.py | 8 ++++---- 7 files changed, 29 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eca64385a..e932d4a6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,8 @@ To release a new version, please update the changelog as followed: ### Changed - remove `tl.layers.initialize_global_variables(sess)` (PR #931) +- change `tl.layers.core`, `tl.models.core` (PR #966) + - change `weights` into `all_weights`, `trainable_weights`, `nontrainable_weights` ### Dependencies Update - nltk>=3.3,<3.4 => nltk>=3.3,<3.5 (PR #892) diff --git a/tensorlayer/layers/convolution/separable_conv.py b/tensorlayer/layers/convolution/separable_conv.py index d2fbf3515..b6ae62446 100644 --- a/tensorlayer/layers/convolution/separable_conv.py +++ b/tensorlayer/layers/convolution/separable_conv.py @@ -156,7 +156,7 @@ def build(self, inputs_shape): ) # initialize weights outputs_shape = _out.shape # self._add_weights(self.layer.weights) - self._weights = self.layer.all_weights + self._trainable_weights = self.layer.weights def forward(self, inputs): outputs = self.layer(inputs) @@ -302,7 +302,7 @@ def build(self, inputs_shape): tf.convert_to_tensor(np.random.uniform(size=list(inputs_shape)), dtype=np.float) ) # initialize weights outputs_shape = _out.shape - self._weights = self.layer.all_weights + self._trainable_weights = self.layer.weights def forward(self, inputs): outputs = self.layer(inputs) diff --git a/tensorlayer/layers/convolution/simplified_deconv.py b/tensorlayer/layers/convolution/simplified_deconv.py index 7c147b729..847062859 100644 --- a/tensorlayer/layers/convolution/simplified_deconv.py +++ b/tensorlayer/layers/convolution/simplified_deconv.py @@ -141,7 +141,7 @@ def build(self, inputs_shape): tf.convert_to_tensor(np.random.uniform(size=inputs_shape), dtype=np.float32) ) #np.random.uniform([1] + list(inputs_shape))) # initialize weights outputs_shape = _out.shape - self._weights = self.layer.all_weights + self._trainable_weights = self.layer.weights def forward(self, inputs): outputs = self.layer(inputs) @@ -264,7 +264,7 @@ def build(self, inputs_shape): ) #self.layer(np.random.uniform([1] + list(inputs_shape))) # initialize weights outputs_shape = _out.shape # self._add_weights(self.layer.weights) - self._weights = self.layer.all_weights + self._trainable_weights = self.layer.weights def forward(self, inputs): outputs = self.layer(inputs) diff --git a/tensorlayer/layers/lambda_layers.py b/tensorlayer/layers/lambda_layers.py index 8bb035c9f..8f2438fc1 100644 --- a/tensorlayer/layers/lambda_layers.py +++ b/tensorlayer/layers/lambda_layers.py @@ -111,14 +111,14 @@ def __init__( super(Lambda, self).__init__(name=name) self.fn = fn - self._weights = fn_weights if fn_weights is not None else [] + self._trainable_weights = fn_weights if fn_weights is not None else [] self.fn_args = fn_args if fn_args is not None else {} try: fn_name = repr(self.fn) except: fn_name = 'name not available' - logging.info("Lambda %s: func: %s, len_weights: %s" % (self.name, fn_name, len(self._weights))) + logging.info("Lambda %s: func: %s, len_weights: %s" % (self.name, fn_name, len(self._trainable_weights))) self.build() self._built = True @@ -134,7 +134,7 @@ def __repr__(self): except: fn_name = 'name not available' return s.format( - classname=self.__class__.__name__, fn_name=fn_name, len_weights=len(self._weights), **self.__dict__ + classname=self.__class__.__name__, fn_name=fn_name, len_weights=len(self._trainable_weights), **self.__dict__ ) def build(self, inputs_shape=None): @@ -233,14 +233,14 @@ def __init__( super(ElementwiseLambda, self).__init__(name=name) self.fn = fn - self._weights = fn_weights if fn_weights is not None else [] + self._trainable_weights = fn_weights if fn_weights is not None else [] self.fn_args = fn_args if fn_args is not None else {} try: fn_name = repr(self.fn) except: fn_name = 'name not available' - logging.info("ElementwiseLambda %s: func: %s, len_weights: %s" % (self.name, fn_name, len(self._weights))) + logging.info("ElementwiseLambda %s: func: %s, len_weights: %s" % (self.name, fn_name, len(self._trainable_weights))) self.build() self._built = True @@ -256,7 +256,7 @@ def __repr__(self): except: fn_name = 'name not available' return s.format( - classname=self.__class__.__name__, fn_name=fn_name, len_weights=len(self._weights), **self.__dict__ + classname=self.__class__.__name__, fn_name=fn_name, len_weights=len(self._trainable_weights), **self.__dict__ ) def build(self, inputs_shape=None): diff --git a/tensorlayer/layers/recurrent.py b/tensorlayer/layers/recurrent.py index acc6dba05..16b7208d0 100644 --- a/tensorlayer/layers/recurrent.py +++ b/tensorlayer/layers/recurrent.py @@ -149,10 +149,10 @@ def build(self, inputs_shape): with tf.name_scope(self.name) as scope: self.cell.build(tuple(inputs_shape)) - if self._weights is None: - self._weights = list() + if self._trainable_weights is None: + self._trainable_weights = list() for var in self.cell.trainable_variables: - self._weights.append(var) + self._trainable_weights.append(var) # @tf.function def forward(self, inputs, initial_state=None, **kwargs): @@ -341,12 +341,12 @@ def build(self, inputs_shape): self.fw_cell.build(tuple(inputs_shape)) self.bw_cell.build(tuple(inputs_shape)) - if self._weights is None: - self._weights = list() + if self._trainable_weights is None: + self._trainable_weights = list() for var in self.fw_cell.trainable_variables: - self._weights.append(var) + self._trainable_weights.append(var) for var in self.bw_cell.trainable_variables: - self._weights.append(var) + self._trainable_weights.append(var) # @tf.function def forward(self, inputs, fw_initial_state=None, bw_initial_state=None, **kwargs): diff --git a/tests/files/test_utils_saveload.py b/tests/files/test_utils_saveload.py index 2cd329735..58a1d374a 100644 --- a/tests/files/test_utils_saveload.py +++ b/tests/files/test_utils_saveload.py @@ -79,12 +79,12 @@ def test_hdf5(self): tl.files.load_hdf5_to_weights("./model_basic.h5", self.static_model) self.assertLess(np.max(np.abs(ori_val - self.static_model.all_weights[-2].numpy())), 1e-7) - ori_weights = self.static_model._weights - self.static_model._weights = self.static_model._weights[1:] + ori_weights = self.static_model._all_weights + self.static_model._all_weights = self.static_model._all_weights[1:] self.static_model.all_weights[-2].assign(modify_val) tl.files.load_hdf5_to_weights("./model_basic.h5", self.static_model, skip=True) self.assertLess(np.max(np.abs(ori_val - self.static_model.all_weights[-2].numpy())), 1e-7) - self.static_model._weights = ori_weights + self.static_model._all_weights = ori_weights def test_npz(self): modify_val = np.zeros_like(self.dynamic_model.all_weights[-2].numpy()) @@ -104,12 +104,12 @@ def test_npz_dict(self): tl.files.load_and_assign_npz_dict("./model_basic.npz", self.dynamic_model) self.assertLess(np.max(np.abs(ori_val - self.dynamic_model.all_weights[-2].numpy())), 1e-7) - ori_weights = self.dynamic_model._weights - self.dynamic_model._weights = self.static_model._weights[1:] + ori_weights = self.dynamic_model._all_weights + self.dynamic_model._all_weights = self.static_model._all_weights[1:] self.dynamic_model.all_weights[-2].assign(modify_val) tl.files.load_and_assign_npz_dict("./model_basic.npz", self.dynamic_model, skip=True) self.assertLess(np.max(np.abs(ori_val - self.dynamic_model.all_weights[-2].numpy())), 1e-7) - self.dynamic_model._weights = ori_weights + self.dynamic_model._all_weights = ori_weights if __name__ == '__main__': diff --git a/tests/models/test_model_core.py b/tests/models/test_model_core.py index ac61a2097..caf3044b2 100644 --- a/tests/models/test_model_core.py +++ b/tests/models/test_model_core.py @@ -70,7 +70,7 @@ def test_dynamic_basic(self): # test empty model before calling self.assertEqual(model_basic.is_train, None) - self.assertEqual(model_basic._weights, None) + self.assertEqual(model_basic._all_weights, None) self.assertEqual(model_basic._inputs, None) self.assertEqual(model_basic._outputs, None) self.assertEqual(model_basic._model_layer, None) @@ -80,7 +80,7 @@ def test_dynamic_basic(self): # test layer and weights access all_layers = model_basic.all_layers self.assertEqual(len(model_basic.all_layers), 7) - self.assertEqual(model_basic._weights, None) + self.assertEqual(model_basic._all_weights, None) self.assertIsNotNone(model_basic.all_weights) print([w.name for w in model_basic.all_weights]) @@ -139,7 +139,7 @@ def test_static_basic(self): # test empty model before calling self.assertEqual(model_basic.is_train, None) - self.assertEqual(model_basic._weights, None) + self.assertEqual(model_basic._all_weights, None) self.assertIsNotNone(model_basic._inputs) self.assertIsNotNone(model_basic._outputs) self.assertEqual(model_basic._model_layer, None) @@ -149,7 +149,7 @@ def test_static_basic(self): # test layer and weights access all_layers = model_basic.all_layers self.assertEqual(len(model_basic.all_layers), 8) - self.assertEqual(model_basic._weights, None) + self.assertEqual(model_basic._all_weights, None) self.assertIsNotNone(model_basic.all_weights) print([w.name for w in model_basic.all_weights]) From be27eb2aa83a19f652b0a9072daaaccbe221453e Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Mon, 13 May 2019 20:02:10 +0800 Subject: [PATCH 06/19] fix bugs --- tensorlayer/models/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorlayer/models/core.py b/tensorlayer/models/core.py index bb1035feb..c811b9648 100644 --- a/tensorlayer/models/core.py +++ b/tensorlayer/models/core.py @@ -356,7 +356,9 @@ def all_layers(self): # dynamic model self._all_layers = list() attr_list = [attr for attr in dir(self) if attr[:2] != "__"] - attr_list.remove("weights") + attr_list.remove("all_weights") + attr_list.remove("trainable_weights") + attr_list.remove("nontrainable_weights") attr_list.remove("all_layers") for idx, attr in enumerate(attr_list): try: From 760b219255be35555afc8c59b41e43684e6d1a23 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Mon, 13 May 2019 20:17:25 +0800 Subject: [PATCH 07/19] fix bugs --- tensorlayer/layers/lambda_layers.py | 10 +++++++--- tests/layers/test_layers_core_basedense_dropout.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tensorlayer/layers/lambda_layers.py b/tensorlayer/layers/lambda_layers.py index 8f2438fc1..13bc3ecbe 100644 --- a/tensorlayer/layers/lambda_layers.py +++ b/tensorlayer/layers/lambda_layers.py @@ -134,7 +134,8 @@ def __repr__(self): except: fn_name = 'name not available' return s.format( - classname=self.__class__.__name__, fn_name=fn_name, len_weights=len(self._trainable_weights), **self.__dict__ + classname=self.__class__.__name__, fn_name=fn_name, len_weights=len(self._trainable_weights), + **self.__dict__ ) def build(self, inputs_shape=None): @@ -240,7 +241,9 @@ def __init__( fn_name = repr(self.fn) except: fn_name = 'name not available' - logging.info("ElementwiseLambda %s: func: %s, len_weights: %s" % (self.name, fn_name, len(self._trainable_weights))) + logging.info( + "ElementwiseLambda %s: func: %s, len_weights: %s" % (self.name, fn_name, len(self._trainable_weights)) + ) self.build() self._built = True @@ -256,7 +259,8 @@ def __repr__(self): except: fn_name = 'name not available' return s.format( - classname=self.__class__.__name__, fn_name=fn_name, len_weights=len(self._trainable_weights), **self.__dict__ + classname=self.__class__.__name__, fn_name=fn_name, len_weights=len(self._trainable_weights), + **self.__dict__ ) def build(self, inputs_shape=None): diff --git a/tests/layers/test_layers_core_basedense_dropout.py b/tests/layers/test_layers_core_basedense_dropout.py index 06f129956..19178f5d6 100644 --- a/tests/layers/test_layers_core_basedense_dropout.py +++ b/tests/layers/test_layers_core_basedense_dropout.py @@ -73,8 +73,8 @@ def test_net1(self): def test_net2(self): # test weights - self.assertEqual(self.innet._info[0].layer.all_weights, None) - self.assertEqual(self.dropout1._info[0].layer.all_weights, None) + self.assertEqual(self.innet._info[0].layer.all_weights, []) + self.assertEqual(self.dropout1._info[0].layer.all_weights, []) self.assertEqual(self.dense1._info[0].layer.all_weights[0].get_shape().as_list(), [784, 800]) self.assertEqual(self.dense1._info[0].layer.all_weights[1].get_shape().as_list(), [ 800, From cc1feb79d3fb0b71a87c789cf7fd69628f47b8a1 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 11:57:08 +0800 Subject: [PATCH 08/19] alpha version, update network config --- tensorlayer/files/utils.py | 219 ++++++++++++++++++++----------------- tensorlayer/models/core.py | 42 ++++++- 2 files changed, 155 insertions(+), 106 deletions(-) diff --git a/tensorlayer/files/utils.py b/tensorlayer/files/utils.py index 72fcb1824..266113cca 100644 --- a/tensorlayer/files/utils.py +++ b/tensorlayer/files/utils.py @@ -31,6 +31,7 @@ from tensorflow.python.util.tf_export import keras_export from tensorflow.python.util import serialization import json +import datetime # from six.moves import zip @@ -92,29 +93,29 @@ def str2func(s): return expr -def net2static_graph(network): - saved_file = dict() - if network._NameNone is True: - saved_file.update({"name": None}) - else: - saved_file.update({"name": network.name}) - if not isinstance(network.inputs, list): - saved_file.update({"inputs": network.inputs._info[0].name}) - else: - saved_inputs = [] - for saved_input in network.inputs: - saved_inputs.append(saved_input._info[0].name) - saved_file.update({"inputs": saved_inputs}) - if not isinstance(network.outputs, list): - saved_file.update({"outputs": network.outputs._info[0].name}) - else: - saved_outputs = [] - for saved_output in network.outputs: - saved_outputs.append(saved_output._info[0].name) - saved_file.update({"outputs": saved_outputs}) - saved_file.update({"config": network.config}) - - return saved_file +# def net2static_graph(network): +# saved_file = dict() +# # if network._NameNone is True: +# # saved_file.update({"name": None}) +# # else: +# # saved_file.update({"name": network.name}) +# # if not isinstance(network.inputs, list): +# # saved_file.update({"inputs": network.inputs._info[0].name}) +# # else: +# # saved_inputs = [] +# # for saved_input in network.inputs: +# # saved_inputs.append(saved_input._info[0].name) +# # saved_file.update({"inputs": saved_inputs}) +# # if not isinstance(network.outputs, list): +# # saved_file.update({"outputs": network.outputs._info[0].name}) +# # else: +# # saved_outputs = [] +# # for saved_output in network.outputs: +# # saved_outputs.append(saved_output._info[0].name) +# # saved_file.update({"outputs": saved_outputs}) +# saved_file.update({"config": network.config}) +# +# return saved_file @keras_export('keras.models.save_model') @@ -149,7 +150,7 @@ def load_keras_model(model_config): return model -def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False): +def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False, customized_data=None): """Save the architecture of TL model into a hdf5 file. Support saving model weights. Parameters @@ -160,6 +161,8 @@ def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False): The name of model file. save_weights : bool Whether to save model weights. + customized_data : dict + The user customized meta data. Examples -------- @@ -177,11 +180,22 @@ def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False): logging.info("[*] Saving TL model into {}, saving weights={}".format(filepath, save_weights)) - saved_file = net2static_graph(network) - saved_file_str = str(saved_file) + network_config = network.config # net2static_graph(network) + network_config_str = str(network_config) + customized_data_str = str(customized_data) + version_info = { + "tensorlayer_version": tl.__version__, + "backend": "tensorflow", + "backend_version": tf.__version__, + "training_device": "gpu", + "save_date": datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() + } + version_info_str = str(version_info) with h5py.File(filepath, 'w') as f: - f.attrs["model_structure"] = saved_file_str.encode('utf8') + f.attrs["network_config"] = network_config_str.encode('utf8') + f.attrs["customized_data"] = customized_data_str.encode('utf8') + f.attrs["version_info"] = version_info_str.encode('utf8') if save_weights: _save_weights_to_hdf5_group(f, network.all_layers) f.flush() @@ -237,29 +251,15 @@ def eval_layer(layer_kwargs): raise RuntimeError("Unknown layer type.") -def static_graph2net(saved_file): +def static_graph2net(network_config): layer_dict = {} - model_name = saved_file['name'] - inputs_tensors = saved_file['inputs'] - outputs_tensors = saved_file['outputs'] - all_args = saved_file['config'] - tf_version = saved_file['config'].pop(0)['tf_version'] - tl_version = saved_file['config'].pop(0)['tl_version'] - if tf_version != tf.__version__: - logging.warning( - "Saved model uses tensorflow version {}, but now you are using tensorflow version {}".format( - tf_version, tf.__version__ - ) - ) - if tl_version != tl.__version__: - logging.warning( - "Saved model uses tensorlayer version {}, but now you are using tensorlayer version {}".format( - tl_version, tl.__version__ - ) - ) + model_name = network_config["name"] + inputs_tensors = network_config["inputs"] + outputs_tensors = network_config["outputs"] + all_args = network_config["model_architecture"] for idx, layer_kwargs in enumerate(all_args): - layer_class = layer_kwargs['class'] # class of current layer - prev_layers = layer_kwargs.pop('prev_layer') # name of previous layers + layer_class = layer_kwargs["class"] # class of current layer + prev_layers = layer_kwargs.pop("prev_layer") # name of previous layers net = eval_layer(layer_kwargs) if layer_class in tl.layers.inputs.__all__: net = net._nodes[0].out_tensors[0] @@ -312,11 +312,30 @@ def load_hdf5_graph(filepath='model.hdf5', load_weights=False): - see ``tl.files.save_hdf5_graph`` """ logging.info("[*] Loading TL model from {}, loading weights={}".format(filepath, load_weights)) + f = h5py.File(filepath, 'r') - saved_file_str = f.attrs["model_structure"].decode('utf8') - saved_file = eval(saved_file_str) - M = static_graph2net(saved_file) + version_info_str = f.attrs["version_info"].decode('utf8') + version_info = eval(version_info_str) + backend_version = version_info["backend_version"] + tensorlayer_version =version_info["tensorlayer_version"] + if backend_version != tf.__version__: + logging.warning( + "Saved model uses tensorflow version {}, but now you are using tensorflow version {}".format( + backend_version, tf.__version__ + ) + ) + if tensorlayer_version != tl.__version__: + logging.warning( + "Saved model uses tensorlayer version {}, but now you are using tensorlayer version {}".format( + tensorlayer_version, tl.__version__ + ) + ) + + network_config_str = f.attrs["network_config"].decode('utf8') + network_config = eval(network_config_str) + + M = static_graph2net(network_config) if load_weights: if not ('layer_names' in f.attrs.keys()): raise RuntimeError("Saved model does not contain weights.") @@ -329,55 +348,55 @@ def load_hdf5_graph(filepath='model.hdf5', load_weights=False): return M -def load_pkl_graph(name='model.pkl'): - """Restore TL model archtecture from a a pickle file. No parameters be restored. - - Parameters - ----------- - name : str - The name of graph file. - - Returns - -------- - network : TensorLayer Model. - - Examples - -------- - >>> # It is better to use load_hdf5_graph - """ - logging.info("[*] Loading TL graph from {}".format(name)) - with open(name, 'rb') as file: - saved_file = pickle.load(file) - - M = static_graph2net(saved_file) - - return M - - -def save_pkl_graph(network, name='model.pkl'): - """Save the architecture of TL model into a pickle file. No parameters be saved. - - Parameters - ----------- - network : TensorLayer layer - The network to save. - name : str - The name of graph file. - - Example - -------- - >>> # It is better to use save_hdf5_graph - """ - if network.outputs is None: - raise AssertionError("save_graph not support dynamic mode yet") - - logging.info("[*] Saving TL graph into {}".format(name)) - - saved_file = net2static_graph(network) - - with open(name, 'wb') as file: - pickle.dump(saved_file, file, protocol=pickle.HIGHEST_PROTOCOL) - logging.info("[*] Saved graph") +# def load_pkl_graph(name='model.pkl'): +# """Restore TL model archtecture from a a pickle file. No parameters be restored. +# +# Parameters +# ----------- +# name : str +# The name of graph file. +# +# Returns +# -------- +# network : TensorLayer Model. +# +# Examples +# -------- +# >>> # It is better to use load_hdf5_graph +# """ +# logging.info("[*] Loading TL graph from {}".format(name)) +# with open(name, 'rb') as file: +# saved_file = pickle.load(file) +# +# M = static_graph2net(saved_file) +# +# return M +# +# +# def save_pkl_graph(network, name='model.pkl'): +# """Save the architecture of TL model into a pickle file. No parameters be saved. +# +# Parameters +# ----------- +# network : TensorLayer layer +# The network to save. +# name : str +# The name of graph file. +# +# Example +# -------- +# >>> # It is better to use save_hdf5_graph +# """ +# if network.outputs is None: +# raise AssertionError("save_graph not support dynamic mode yet") +# +# logging.info("[*] Saving TL graph into {}".format(name)) +# +# saved_file = net2static_graph(network) +# +# with open(name, 'wb') as file: +# pickle.dump(saved_file, file, protocol=pickle.HIGHEST_PROTOCOL) +# logging.info("[*] Saved graph") # Load dataset functions diff --git a/tensorlayer/models/core.py b/tensorlayer/models/core.py index c811b9648..369598970 100644 --- a/tensorlayer/models/core.py +++ b/tensorlayer/models/core.py @@ -437,15 +437,43 @@ def config(self): if self._config is not None and len(self._config) > 0: return self._config else: - _config = [] - _config.append({"tf_version": tf.__version__}) - _config.append({"tl_version": tl.__version__}) + # _config = [] + _config = {} + if self._NameNone is True: + _config.update({"name": None}) + else: + _config.update({"name": self.name}) + # versionInfo = { + # "tensorlayer_version": tl.__version__, + # "backend": "tensorflow", + # "backend_version": tf.__version__, + # "training_device": "gpu", + # } + # _config.update(versionInfo) # if self.outputs is None: # raise RuntimeError( # "Dynamic mode does not support config yet." # ) + model_architecture = [] for layer in self.all_layers: - _config.append(layer.config) + model_architecture.append(layer.config) + _config["model_architecture"] = model_architecture + if self.inputs is not None: + if not isinstance(self.inputs, list): + _config.update({"inputs": self.inputs._info[0].name}) + else: + config_inputs = [] + for config_input in self.inputs: + config_inputs.append(config_input._info[0].name) + _config.update({"inputs": config_inputs}) + if self.outputs is not None: + if not isinstance(self.outputs, list): + _config.update({"outputs": self.outputs._info[0].name}) + else: + config_outputs = [] + for config_output in self.outputs: + config_outputs.append(config_output._info[0].name) + _config.update({"outputs": config_outputs}) if self._nodes_fixed or self.outputs is None: self._config = _config @@ -714,7 +742,7 @@ def release_memory(self): for layer in self.all_layers: layer._release_memory() - def save(self, filepath, save_weights=True): + def save(self, filepath, save_weights=True, customized_data=None): """ Save model into a given file. This function save can save both the architecture of neural networks and weights (optional). @@ -726,6 +754,8 @@ def save(self, filepath, save_weights=True): Filename into which the model will be saved. save_weights : bool Whether to save model weights. + customized_data : dict + The user customized meta data. Examples -------- @@ -739,7 +769,7 @@ def save(self, filepath, save_weights=True): raise RuntimeError( "Model save() not support dynamic mode yet.\nHint: you can use Model save_weights() to save the weights in dynamic mode." ) - utils.save_hdf5_graph(network=self, filepath=filepath, save_weights=save_weights) + utils.save_hdf5_graph(network=self, filepath=filepath, save_weights=save_weights, customized_data=customized_data) @staticmethod def load(filepath, load_weights=True): From 372aa6b4c756bd704cbda9815186d49605732848 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 12:06:14 +0800 Subject: [PATCH 09/19] fix bug --- tensorlayer/files/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorlayer/files/utils.py b/tensorlayer/files/utils.py index 266113cca..0d4acedbe 100644 --- a/tensorlayer/files/utils.py +++ b/tensorlayer/files/utils.py @@ -74,7 +74,7 @@ 'load_hdf5_to_weights', 'save_hdf5_graph', 'load_hdf5_graph', - 'net2static_graph', + # 'net2static_graph', 'static_graph2net', # 'save_pkl_graph', # 'load_pkl_graph', From e79b8d629ab9dd7e3cc7ba942455b161624fa990 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 12:17:25 +0800 Subject: [PATCH 10/19] add files --- CHANGELOG.md | 463 ++++++++++++++++++ .../text_generation/tutorial_generate_text.py | 332 +++++++++++++ 2 files changed, 795 insertions(+) create mode 100644 CHANGELOG.md create mode 100644 examples/text_generation/tutorial_generate_text.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..630b2b2db --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,463 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + + + + +## [2.0.1] - 2019-5-17 + + +A maintain release. + +### Changed +- remove `tl.layers.initialize_global_variables(sess)` (PR #931) +- support `trainable_weights` (PR #966) + +### Added + - Layer + - `InstanceNorm`, `InstanceNorm1d`, `InstanceNorm2d`, `InstanceNorm3d` (PR #963) + +### Changed +- remove `tl.layers.initialize_global_variables(sess)` (PR #931) +- update `tutorial_generate_text.py`, `tutorial_ptb_lstm.py`. remove `tutorial_ptb_lstm_state_is_tuple.py` (PR #958) +- change `tl.layers.core`, `tl.models.core` (PR #966) +- change `weights` into `all_weights`, `trainable_weights`, `nontrainable_weights` + +### Dependencies Update +- nltk>=3.3,<3.4 => nltk>=3.3,<3.5 (PR #892) +- pytest>=3.6,<3.11 => pytest>=3.6,<4.1 (PR #889) +- yapf>=0.22,<0.25 => yapf==0.25.0 (PR #896) +- imageio==2.5.0 progressbar2==3.39.3 scikit-learn==0.21.0 scikit-image==0.15.0 scipy==1.2.1 wrapt==1.11.1 pymongo==3.8.0 sphinx==2.0.1 wrapt==1.11.1 opencv-python==4.1.0.25 requests==2.21.0 tqdm==4.31.1 lxml==4.3.3 pycodestyle==2.5.0 sphinx==2.0.1 yapf==0.27.0(PR #967) + +### Fixed +- fix docs of models @zsdonghao #957 +- In `BatchNorm`, keep dimensions of mean and variance to suit `channels first` (PR #963) + +### Contributors +- @warshallrho: #PR966 +- @zsdonghao: #931 +- @yd-yin: #963 +- @1FengL: #958 +- @dvklopfenstein: #971 + + +## [2.0.0] - 2019-05-04 + +To many PR for this update, please check [here](https://github.com/tensorlayer/tensorlayer/releases/tag/2.0.0) for more details. + +### Changed +* update for TensorLayer 2.0.0 alpha version (PR #952) +* support TensorFlow 2.0.0-alpha +* support both static and dynamic model building + +### Dependencies Update +- tensorflow>=1.6,<1.13 => tensorflow>=2.0.0-alpha (PR #952) +- h5py>=2.9 (PR #952) +- cloudpickle>=0.8.1 (PR #952) +- remove matplotlib + +### Contributors +- @zsdonghao +- @JingqingZ +- @ChrisWu1997 +- @warshallrho + + +## [1.11.1] - 2018-11-15 + +### Changed +* guide for pose estimation - flipping (PR #884) +* cv2 transform support 2 modes (PR #885) + +### Dependencies Update +- pytest>=3.6,<3.9 => pytest>=3.6,<3.10 (PR #874) +- requests>=2.19,<2.20 => requests>=2.19,<2.21 (PR #874) +- tqdm>=4.23,<4.28 => tqdm>=4.23,<4.29 (PR #878) +- pytest>=3.6,<3.10 => pytest>=3.6,<3.11 (PR #886) +- pytest-xdist>=1.22,<1.24 => pytest-xdist>=1.22,<1.25 (PR #883) +- tensorflow>=1.6,<1.12 => tensorflow>=1.6,<1.13 (PR #886) + +### Contributors +- @zsdonghao: #884 #885 + +## [1.11.0] - 2018-10-18 + +### Added +- Layer: + - Release `GroupNormLayer` (PR #850) +- Image affine transformation APIs + - `affine_rotation_matrix` (PR #857) + - `affine_horizontal_flip_matrix` (PR #857) + - `affine_vertical_flip_matrix` (PR #857) + - `affine_shift_matrix` (PR #857) + - `affine_shear_matrix` (PR #857) + - `affine_zoom_matrix` (PR #857) + - `affine_transform_cv2` (PR #857) + - `affine_transform_keypoints` (PR #857) +- Affine transformation tutorial + - `examples/data_process/tutorial_fast_affine_transform.py` (PR #857) + +### Changed +- BatchNormLayer: support `data_format` + +### Dependencies Update +- matplotlib>=2.2,<2.3 => matplotlib>=2.2,<3.1 (PR #845) +- pydocstyle>=2.1,<2.2 => pydocstyle>=2.1,<3.1 (PR #866) +- scikit-learn>=0.19,<0.20 => scikit-learn>=0.19,<0.21 (PR #851) +- sphinx>=1.7,<1.8 => sphinx>=1.7,<1.9 (PR #842) +- tensorflow>=1.6,<1.11 => tensorflow>=1.6,<1.12 (PR #853) +- tqdm>=4.23,<4.26 => tqdm>=4.23,<4.28 (PR #862 & #868) +- yapf>=0.22,<0.24 => yapf>=0.22,<0.25 (PR #829) + +### Fixed +- Correct offset calculation in `tl.prepro.transform_matrix_offset_center` (PR #855) + +### Contributors +- @2wins: #850 #855 +- @DEKHTIARJonathan: #853 +- @zsdonghao: #857 +- @luomai: #857 + +## [1.10.1] - 2018-09-07 + +### Added +- unittest `tests\test_timeout.py` has been added to ensure the network creation process does not freeze. + +### Changed + - remove 'tensorboard' param, replaced by 'tensorboard_dir' in `tensorlayer/utils.py` with customizable tensorboard directory (PR #819) + +### Removed +- TL Graph API removed. Memory Leaks Issues with this API, will be fixed and integrated in TL 2.0 (PR #818) + +### Fixed +- Issue #817 fixed: TL 1.10.0 - Memory Leaks and very slow network creation. + +### Dependencies Update +- autopep8>=1.3,<1.4 => autopep8>=1.3,<1.5 (PR #815) +- imageio>=2.3,<2.4 => imageio>=2.3,<2.5 (PR #823) +- pytest>=3.6,<3.8 => pytest>=3.6,<3.9 (PR #823) +- pytest-cov>=2.5,<2.6 => pytest-cov>=2.5,<2.7 (PR #820) + +### Contributors +- @DEKHTIARJonathan: #815 #818 #820 #823 +- @ndiy: #819 +- @zsdonghao: #818 + + +## [1.10.0] - 2018-09-02 + +### Added +- API: + - Add `tl.model.vgg19` (PR #698) + - Add `tl.logging.contrib.hyperdash` (PR #739) + - Add `tl.distributed.trainer` (PR #700) + - Add `prefetch_buffer_size` to the `tl.distributed.trainer` (PR #766) + - Add `tl.db.TensorHub` (PR ＃751) + - Add `tl.files.save_graph` (PR ＃751) + - Add `tl.files.load_graph_` (PR ＃751) + - Add `tl.files.save_graph_and_params` (PR ＃751) + - Add `tl.files.load_graph_and_params` (PR ＃751) + - Add `tl.prepro.keypoint_random_xxx` (PR #787) +- Documentation: + - Add binary, ternary and dorefa links (PR #711) + - Update input scale of VGG16 and VGG19 to 0~1 (PR #736) + - Update database (PR ＃751) +- Layer: + - Release SwitchNormLayer (PR #737) + - Release QuanConv2d, QuanConv2dWithBN, QuanDenseLayer, QuanDenseLayerWithBN (PR#735) + - Update Core Layer to support graph (PR ＃751) + - All Pooling layers support `data_format` (PR #809) +- Setup: + - Creation of installation flaggs `all_dev`, `all_cpu_dev`, and `all_gpu_dev` (PR #739) +- Examples: + - change folder struction (PR #802) + - `tutorial_models_vgg19` has been introduced to show how to use `tl.model.vgg19` (PR #698). + - fix bug of `tutorial_bipedalwalker_a3c_continuous_action.py` (PR #734, Issue #732) + - `tutorial_models_vgg16` and `tutorial_models_vgg19` has been changed the input scale from [0,255] to [0,1](PR #710) + - `tutorial_mnist_distributed_trainer.py` and `tutorial_cifar10_distributed_trainer.py` are added to explain the uses of Distributed Trainer (PR #700) + - add `tutorial_quanconv_cifar10.py` and `tutorial_quanconv_mnist.py` (PR #735) + - add `tutorial_work_with_onnx.py`(PR #775) +- Applications: + - [Arbitrary Style Transfer in Real-time with Adaptive Instance Normalization](https://arxiv.org/abs/1703.06868) (PR #799) + +### Changed + - function minibatches changed to avoid wasting samples.(PR #762) + - all the input scale in both vgg16 and vgg19 has been changed the input scale from [0,255] to [0,1](PR #710) + - Dockerfiles merged and refactored into one file (PR #747) + - LazyImports move to the most **top level** imports as possible (PR #739) + - some new test functions have been added in `test_layers_convolution.py`, `test_layers_normalization.py`, `test_layers_core.py` (PR #735) + - documentation now uses mock imports reducing the number of dependencies to compile the documentation (PR #785) + - fixed and enforced pydocstyle D210, D200, D301, D207, D403, D204, D412, D402, D300, D208 (PR #784) + +### Deprecated + - `tl.logging.warn` has been deprecated in favor of `tl.logging.warning` (PR #739) + +### Removed + - `conv_layers()` has been removed in both vgg16 and vgg19(PR #710) + - graph API (PR #818) + +### Fixed +- import error caused by matplotlib on OSX (PR #705) +- missing import in tl.prepro (PR #712) +- Dockerfiles import error fixed - issue #733 (PR #747) +- Fix a typo in `absolute_difference_error` in file: `tensorlayer/cost.py` - Issue #753 (PR #759) +- Fix the bug of scaling the learning rate of trainer (PR #776) +- log error instead of info when npz file not found. (PR #812) + +### Dependencies Update +- numpy>=1.14,<1.15 => numpy>=1.14,<1.16 (PR #754) +- pymongo>=3.6,<3.7 => pymongo>=3.6,<3.8 (PR #750) +- pytest>=3.6,<3.7 => tqdm>=3.6,<3.8 (PR #798) +- pytest-xdist>=1.22,<1.23 => pytest-xdist>=1.22,<1.24 (PR #805 and #806) +- tensorflow>=1.8,<1.9 => tensorflow>=1.6,<1.11 (PR #739 and PR #798) +- tqdm>=4.23,<4.25 => tqdm>=4.23,<4.26 (PR #798) +- yapf>=0.21,<0.22 => yapf>=0.22,<0.24 (PR #798 #808) + +### Contributors +- @DEKHTIARJonathan: #739 #747 #750 #754 +- @lgarithm: #705 #700 +- @OwenLiuzZ: #698 #710 #775 #776 +- @zsdonghao: #711 #712 #734 #736 #737 #700 #751 #809 #818 +- @luomai: #700 #751 #766 #802 +- @XJTUWYD: #735 +- @mutewall: #735 +- @thangvubk: #759 +- @JunbinWang: #796 +- @boldjoel: #787 + +## [1.9.1] - 2018-07-30 + +### Fixed +- Issue with tensorflow 1.10.0 fixed + +## [1.9.0] - 2018-06-16 + +### Added +- API: + - `tl.alphas` and `tl.alphas_like` added following the tf.ones/zeros and tf.zeros_like/ones_like (PR #580) + - `tl.lazy_imports.LazyImport` to import heavy libraries only when necessary (PR #667) + - `tl.act.leaky_relu6` and `tl.layers.PRelu6Layer` have been deprecated (PR #686) + - `tl.act.leaky_twice_relu6` and `tl.layers.PTRelu6Layer` have been deprecated (PR #686) +- CI Tool: + - [Stale Probot](https://github.com/probot/stale) added to clean stale issues (PR #573) + - [Changelog Probot](https://github.com/mikz/probot-changelog) Configuration added (PR #637) + - Travis Builds now handling a matrix of TF Version from TF==1.6.0 to TF==1.8.0 (PR #644) + - CircleCI added to build and upload Docker Containers for each PR merged and tag release (PR #648) +- Decorator: + - `tl.decorators` API created including `deprecated_alias` and `private_method` (PR #660) + - `tl.decorators` API enriched with `protected_method` (PR #675) + - `tl.decorators` API enriched with `deprecated` directly raising warning and modifying documentation (PR #691) +- Docker: + - Containers for each release and for each PR merged on master built (PR #648) + - Containers built in the following configurations (PR #648): + - py2 + cpu + - py2 + gpu + - py3 + cpu + - py3 + gpu +- Documentation: + - Clean README.md (PR #677) + - Release semantic version added on index page (PR #633) + - Optimizers page added (PR #636) + - `AMSGrad` added on Optimizers page added (PR #636) +- Layer: + - ElementwiseLambdaLayer added to use custom function to connect multiple layer inputs (PR #579) + - AtrousDeConv2dLayer added (PR #662) + - Fix bugs of using `tf.layers` in CNN (PR #686) +- Optimizer: + - AMSGrad Optimizer added based on `On the Convergence of Adam and Beyond (ICLR 2018)` (PR #636) +- Setup: + - Creation of installation flaggs `all`, `all_cpu`, and `all_gpu` (PR #660) +- Test: + - `test_utils_predict.py` added to reproduce and fix issue #288 (PR #566) + - `Layer_DeformableConvolution_Test` added to reproduce issue #572 with deformable convolution (PR #573) + - `Array_Op_Alphas_Test` and `Array_Op_Alphas_Like_Test` added to test `tensorlayer/array_ops.py` file (PR #580) + - `test_optimizer_amsgrad.py` added to test `AMSGrad` optimizer (PR #636) + - `test_logging.py` added to insure robustness of the logging API (PR #645) + - `test_decorators.py` added (PR #660) + - `test_activations.py` added (PR #686) +- Tutorials: + - `tutorial_tfslim` has been introduced to show how to use `SlimNetsLayer` (PR #560). + - add the following to all tutorials (PR #697): + ```python + tf.logging.set_verbosity(tf.logging.DEBUG) + tl.logging.set_verbosity(tl.logging.DEBUG) + ``` + +### Changed +- Tensorflow CPU & GPU dependencies moved to separated requirement files in order to allow PyUP.io to parse them (PR #573) +- The document of LambdaLayer for linking it with ElementwiseLambdaLayer (PR #587) +- RTD links point to stable documentation instead of latest used for development (PR #633) +- TF Version older than 1.6.0 are officially unsupported and raises an exception (PR #644) +- README.md Badges Updated with Support Python and Tensorflow Versions (PR #644) +- TL logging API has been consistent with TF logging API and thread-safe (PR #645) +- Relative Imports changed for absolute imports (PR #657) +- `tl.files` refactored into a directory with numerous files (PR #657) +- `tl.files.voc_dataset` fixed because of original Pascal VOC website was down (PR #657) +- extra requirements hidden inside the library added in the project requirements (PR #657) +- requirements files refactored in `requirements/` directory (PR #657) +- README.md and other markdown files have been refactored and cleaned. (PR #639) +- Ternary Convolution Layer added in unittest (PR #658) +- Convolution Layers unittests have been cleaned & refactored (PR #658) +- All the tests are now using a DEBUG level verbosity when run individualy (PR #660) +- `tf.identity` as activation is **ignored**, thus reducing the size of the graph by removing useless operation (PR #667) +- argument dictionaries are now checked and saved within the `Layer` Base Class (PR #667) +- `Layer` Base Class now presenting methods to update faultlessly `all_layers`, `all_params`, and `all_drop` (PR #675) +- Input Layers have been removed from `tl.layers.core` and added to `tl.layers.inputs` (PR #675) +- Input Layers are now considered as true layers in the graph (they represent a placeholder), unittests have been updated (PR #675) +- Layer API is simplified, with automatic feeding `prev_layer` into `self.inputs` (PR #675) +- Complete Documentation Refactoring and Reorganization (namely Layer APIs) (PR #691) + +### Deprecated +- `tl.layers.TimeDistributedLayer` argurment `args` is deprecated in favor of `layer_args` (PR #667) +- `tl.act.leaky_relu` have been deprecated in favor of `tf.nn.leaky_relu` (PR #686) + +### Removed +- `assert()` calls remove and replaced by `raise AssertionError()` (PR #667) +- `tl.identity` is removed, not used anymore and deprecated for a long time (PR #667) +- All Code specific to `TF.__version__ < "1.6"` have been removed (PR #675) + +### Fixed +- Issue #498 - Deprecation Warning Fix in `tl.layers.RNNLayer` with `inspect` (PR #574) +- Issue #498 - Deprecation Warning Fix in `tl.files` with truth value of an empty array is ambiguous (PR #575) +- Issue #565 related to `tl.utils.predict` fixed - `np.hstack` problem in which the results for multiple batches are stacked along `axis=1` (PR #566) +- Issue #572 with `tl.layers.DeformableConv2d` fixed (PR #573) +- Issue #664 with `tl.layers.ConvLSTMLayer` fixed (PR #676) +- Typo of the document of ElementwiseLambdaLayer (PR #588) +- Error in `tl.layers.TernaryConv2d` fixed - self.inputs not defined (PR #658) +- Deprecation warning fixed in `tl.layers.binary._compute_threshold()` (PR #658) +- All references to `tf.logging` replaced by `tl.logging` (PR #661) +- Duplicated code removed when bias was used (PR #667) +- `tensorlayer.third_party.roi_pooling.roi_pooling.roi_pooling_ops` is now lazy loaded to prevent systematic error raised (PR #675) +- Documentation not build in RTD due to old version of theme in docs directory fixed (PR #703) +- Tutorial: + - `tutorial_word2vec_basic.py` saving issue #476 fixed (PR #635) + - All tutorials tested and errors have been fixed (PR #635) + +### Dependencies Update +- Update pytest from 3.5.1 to 3.6.0 (PR #647) +- Update progressbar2 from 3.37.1 to 3.38.0 (PR #651) +- Update scikit-image from 0.13.1 to 0.14.0 (PR #656) +- Update keras from 2.1.6 to 2.2.0 (PR #684) +- Update requests from 2.18.4 to 2.19.0 (PR #695) + +### Contributors +- @lgarithm: #563 +- @DEKHTIARJonathan: #573 #574 #575 #580 #633 #635 #636 #639 #644 #645 #648 #657 #667 #658 #659 #660 #661 #666 #667 #672 #675 #683 #686 #687 #690 #691 #692 #703 +- @2wins: #560 #566 #662 +- @One-sixth: #579 +- @zsdonghao: #587 #588 #639 #685 #697 +- @luomai: #639 #677 +- @dengyueyun666: #676 + +## [1.8.5] - 2018-05-09 + +### Added +- Github Templates added (by @DEKHTIARJonathan) + - New issues Template + - New PR Template +- Travis Deploy Automation on new Tag (by @DEKHTIARJonathan) + - Deploy to PyPI and create a new version. + - Deploy to Github Releases and upload the wheel files +- PyUP.io has been added to ensure we are compatible with the latest libraries (by @DEKHTIARJonathan) +- `deconv2d` now handling dilation_rate (by @zsdonghao) +- Documentation unittest added (by @DEKHTIARJonathan) +- `test_layers_core` has been added to ensure that `LayersConfig` is abstract. + +### Changed +- All Tests Refactored - Now using unittests and runned with PyTest (by @DEKHTIARJonathan) +- Documentation updated (by @zsdonghao) +- Package Setup Refactored (by @DEKHTIARJonathan) +- Dataset Downlaod now using library progressbar2 (by @DEKHTIARJonathan) +- `deconv2d` function transformed into Class (by @zsdonghao) +- `conv1d` function transformed into Class (by @zsdonghao) +- super resolution functions transformed into Class (by @zsdonghao) +- YAPF coding style improved and enforced (by @DEKHTIARJonathan) + +### Fixed +- Backward Compatibility Restored with deprecation warnings (by @DEKHTIARJonathan) +- Tensorflow Deprecation Fix (Issue #498): + - AverageEmbeddingInputlayer (by @zsdonghao) + - load_mpii_pose_dataset (by @zsdonghao) +- maxPool2D initializer issue #551 (by @zsdonghao) +- `LayersConfig` class has been enforced as abstract +- Pooling Layer Issue #557 fixed (by @zsdonghao) + +### Dependencies Update +- scipy>=1.0,<1.1 => scipy>=1.1,<1.2 + +### Contributors +@zsdonghao @luomai @DEKHTIARJonathan + +[Unreleased]: https://github.com/tensorlayer/tensorlayer/compare/1.11....master +[2.0.1]: https://github.com/tensorlayer/tensorlayer/compare/2.0.1...2.0.1 +[2.0.0]: https://github.com/tensorlayer/tensorlayer/compare/2.0.0...2.0.0 +[1.11.1]: https://github.com/tensorlayer/tensorlayer/compare/1.11.0...1.11.0 +[1.11.0]: https://github.com/tensorlayer/tensorlayer/compare/1.10.1...1.11.0 +[1.10.1]: https://github.com/tensorlayer/tensorlayer/compare/1.10.0...1.10.1 +[1.10.0]: https://github.com/tensorlayer/tensorlayer/compare/1.9.1...1.10.0 +[1.9.1]: https://github.com/tensorlayer/tensorlayer/compare/1.9.0...1.9.1 +[1.9.0]: https://github.com/tensorlayer/tensorlayer/compare/1.8.5...1.9.0 +[1.8.5]: https://github.com/tensorlayer/tensorlayer/compare/1.8.4...1.8.5 \ No newline at end of file diff --git a/examples/text_generation/tutorial_generate_text.py b/examples/text_generation/tutorial_generate_text.py new file mode 100644 index 000000000..d157b1ed5 --- /dev/null +++ b/examples/text_generation/tutorial_generate_text.py @@ -0,0 +1,332 @@ +#! /usr/bin/python +# -*- coding: utf-8 -*- +# Copyright 2019 TensorLayer. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Example of Synced sequence input and output. + +Generate text using LSTM. + +Data: https://github.com/tensorlayer/tensorlayer/tree/master/example/data/ + +""" + +import os +import re +import time + +import nltk +import numpy as np +import tensorflow as tf + +import tensorlayer as tl +from tensorlayer.layers import * +from tensorlayer.models import Model + +tl.logging.set_verbosity(tl.logging.DEBUG) + +_UNK = "_UNK" + + +def basic_clean_str(string): + """Tokenization/string cleaning for a datasets.""" + string = re.sub(r"\n", " ", string) # '\n' --> ' ' + string = re.sub(r"\'s", " \'s", string) # it's --> it 's + string = re.sub(r"\’s", " \'s", string) + string = re.sub(r"\'ve", " have", string) # they've --> they have + string = re.sub(r"\’ve", " have", string) + string = re.sub(r"\'t", " not", string) # can't --> can not + string = re.sub(r"\’t", " not", string) + string = re.sub(r"\'re", " are", string) # they're --> they are + string = re.sub(r"\’re", " are", string) + string = re.sub(r"\'d", "", string) # I'd (I had, I would) --> I + string = re.sub(r"\’d", "", string) + string = re.sub(r"\'ll", " will", string) # I'll --> I will + string = re.sub(r"\’ll", " will", string) + string = re.sub(r"\“", " ", string) # “a” --> “ a ” + string = re.sub(r"\”", " ", string) + string = re.sub(r"\"", " ", string) # "a" --> " a " + string = re.sub(r"\'", " ", string) # they' --> they ' + string = re.sub(r"\’", " ", string) # they’ --> they ’ + string = re.sub(r"\.", " . ", string) # they. --> they . + string = re.sub(r"\,", " , ", string) # they, --> they , + string = re.sub(r"\!", " ! ", string) + string = re.sub(r"\-", " ", string) # "low-cost"--> lost cost + string = re.sub(r"\(", " ", string) # (they) --> ( they) + string = re.sub(r"\)", " ", string) # ( they) --> ( they ) + string = re.sub(r"\]", " ", string) # they] --> they ] + string = re.sub(r"\[", " ", string) # they[ --> they [ + string = re.sub(r"\?", " ", string) # they? --> they ? + string = re.sub(r"\>", " ", string) # they> --> they > + string = re.sub(r"\<", " ", string) # they< --> they < + string = re.sub(r"\=", " ", string) # easier= --> easier = + string = re.sub(r"\;", " ", string) # easier; --> easier ; + string = re.sub(r"\;", " ", string) + string = re.sub(r"\:", " ", string) # easier: --> easier : + string = re.sub(r"\"", " ", string) # easier" --> easier " + string = re.sub(r"\$", " ", string) # $380 --> $ 380 + string = re.sub(r"\_", " ", string) # _100 --> _ 100 + string = re.sub(r"\s{2,}", " ", string) # Akara is handsome --> Akara is handsome + return string.strip().lower() # lowercase + + +def customized_clean_str(string): + """Tokenization/string cleaning for a datasets.""" + string = re.sub(r"\n", " ", string) # '\n' --> ' ' + string = re.sub(r"\'s", " \'s", string) # it's --> it 's + string = re.sub(r"\’s", " \'s", string) + string = re.sub(r"\'ve", " have", string) # they've --> they have + string = re.sub(r"\’ve", " have", string) + string = re.sub(r"\'t", " not", string) # can't --> can not + string = re.sub(r"\’t", " not", string) + string = re.sub(r"\'re", " are", string) # they're --> they are + string = re.sub(r"\’re", " are", string) + string = re.sub(r"\'d", "", string) # I'd (I had, I would) --> I + string = re.sub(r"\’d", "", string) + string = re.sub(r"\'ll", " will", string) # I'll --> I will + string = re.sub(r"\’ll", " will", string) + string = re.sub(r"\“", " “ ", string) # “a” --> “ a ” + string = re.sub(r"\”", " ” ", string) + string = re.sub(r"\"", " “ ", string) # "a" --> " a " + string = re.sub(r"\'", " ' ", string) # they' --> they ' + string = re.sub(r"\’", " ' ", string) # they’ --> they ' + string = re.sub(r"\.", " . ", string) # they. --> they . + string = re.sub(r"\,", " , ", string) # they, --> they , + string = re.sub(r"\-", " ", string) # "low-cost"--> lost cost + string = re.sub(r"\(", " ( ", string) # (they) --> ( they) + string = re.sub(r"\)", " ) ", string) # ( they) --> ( they ) + string = re.sub(r"\!", " ! ", string) # they! --> they ! + string = re.sub(r"\]", " ] ", string) # they] --> they ] + string = re.sub(r"\[", " [ ", string) # they[ --> they [ + string = re.sub(r"\?", " ? ", string) # they? --> they ? + string = re.sub(r"\>", " > ", string) # they> --> they > + string = re.sub(r"\<", " < ", string) # they< --> they < + string = re.sub(r"\=", " = ", string) # easier= --> easier = + string = re.sub(r"\;", " ; ", string) # easier; --> easier ; + string = re.sub(r"\;", " ; ", string) + string = re.sub(r"\:", " : ", string) # easier: --> easier : + string = re.sub(r"\"", " \" ", string) # easier" --> easier " + string = re.sub(r"\$", " $ ", string) # $380 --> $ 380 + string = re.sub(r"\_", " _ ", string) # _100 --> _ 100 + string = re.sub(r"\s{2,}", " ", string) # Akara is handsome --> Akara is handsome + return string.strip().lower() # lowercase + + +def customized_read_words(input_fpath): # , dictionary): + with open(input_fpath, "r", encoding="utf8") as f: + words = f.read() + # Clean the data + words = customized_clean_str(words) + # Split each word + return words.split() + + +def main_restore_embedding_layer(): + """How to use Embedding layer, and how to convert IDs to vector, + IDs to words, etc. + """ + # Step 1: Build the embedding matrix and load the existing embedding matrix. + vocabulary_size = 50000 + embedding_size = 128 + model_file_name = "model_word2vec_50k_128" + batch_size = None + + if not os.path.exists(model_file_name + ".npy"): + raise Exception( + "Pretrained embedding matrix not found. " + "Hint: Please pre-train the default model in " + "`examples/text_word_embedding/tutorial_word2vec_basic.py`." + ) + + print("Load existing embedding matrix and dictionaries") + all_var = tl.files.load_npy_to_any(name=model_file_name + '.npy') + data = all_var['data'] + count = all_var['count'] + dictionary = all_var['dictionary'] + reverse_dictionary = all_var['reverse_dictionary'] + + tl.nlp.save_vocab(count, name='vocab_' + model_file_name + '.txt') + + del all_var, data, count + + class Embedding_Model(Model): + + def __init__(self): + super(Embedding_Model, self).__init__() + self.embedding = Embedding(vocabulary_size, embedding_size) + + def forward(self, inputs): + return self.embedding(inputs) + + model = Embedding_Model() + model.eval() + + # TODO: assign certain parameters to model + model.load_weights(model_file_name + ".hdf5", skip=True, in_order=False) + + # Step 2: Input word(s), output the word vector(s). + word = 'hello' + word_id = dictionary[word] + print('word_id:', word_id) + + words = ['i', 'am', 'tensor', 'layer'] + word_ids = tl.nlp.words_to_word_ids(words, dictionary, _UNK) + context = tl.nlp.word_ids_to_words(word_ids, reverse_dictionary) + print('word_ids:', word_ids) + print('context:', context) + + vector = model(word_id) + print('vector:', vector.shape) + print(vector) + + vectors = model(word_ids) + print('vectors:', vectors.shape) + print(vectors) + + +class Text_Generation_Net(Model): + + def __init__(self, vocab_size, hidden_size, init): + super(Text_Generation_Net, self).__init__() + + self.embedding = Embedding(vocab_size, hidden_size, init, name='embedding') + self.lstm = tl.layers.RNN( + cell=tf.keras.layers.LSTMCell(hidden_size), return_last_output=False, return_last_state=True, + return_seq_2d=True, in_channels=hidden_size + ) + self.out_dense = Dense(vocab_size, in_channels=hidden_size, W_init=init, b_init=init, act=None, name='output') + + def forward(self, inputs, initial_state=None): + embedding_vector = self.embedding(inputs) + lstm_out, final_state = self.lstm(embedding_vector, initial_state=initial_state) + logits = self.out_dense(lstm_out) + return logits, final_state + + +def main_lstm_generate_text(): + """Generate text by Synced sequence input and output.""" + # rnn model and update (describtion: see tutorial_ptb_lstm.py) + init_scale = 0.1 + learning_rate = 1e-3 + sequence_length = 20 + hidden_size = 200 + max_epoch = 100 + batch_size = 16 + + top_k_list = [1, 3, 5, 10] + print_length = 30 + + model_file_name = "model_generate_text.hdf5" + + # ===== Prepare Data + words = customized_read_words(input_fpath="data/trump/trump_text.txt") + + vocab = tl.nlp.create_vocab([words], word_counts_output_file='vocab.txt', min_word_count=1) + vocab = tl.nlp.Vocabulary('vocab.txt', unk_word="") + vocab_size = vocab.unk_id + 1 + train_data = [vocab.word_to_id(word) for word in words] + + # Set the seed to generate sentence. + seed = "it is a" + # seed = basic_clean_str(seed).split() + seed = nltk.tokenize.word_tokenize(seed) + print('seed : %s' % seed) + + init = tl.initializers.random_uniform(-init_scale, init_scale) + + net = Text_Generation_Net(vocab_size, hidden_size, init) + + train_weights = net.weights + optimizer = tf.optimizers.Adam(lr=learning_rate) + + # ===== Training + + print("\nStart learning a model to generate text") + for i in range(max_epoch): + + print("Epoch: %d/%d" % (i + 1, max_epoch)) + epoch_size = ((len(train_data) // batch_size) - 1) // sequence_length + + start_time = time.time() + costs = 0.0 + iters = 0 + + net.train() + # reset all states at the begining of every epoch + lstm_state = None + for step, (x, y) in enumerate(tl.iterate.ptb_iterator(train_data, batch_size, sequence_length)): + with tf.GradientTape() as tape: + ## compute outputs + logits, lstm_state = net(x, initial_state=lstm_state) + ## compute loss and update model + cost = tl.cost.cross_entropy(logits, tf.reshape(y, [-1]), name='train_loss') + + grad = tape.gradient(cost, train_weights) + optimizer.apply_gradients(zip(grad, train_weights)) + + costs += cost + iters += 1 + + if step % (epoch_size // 10) == 1: + print( + "%.3f perplexity: %.3f speed: %.0f wps" % ( + step * 1.0 / epoch_size, np.exp(costs / iters), + iters * batch_size * sequence_length * batch_size / (time.time() - start_time) + ) + ) + train_perplexity = np.exp(costs / iters) + # print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity)) + print("Epoch: %d/%d Train Perplexity: %.3f" % (i + 1, max_epoch, train_perplexity)) + + net.eval() + # for diversity in diversity_list: + # testing: sample from top k words + for top_k in top_k_list: + # Testing, generate some text from a given seed. + lstm_state = None + outs_id = [vocab.word_to_id(w) for w in seed] + # feed the seed to initialize the state for generation. + for ids in outs_id[:-1]: + a_id = np.asarray(ids).reshape(1, 1) + _, lstm_state = net(a_id, initial_state=lstm_state) + + # feed the last word in seed, and start to generate sentence. + a_id = outs_id[-1] + for _ in range(print_length): + a_id = np.asarray(a_id).reshape(1, 1) + logits, lstm_state = net(a_id, initial_state=lstm_state) + out = tf.nn.softmax(logits) + # Without sampling + # a_id = np.argmax(out[0]) + # Sample from all words, if vocab_size is large, + # this may have numeric error. + # a_id = tl.nlp.sample(out[0], diversity) + # Sample from the top k words. + a_id = tl.nlp.sample_top(out[0].numpy(), top_k=top_k) + outs_id.append(a_id) + sentence = [vocab.id_to_word(w) for w in outs_id] + sentence = " ".join(sentence) + # print(diversity, ':', sentence) + print(top_k, ':', sentence) + + print("Save model") + net.save_weights(model_file_name) + + +if __name__ == '__main__': + # Restore a pretrained embedding matrix + # main_restore_embedding_layer() + + # How to generate text from a given context + main_lstm_generate_text() \ No newline at end of file From 74c6ada318716f1d7eeee8c65293bc5989bb6859 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 12:24:36 +0800 Subject: [PATCH 11/19] Update CHANGELOG.md --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 630b2b2db..42dbca217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,20 @@ To release a new version, please update the changelog as followed: +## [2.1.0] - 2019-5-25 + +### Changed +- change the format of network config, change related code and files + +### Added + +### Dependencies Update + +### Fixed + +### Contributors + + ## [2.0.1] - 2019-5-17 From 0866b5635075559f46c728b4c17bc69b85fb1635 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 13:42:44 +0800 Subject: [PATCH 12/19] fix bugs --- tensorlayer/db.py | 4 ++-- tensorlayer/files/utils.py | 22 +++++++++++----------- tensorlayer/layers/core.py | 3 ++- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tensorlayer/db.py b/tensorlayer/db.py index cb8db8e10..7ca3e7bb5 100644 --- a/tensorlayer/db.py +++ b/tensorlayer/db.py @@ -13,7 +13,7 @@ import tensorflow as tf from tensorlayer import logging -from tensorlayer.files import net2static_graph, static_graph2net, assign_weights +from tensorlayer.files import static_graph2net, assign_weights from tensorlayer.files import save_weights_to_hdf5, load_hdf5_to_weights from tensorlayer.files import del_folder, exists_or_mkdir @@ -153,7 +153,7 @@ def save_model(self, network=None, model_name='model', **kwargs): s = time.time() # kwargs.update({'architecture': network.all_graphs, 'time': datetime.utcnow()}) - kwargs.update({'architecture': net2static_graph(network), 'time': datetime.utcnow()}) + kwargs.update({'architecture': network.config, 'time': datetime.utcnow()}) try: params_id = self.model_fs.put(self._serialization(params)) diff --git a/tensorlayer/files/utils.py b/tensorlayer/files/utils.py index 0d4acedbe..488fbb235 100644 --- a/tensorlayer/files/utils.py +++ b/tensorlayer/files/utils.py @@ -180,8 +180,8 @@ def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False, customiz logging.info("[*] Saving TL model into {}, saving weights={}".format(filepath, save_weights)) - network_config = network.config # net2static_graph(network) - network_config_str = str(network_config) + model_config = network.config # net2static_graph(network) + model_config_str = str(model_config) customized_data_str = str(customized_data) version_info = { "tensorlayer_version": tl.__version__, @@ -193,7 +193,7 @@ def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False, customiz version_info_str = str(version_info) with h5py.File(filepath, 'w') as f: - f.attrs["network_config"] = network_config_str.encode('utf8') + f.attrs["model_config"] = model_config_str.encode('utf8') f.attrs["customized_data"] = customized_data_str.encode('utf8') f.attrs["version_info"] = version_info_str.encode('utf8') if save_weights: @@ -251,12 +251,12 @@ def eval_layer(layer_kwargs): raise RuntimeError("Unknown layer type.") -def static_graph2net(network_config): +def static_graph2net(model_config): layer_dict = {} - model_name = network_config["name"] - inputs_tensors = network_config["inputs"] - outputs_tensors = network_config["outputs"] - all_args = network_config["model_architecture"] + model_name = model_config["name"] + inputs_tensors = model_config["inputs"] + outputs_tensors = model_config["outputs"] + all_args = model_config["model_architecture"] for idx, layer_kwargs in enumerate(all_args): layer_class = layer_kwargs["class"] # class of current layer prev_layers = layer_kwargs.pop("prev_layer") # name of previous layers @@ -332,10 +332,10 @@ def load_hdf5_graph(filepath='model.hdf5', load_weights=False): ) ) - network_config_str = f.attrs["network_config"].decode('utf8') - network_config = eval(network_config_str) + model_config_str = f.attrs["model_config"].decode('utf8') + model_config = eval(model_config_str) - M = static_graph2net(network_config) + M = static_graph2net(model_config) if load_weights: if not ('layer_names' in f.attrs.keys()): raise RuntimeError("Saved model does not contain weights.") diff --git a/tensorlayer/layers/core.py b/tensorlayer/layers/core.py index ce98f156c..bce74f389 100644 --- a/tensorlayer/layers/core.py +++ b/tensorlayer/layers/core.py @@ -479,7 +479,8 @@ def _release_memory(self): def get_args(self): init_args = {} init_args.update({"layer_type": "modellayer"}) - init_args["model"] = utils.net2static_graph(self.layer_args["model"]) + # init_args["model"] = utils.net2static_graph(self.layer_args["model"]) + init_args["model"] = self.layer_args["model"].config return init_args From a795f726c88d23cb80a307d4cb2b41becdfde9ba Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 13:50:39 +0800 Subject: [PATCH 13/19] yapf --- CHANGELOG.md | 3 ++- tensorlayer/files/utils.py | 14 +++++++------- tensorlayer/models/core.py | 4 +++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42dbca217..6a1f58ffd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,7 @@ To release a new version, please update the changelog as followed: ## [2.1.0] - 2019-5-25 ### Changed -- change the format of network config, change related code and files +- change the format of network config, change related code and files (PR #980) ### Added @@ -79,6 +79,7 @@ To release a new version, please update the changelog as followed: ### Fixed ### Contributors +- @warshallrho: #PR980 ## [2.0.1] - 2019-5-17 diff --git a/tensorlayer/files/utils.py b/tensorlayer/files/utils.py index 488fbb235..60521c309 100644 --- a/tensorlayer/files/utils.py +++ b/tensorlayer/files/utils.py @@ -184,12 +184,12 @@ def save_hdf5_graph(network, filepath='model.hdf5', save_weights=False, customiz model_config_str = str(model_config) customized_data_str = str(customized_data) version_info = { - "tensorlayer_version": tl.__version__, - "backend": "tensorflow", - "backend_version": tf.__version__, - "training_device": "gpu", - "save_date": datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() - } + "tensorlayer_version": tl.__version__, + "backend": "tensorflow", + "backend_version": tf.__version__, + "training_device": "gpu", + "save_date": datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() + } version_info_str = str(version_info) with h5py.File(filepath, 'w') as f: @@ -318,7 +318,7 @@ def load_hdf5_graph(filepath='model.hdf5', load_weights=False): version_info_str = f.attrs["version_info"].decode('utf8') version_info = eval(version_info_str) backend_version = version_info["backend_version"] - tensorlayer_version =version_info["tensorlayer_version"] + tensorlayer_version = version_info["tensorlayer_version"] if backend_version != tf.__version__: logging.warning( "Saved model uses tensorflow version {}, but now you are using tensorflow version {}".format( diff --git a/tensorlayer/models/core.py b/tensorlayer/models/core.py index 369598970..c1197ed55 100644 --- a/tensorlayer/models/core.py +++ b/tensorlayer/models/core.py @@ -769,7 +769,9 @@ def save(self, filepath, save_weights=True, customized_data=None): raise RuntimeError( "Model save() not support dynamic mode yet.\nHint: you can use Model save_weights() to save the weights in dynamic mode." ) - utils.save_hdf5_graph(network=self, filepath=filepath, save_weights=save_weights, customized_data=customized_data) + utils.save_hdf5_graph( + network=self, filepath=filepath, save_weights=save_weights, customized_data=customized_data + ) @staticmethod def load(filepath, load_weights=True): From 03e990cfe695e3c23843cab262f022adaf1f2408 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 20:05:25 +0800 Subject: [PATCH 14/19] update act in base layer and related layers --- docs/user/get_start_model.rst | 3 +-- tensorlayer/layers/convolution/binary_conv.py | 3 +-- .../layers/convolution/deformable_conv.py | 3 +-- .../layers/convolution/depthwise_conv.py | 3 +-- tensorlayer/layers/convolution/dorefa_conv.py | 3 +-- tensorlayer/layers/convolution/expert_conv.py | 9 +++---- .../layers/convolution/expert_deconv.py | 9 +++---- tensorlayer/layers/convolution/group_conv.py | 3 +-- tensorlayer/layers/convolution/quan_conv.py | 3 +-- .../layers/convolution/separable_conv.py | 6 ++--- .../layers/convolution/simplified_conv.py | 9 +++---- .../layers/convolution/simplified_deconv.py | 6 ++--- .../layers/convolution/super_resolution.py | 6 ++--- .../layers/convolution/ternary_conv.py | 3 +-- tensorlayer/layers/core.py | 19 +++++++++++++- tensorlayer/layers/dense/base_dense.py | 3 +-- tensorlayer/layers/dense/binary_dense.py | 3 +-- tensorlayer/layers/dense/dorefa_dense.py | 3 +-- tensorlayer/layers/dense/dropconnect.py | 3 +-- tensorlayer/layers/dense/quan_dense.py | 3 +-- tensorlayer/layers/dense/ternary_dense.py | 3 +-- tensorlayer/layers/merge.py | 3 +-- tensorlayer/layers/normalization.py | 15 ++++------- tests/layers/test_layers_convolution.py | 25 ++++++++++++++++++- tests/models/test_model_save_graph.py | 12 ++++----- 25 files changed, 83 insertions(+), 78 deletions(-) diff --git a/docs/user/get_start_model.rst b/docs/user/get_start_model.rst index bc44ebcff..3026c5198 100644 --- a/docs/user/get_start_model.rst +++ b/docs/user/get_start_model.rst @@ -208,10 +208,9 @@ z = f(x*W+b) class Dense(Layer): def __init__(self, n_units, act=None, in_channels=None, name=None): - super(Dense, self).__init__(name) + super(Dense, self).__init__(name, act=act) self.n_units = n_units - self.act = act self.in_channels = in_channels # for dynamic model, it needs the input shape to get the shape of W diff --git a/tensorlayer/layers/convolution/binary_conv.py b/tensorlayer/layers/convolution/binary_conv.py index 23448cf6f..cf55127d5 100644 --- a/tensorlayer/layers/convolution/binary_conv.py +++ b/tensorlayer/layers/convolution/binary_conv.py @@ -75,11 +75,10 @@ def __init__( in_channels=None, name=None # 'binary_cnn2d', ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides - self.act = act self.padding = padding self.use_gemm = use_gemm self.data_format = data_format diff --git a/tensorlayer/layers/convolution/deformable_conv.py b/tensorlayer/layers/convolution/deformable_conv.py index 5f75bbe15..616803ba1 100644 --- a/tensorlayer/layers/convolution/deformable_conv.py +++ b/tensorlayer/layers/convolution/deformable_conv.py @@ -83,12 +83,11 @@ def __init__( in_channels=None, name=None # 'deformable_conv_2d', ): - super().__init__(name) + super().__init__(name, act=act) self.offset_layer = offset_layer self.n_filter = n_filter self.filter_size = filter_size - self.act = act self.padding = padding self.W_init = W_init self.b_init = b_init diff --git a/tensorlayer/layers/convolution/depthwise_conv.py b/tensorlayer/layers/convolution/depthwise_conv.py index d6136ede3..b11233c27 100644 --- a/tensorlayer/layers/convolution/depthwise_conv.py +++ b/tensorlayer/layers/convolution/depthwise_conv.py @@ -82,10 +82,9 @@ def __init__( in_channels=None, name=None # 'depthwise_conv2d' ): - super().__init__(name) + super().__init__(name, act=act) self.filter_size = filter_size self.strides = self._strides = strides - self.act = act self.padding = padding self.dilation_rate = self._dilation_rate = dilation_rate self.data_format = data_format diff --git a/tensorlayer/layers/convolution/dorefa_conv.py b/tensorlayer/layers/convolution/dorefa_conv.py index ed9b32dd8..dc7979967 100644 --- a/tensorlayer/layers/convolution/dorefa_conv.py +++ b/tensorlayer/layers/convolution/dorefa_conv.py @@ -81,13 +81,12 @@ def __init__( in_channels=None, name=None # 'dorefa_cnn2d', ): - super().__init__(name) + super().__init__(name, act=act) self.bitW = bitW self.bitA = bitA self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides - self.act = act self.padding = padding self.use_gemm = use_gemm self.data_format = data_format diff --git a/tensorlayer/layers/convolution/expert_conv.py b/tensorlayer/layers/convolution/expert_conv.py index d7e59a0e8..50ea12cb9 100644 --- a/tensorlayer/layers/convolution/expert_conv.py +++ b/tensorlayer/layers/convolution/expert_conv.py @@ -71,8 +71,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn1d_layer', ): - super().__init__(name) - self.act = act + super().__init__(name, act=act) self.n_filter = shape[-1] self.filter_size = shape[0] self.shape = shape @@ -191,8 +190,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn2d_layer', ): - super().__init__(name) - self.act = act + super().__init__(name, act=act) self.n_filter = shape[-1] self.filter_size = (shape[0], shape[1]) self.shape = shape @@ -310,8 +308,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn3d_layer' ): - super().__init__(name) - self.act = act + super().__init__(name, act=act) self.n_filter = shape[-1] self.filter_size = (shape[0], shape[1], shape[2]) self.shape = shape diff --git a/tensorlayer/layers/convolution/expert_deconv.py b/tensorlayer/layers/convolution/expert_deconv.py index cb5cd6773..f23c752ad 100644 --- a/tensorlayer/layers/convolution/expert_deconv.py +++ b/tensorlayer/layers/convolution/expert_deconv.py @@ -80,8 +80,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn1d_layer', ): - super().__init__(name) - self.act = act + super().__init__(name, act=act) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides @@ -215,8 +214,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn2d_layer', ): - super().__init__(name) - self.act = act + super().__init__(name, act=act) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides @@ -342,8 +340,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn3d_layer', ): - super().__init__(name) - self.act = act + super().__init__(name, act=act) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides diff --git a/tensorlayer/layers/convolution/group_conv.py b/tensorlayer/layers/convolution/group_conv.py index 34d8c10e6..bc35d4e00 100644 --- a/tensorlayer/layers/convolution/group_conv.py +++ b/tensorlayer/layers/convolution/group_conv.py @@ -73,12 +73,11 @@ def __init__( in_channels=None, name=None # 'groupconv', ): # Windaway - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides self.n_group = n_group - self.act = act self.padding = padding self.data_format = data_format self.dilation_rate = self._dilation_rate = dilation_rate diff --git a/tensorlayer/layers/convolution/quan_conv.py b/tensorlayer/layers/convolution/quan_conv.py index e235dfeb4..75ee3943c 100644 --- a/tensorlayer/layers/convolution/quan_conv.py +++ b/tensorlayer/layers/convolution/quan_conv.py @@ -82,13 +82,12 @@ def __init__( in_channels=None, name=None # 'quan_cnn2d', ): - super().__init__(name) + super().__init__(name, act=act) self.bitW = bitW self.bitA = bitA self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides - self.act = act self.padding = padding self.use_gemm = use_gemm self.data_format = data_format diff --git a/tensorlayer/layers/convolution/separable_conv.py b/tensorlayer/layers/convolution/separable_conv.py index b6ae62446..ca1c66d49 100644 --- a/tensorlayer/layers/convolution/separable_conv.py +++ b/tensorlayer/layers/convolution/separable_conv.py @@ -84,11 +84,10 @@ def __init__( in_channels=None, name=None # 'seperable1d', ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides - self.act = act self.padding = padding self.data_format = data_format self.dilation_rate = dilation_rate @@ -232,11 +231,10 @@ def __init__( in_channels=None, name=None # 'seperable2d', ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides - self.act = act self.padding = padding self.data_format = data_format self.dilation_rate = dilation_rate diff --git a/tensorlayer/layers/convolution/simplified_conv.py b/tensorlayer/layers/convolution/simplified_conv.py index c00ff8fe7..536d4e52e 100644 --- a/tensorlayer/layers/convolution/simplified_conv.py +++ b/tensorlayer/layers/convolution/simplified_conv.py @@ -70,11 +70,10 @@ def __init__( in_channels=None, name=None # 'conv1d' ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.stride = stride - self.act = act self.padding = padding self.data_format = data_format self.dilation_rate = dilation_rate @@ -201,11 +200,10 @@ def __init__( in_channels=None, name=None # 'conv2d', ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self._strides = self.strides = strides - self.act = act self.padding = padding self.data_format = data_format self._dilation_rate = self.dilation_rate = dilation_rate @@ -335,11 +333,10 @@ def __init__( in_channels=None, name=None # 'conv3d', ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self._strides = self.strides = strides - self.act = act self.padding = padding self.data_format = data_format self._dilation_rate = self.dilation_rate = dilation_rate diff --git a/tensorlayer/layers/convolution/simplified_deconv.py b/tensorlayer/layers/convolution/simplified_deconv.py index 847062859..57beff0f4 100644 --- a/tensorlayer/layers/convolution/simplified_deconv.py +++ b/tensorlayer/layers/convolution/simplified_deconv.py @@ -71,12 +71,11 @@ def __init__( in_channels=None, name=None # 'decnn2d' ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides self.padding = padding - self.act = act self.data_format = data_format self.dilation_rate = dilation_rate self.W_init = W_init @@ -199,12 +198,11 @@ def __init__( in_channels=None, name=None # 'decnn3d' ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides self.padding = padding - self.act = act self.data_format = data_format self.W_init = W_init self.b_init = b_init diff --git a/tensorlayer/layers/convolution/super_resolution.py b/tensorlayer/layers/convolution/super_resolution.py index 35fee8722..21d765e03 100644 --- a/tensorlayer/layers/convolution/super_resolution.py +++ b/tensorlayer/layers/convolution/super_resolution.py @@ -53,9 +53,8 @@ def __init__( in_channels=None, name=None # 'subpixel_conv1d' ): - super().__init__(name) + super().__init__(name, act=act) self.scale = scale - self.act = act self.in_channels = in_channels self.out_channels = int(self.in_channels / self.scale) @@ -151,10 +150,9 @@ def __init__( in_channels=None, name=None # 'subpixel_conv2d' ): - super().__init__(name) + super().__init__(name, act=act) self.scale = scale self.n_out_channels = n_out_channels - self.act = act self.in_channels = in_channels if self.in_channels is not None: diff --git a/tensorlayer/layers/convolution/ternary_conv.py b/tensorlayer/layers/convolution/ternary_conv.py index 9a97c7bec..33e01507c 100644 --- a/tensorlayer/layers/convolution/ternary_conv.py +++ b/tensorlayer/layers/convolution/ternary_conv.py @@ -75,11 +75,10 @@ def __init__( in_channels=None, name=None # 'ternary_cnn2d', ): - super().__init__(name) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides - self.act = act self.padding = padding self.use_gemm = use_gemm self.data_format = data_format diff --git a/tensorlayer/layers/core.py b/tensorlayer/layers/core.py index bce74f389..2f2fa16d1 100644 --- a/tensorlayer/layers/core.py +++ b/tensorlayer/layers/core.py @@ -17,6 +17,15 @@ _global_layer_name_dict = {} # TODO: better implementation? +_act_dict = { + "relu": tf.nn.relu, + "relu6": tf.nn.relu6, + "leaky_relu": tf.nn.leaky_relu, + "softplus": tf.nn.softplus, + "tanh": tf.nn.tanh, + "sigmoid": tf.nn.sigmoid, +} + class Layer(object): """The basic :class:`Layer` class represents a single layer of a neural network. @@ -47,11 +56,12 @@ class Layer(object): """ - def __init__(self, name=None, *args, **kwargs): + def __init__(self, name=None, act=None, *args, **kwargs): """ Initializing the Layer. :param name: str or None + :param name: str or function or None """ # Layer constants @@ -84,6 +94,13 @@ def __init__(self, name=None, *args, **kwargs): _global_layer_name_dict[name] = 0 self.name = name + if act is not None: + if isinstance(act, str): + if act not in _act_dict.keys(): + raise RuntimeError("No activation function named {}.".format(act)) + self.act = _act_dict[act] + else: + self.act = act # Layer building state self._built = False diff --git a/tensorlayer/layers/dense/base_dense.py b/tensorlayer/layers/dense/base_dense.py index a5b800f04..e4c96af50 100644 --- a/tensorlayer/layers/dense/base_dense.py +++ b/tensorlayer/layers/dense/base_dense.py @@ -63,10 +63,9 @@ def __init__( name=None, # 'dense', ): - super(Dense, self).__init__(name) + super(Dense, self).__init__(name, act=act) self.n_units = n_units - self.act = act self.W_init = W_init self.b_init = b_init self.in_channels = in_channels diff --git a/tensorlayer/layers/dense/binary_dense.py b/tensorlayer/layers/dense/binary_dense.py index 4067ac4c3..77adde3b0 100644 --- a/tensorlayer/layers/dense/binary_dense.py +++ b/tensorlayer/layers/dense/binary_dense.py @@ -49,9 +49,8 @@ def __init__( in_channels=None, name=None, #'binary_dense', ): - super().__init__(name) + super().__init__(name, act=act) self.n_units = n_units - self.act = act self.use_gemm = use_gemm self.W_init = W_init self.b_init = b_init diff --git a/tensorlayer/layers/dense/dorefa_dense.py b/tensorlayer/layers/dense/dorefa_dense.py index 80ae3365c..07e9c339e 100644 --- a/tensorlayer/layers/dense/dorefa_dense.py +++ b/tensorlayer/layers/dense/dorefa_dense.py @@ -56,11 +56,10 @@ def __init__( in_channels=None, name=None, #'dorefa_dense', ): - super().__init__(name) + super().__init__(name, act=act) self.bitW = bitW self.bitA = bitA self.n_units = n_units - self.act = act self.use_gemm = use_gemm self.W_init = W_init self.b_init = b_init diff --git a/tensorlayer/layers/dense/dropconnect.py b/tensorlayer/layers/dense/dropconnect.py index d68e6c762..449221f42 100644 --- a/tensorlayer/layers/dense/dropconnect.py +++ b/tensorlayer/layers/dense/dropconnect.py @@ -65,14 +65,13 @@ def __init__( in_channels=None, name=None, # 'dropconnect', ): - super().__init__(name) + super().__init__(name, act=act) if isinstance(keep, numbers.Real) and not (keep > 0 and keep <= 1): raise ValueError("keep must be a scalar tensor or a float in the " "range (0, 1], got %g" % keep) self.keep = keep self.n_units = n_units - self.act = act self.W_init = W_init self.b_init = b_init self.in_channels = in_channels diff --git a/tensorlayer/layers/dense/quan_dense.py b/tensorlayer/layers/dense/quan_dense.py index 5a2513259..f5c42492f 100644 --- a/tensorlayer/layers/dense/quan_dense.py +++ b/tensorlayer/layers/dense/quan_dense.py @@ -54,9 +54,8 @@ def __init__( in_channels=None, name=None, #'quan_dense', ): - super().__init__(name) + super().__init__(name, act=act) self.n_units = n_units - self.act = act self.bitW = bitW self.bitA = bitA self.use_gemm = use_gemm diff --git a/tensorlayer/layers/dense/ternary_dense.py b/tensorlayer/layers/dense/ternary_dense.py index 27efb9090..778636155 100644 --- a/tensorlayer/layers/dense/ternary_dense.py +++ b/tensorlayer/layers/dense/ternary_dense.py @@ -49,9 +49,8 @@ def __init__( in_channels=None, name=None, #'ternary_dense', ): - super().__init__(name) + super().__init__(name, act=act) self.n_units = n_units - self.act = act self.use_gemm = use_gemm self.W_init = W_init self.b_init = b_init diff --git a/tensorlayer/layers/merge.py b/tensorlayer/layers/merge.py index 346a65962..8791578cc 100644 --- a/tensorlayer/layers/merge.py +++ b/tensorlayer/layers/merge.py @@ -111,9 +111,8 @@ def __init__( name=None, #'elementwise', ): - super(Elementwise, self).__init__(name) + super(Elementwise, self).__init__(name, act) self.combine_fn = combine_fn - self.act = act self.build(None) self._built = True diff --git a/tensorlayer/layers/normalization.py b/tensorlayer/layers/normalization.py index d8cec274c..1f2b25f81 100644 --- a/tensorlayer/layers/normalization.py +++ b/tensorlayer/layers/normalization.py @@ -195,8 +195,7 @@ def __init__( data_format='channels_last', name=None, ): - super(BatchNorm, self).__init__(name=name) - self.act = act + super(BatchNorm, self).__init__(name=name, act=act) self.decay = decay self.epsilon = epsilon self.data_format = data_format @@ -441,8 +440,7 @@ def __init__( gamma_init=tl.initializers.random_normal(mean=1.0, stddev=0.002), num_features=None, data_format='channels_last', name=None ): - super(InstanceNorm, self).__init__(name=name) - self.act = act + super(InstanceNorm, self).__init__(name=name, act=act) self.epsilon = epsilon self.beta_init = beta_init self.gamma_init = gamma_init @@ -651,10 +649,9 @@ def __init__( ): # super(LayerNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) - super(LayerNorm, self).__init__(name) + super(LayerNorm, self).__init__(name, act=act) self.center = center self.scale = scale - self.act = act self.epsilon = epsilon self.begin_norm_axis = begin_norm_axis self.begin_params_axis = begin_params_axis @@ -730,10 +727,9 @@ class GroupNorm(Layer): def __init__(self, groups=32, epsilon=1e-06, act=None, data_format='channels_last', name=None): #'groupnorm'): # super(GroupNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) - super().__init__(name) + super().__init__(name, act=act) self.groups = groups self.epsilon = epsilon - self.act = act self.data_format = data_format logging.info( @@ -847,8 +843,7 @@ def __init__( name=None, #'switchnorm', ): # super(SwitchNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) - super().__init__(name) - self.act = act + super().__init__(name, act=act) self.epsilon = epsilon self.beta_init = beta_init self.gamma_init = gamma_init diff --git a/tests/layers/test_layers_convolution.py b/tests/layers/test_layers_convolution.py index 0f5979d5b..5c9ca755e 100644 --- a/tests/layers/test_layers_convolution.py +++ b/tests/layers/test_layers_convolution.py @@ -33,7 +33,7 @@ def setUpClass(cls): )(cls.n2) cls.n4 = tl.layers.SeparableConv1d( - n_filter=32, filter_size=3, strides=2, padding='SAME', act=tf.nn.relu, name='separable_1d' + n_filter=32, filter_size=3, strides=2, padding='SAME', act='relu', name='separable_1d' )(cls.n3) cls.n5 = tl.layers.SubpixelConv1d(scale=2, act=tf.nn.relu, in_channels=32, name='subpixel_1d')(cls.n4) @@ -462,6 +462,29 @@ def test_layer_n4(self): # self.assertEqual(self.net2.count_params(), 19392) # self.assertEqual(self.net2.outputs.get_shape().as_list()[1:], [299, 299, 64]) + +class Exception_test(CustomTestCase): + + @classmethod + def setUpClass(cls): + print("##### begin testing exception in activation #####") + + def test_exception(cls): + + cls.batch_size = 5 + cls.inputs_shape = [cls.batch_size, 400, 400, 3] + cls.input_layer = Input(cls.inputs_shape, name='input_layer') + + try: + cls.n1 = tl.layers.Conv2dLayer( + act='activation', shape=(5, 5, 3, 32), strides=(1, 2, 2, 1), padding='SAME', + b_init=tf.constant_initializer(value=0.0), name='conv2dlayer' + )(cls.input_layer) + except Exception as e: + cls.assertIsInstance(e, RuntimeError) + print(e) + + if __name__ == '__main__': tl.logging.set_verbosity(tl.logging.DEBUG) diff --git a/tests/models/test_model_save_graph.py b/tests/models/test_model_save_graph.py index 95229938b..057ac921f 100644 --- a/tests/models/test_model_save_graph.py +++ b/tests/models/test_model_save_graph.py @@ -178,7 +178,7 @@ class Reuse_ModelLayer_test(CustomTestCase): @classmethod def setUpClass(cls): - print("##### begin testing save_graph, load_graph, including ModelLayer and reuse #####") + print("##### begin testing save_graph, load_graph, including ModelLayer and reuse #####") def test_save(self): input_shape = (None, 784) @@ -195,7 +195,7 @@ class Vgg_LayerList_test(CustomTestCase): @classmethod def setUpClass(cls): - print("##### begin testing save_graph, load_graph, including LayerList #####") + print("##### begin testing save_graph, load_graph, including LayerList #####") def test_save(self): M1 = tl.models.vgg16(mode='static') @@ -211,7 +211,7 @@ class List_inputs_outputs_test(CustomTestCase): @classmethod def setUpClass(cls): - print("##### begin testing model with list inputs and outputs #####") + print("##### begin testing model with list inputs and outputs #####") def test_list_inputs_outputs(self): ni_1 = Input(shape=[4, 16]) @@ -344,7 +344,7 @@ class ElementWise_lambda_test(CustomTestCase): @classmethod def setUpClass(cls): - print("##### begin testing elementwise lambda layer #####") + print("##### begin testing elementwise lambda layer #####") def test_elementwise_no_para_with_args(self): # z = mean + noise * tf.exp(std * 0.5) + foo @@ -467,7 +467,7 @@ class Dynamic_config_test(CustomTestCase): @classmethod def setUpClass(cls): - print("##### begin testing exception in dynamic mode #####") + print("##### begin testing exception in dynamic mode #####") def test_dynamic_config(self): M1 = basic_dynamic_model() @@ -480,7 +480,7 @@ class Exception_test(CustomTestCase): @classmethod def setUpClass(cls): - print("##### begin testing exception in dynamic mode #####") + print("##### begin testing exception in dynamic mode #####") def test_exception(self): M1 = basic_dynamic_model() From 02615219d3937f603c9653484a1d2173e8b233e0 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 20:25:06 +0800 Subject: [PATCH 15/19] fix bugs --- CHANGELOG.md | 2 +- docs/user/get_start_model.rst | 2 +- tensorlayer/layers/convolution/binary_conv.py | 2 +- .../layers/convolution/deformable_conv.py | 2 +- tensorlayer/layers/convolution/depthwise_conv.py | 2 +- tensorlayer/layers/convolution/dorefa_conv.py | 2 +- tensorlayer/layers/convolution/expert_conv.py | 6 +++--- tensorlayer/layers/convolution/expert_deconv.py | 6 +++--- tensorlayer/layers/convolution/group_conv.py | 2 +- tensorlayer/layers/convolution/quan_conv.py | 2 +- tensorlayer/layers/convolution/quan_conv_bn.py | 2 +- tensorlayer/layers/convolution/separable_conv.py | 4 ++-- .../layers/convolution/simplified_conv.py | 6 +++--- .../layers/convolution/simplified_deconv.py | 4 ++-- .../layers/convolution/super_resolution.py | 4 ++-- tensorlayer/layers/convolution/ternary_conv.py | 2 +- tensorlayer/layers/core.py | 4 ++-- tensorlayer/layers/dense/base_dense.py | 2 +- tensorlayer/layers/dense/binary_dense.py | 2 +- tensorlayer/layers/dense/dorefa_dense.py | 2 +- tensorlayer/layers/dense/dropconnect.py | 2 +- tensorlayer/layers/dense/quan_dense.py | 2 +- tensorlayer/layers/dense/quan_dense_bn.py | 2 +- tensorlayer/layers/dense/ternary_dense.py | 2 +- tensorlayer/layers/normalization.py | 16 ++++++++-------- 25 files changed, 42 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a1f58ffd..89939c54d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,7 +70,7 @@ To release a new version, please update the changelog as followed: ## [2.1.0] - 2019-5-25 ### Changed -- change the format of network config, change related code and files (PR #980) +- change the format of network config, change related code and files; change layer act (PR #980) ### Added diff --git a/docs/user/get_start_model.rst b/docs/user/get_start_model.rst index 3026c5198..261f1d880 100644 --- a/docs/user/get_start_model.rst +++ b/docs/user/get_start_model.rst @@ -208,7 +208,7 @@ z = f(x*W+b) class Dense(Layer): def __init__(self, n_units, act=None, in_channels=None, name=None): - super(Dense, self).__init__(name, act=act) + super(Dense, self).__init__(name, act=act, haveact=True) self.n_units = n_units self.in_channels = in_channels diff --git a/tensorlayer/layers/convolution/binary_conv.py b/tensorlayer/layers/convolution/binary_conv.py index cf55127d5..355eea0ad 100644 --- a/tensorlayer/layers/convolution/binary_conv.py +++ b/tensorlayer/layers/convolution/binary_conv.py @@ -75,7 +75,7 @@ def __init__( in_channels=None, name=None # 'binary_cnn2d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides diff --git a/tensorlayer/layers/convolution/deformable_conv.py b/tensorlayer/layers/convolution/deformable_conv.py index 616803ba1..48520fe15 100644 --- a/tensorlayer/layers/convolution/deformable_conv.py +++ b/tensorlayer/layers/convolution/deformable_conv.py @@ -83,7 +83,7 @@ def __init__( in_channels=None, name=None # 'deformable_conv_2d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.offset_layer = offset_layer self.n_filter = n_filter diff --git a/tensorlayer/layers/convolution/depthwise_conv.py b/tensorlayer/layers/convolution/depthwise_conv.py index b11233c27..782141926 100644 --- a/tensorlayer/layers/convolution/depthwise_conv.py +++ b/tensorlayer/layers/convolution/depthwise_conv.py @@ -82,7 +82,7 @@ def __init__( in_channels=None, name=None # 'depthwise_conv2d' ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.filter_size = filter_size self.strides = self._strides = strides self.padding = padding diff --git a/tensorlayer/layers/convolution/dorefa_conv.py b/tensorlayer/layers/convolution/dorefa_conv.py index dc7979967..21f2f15d4 100644 --- a/tensorlayer/layers/convolution/dorefa_conv.py +++ b/tensorlayer/layers/convolution/dorefa_conv.py @@ -81,7 +81,7 @@ def __init__( in_channels=None, name=None # 'dorefa_cnn2d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.bitW = bitW self.bitA = bitA self.n_filter = n_filter diff --git a/tensorlayer/layers/convolution/expert_conv.py b/tensorlayer/layers/convolution/expert_conv.py index 50ea12cb9..a4c241454 100644 --- a/tensorlayer/layers/convolution/expert_conv.py +++ b/tensorlayer/layers/convolution/expert_conv.py @@ -71,7 +71,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn1d_layer', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = shape[-1] self.filter_size = shape[0] self.shape = shape @@ -190,7 +190,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn2d_layer', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = shape[-1] self.filter_size = (shape[0], shape[1]) self.shape = shape @@ -308,7 +308,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn3d_layer' ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = shape[-1] self.filter_size = (shape[0], shape[1], shape[2]) self.shape = shape diff --git a/tensorlayer/layers/convolution/expert_deconv.py b/tensorlayer/layers/convolution/expert_deconv.py index f23c752ad..7a5ef874e 100644 --- a/tensorlayer/layers/convolution/expert_deconv.py +++ b/tensorlayer/layers/convolution/expert_deconv.py @@ -80,7 +80,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn1d_layer', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides @@ -214,7 +214,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn2d_layer', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides @@ -340,7 +340,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn3d_layer', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides diff --git a/tensorlayer/layers/convolution/group_conv.py b/tensorlayer/layers/convolution/group_conv.py index bc35d4e00..c9555df72 100644 --- a/tensorlayer/layers/convolution/group_conv.py +++ b/tensorlayer/layers/convolution/group_conv.py @@ -73,7 +73,7 @@ def __init__( in_channels=None, name=None # 'groupconv', ): # Windaway - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides diff --git a/tensorlayer/layers/convolution/quan_conv.py b/tensorlayer/layers/convolution/quan_conv.py index 75ee3943c..12302e8b5 100644 --- a/tensorlayer/layers/convolution/quan_conv.py +++ b/tensorlayer/layers/convolution/quan_conv.py @@ -82,7 +82,7 @@ def __init__( in_channels=None, name=None # 'quan_cnn2d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.bitW = bitW self.bitA = bitA self.n_filter = n_filter diff --git a/tensorlayer/layers/convolution/quan_conv_bn.py b/tensorlayer/layers/convolution/quan_conv_bn.py index ef0f9bfda..ee3ce5c97 100644 --- a/tensorlayer/layers/convolution/quan_conv_bn.py +++ b/tensorlayer/layers/convolution/quan_conv_bn.py @@ -113,7 +113,7 @@ def __init__( data_format=None, name='quan_cnn2d_bn', ): - super(QuanConv2dWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name) + super(QuanConv2dWithBN, self).__init__(prev_layer=prev_layer, act=act, haveact=True, W_init_args=W_init_args, name=name) logging.info( "QuanConv2dWithBN %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s " % ( diff --git a/tensorlayer/layers/convolution/separable_conv.py b/tensorlayer/layers/convolution/separable_conv.py index ca1c66d49..71ebb6e41 100644 --- a/tensorlayer/layers/convolution/separable_conv.py +++ b/tensorlayer/layers/convolution/separable_conv.py @@ -84,7 +84,7 @@ def __init__( in_channels=None, name=None # 'seperable1d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides @@ -231,7 +231,7 @@ def __init__( in_channels=None, name=None # 'seperable2d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides diff --git a/tensorlayer/layers/convolution/simplified_conv.py b/tensorlayer/layers/convolution/simplified_conv.py index 536d4e52e..6a6d378b1 100644 --- a/tensorlayer/layers/convolution/simplified_conv.py +++ b/tensorlayer/layers/convolution/simplified_conv.py @@ -70,7 +70,7 @@ def __init__( in_channels=None, name=None # 'conv1d' ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self.stride = stride @@ -200,7 +200,7 @@ def __init__( in_channels=None, name=None # 'conv2d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self._strides = self.strides = strides @@ -333,7 +333,7 @@ def __init__( in_channels=None, name=None # 'conv3d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self._strides = self.strides = strides diff --git a/tensorlayer/layers/convolution/simplified_deconv.py b/tensorlayer/layers/convolution/simplified_deconv.py index 57beff0f4..eb3c72fec 100644 --- a/tensorlayer/layers/convolution/simplified_deconv.py +++ b/tensorlayer/layers/convolution/simplified_deconv.py @@ -71,7 +71,7 @@ def __init__( in_channels=None, name=None # 'decnn2d' ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides @@ -198,7 +198,7 @@ def __init__( in_channels=None, name=None # 'decnn3d' ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides diff --git a/tensorlayer/layers/convolution/super_resolution.py b/tensorlayer/layers/convolution/super_resolution.py index 21d765e03..837a61b36 100644 --- a/tensorlayer/layers/convolution/super_resolution.py +++ b/tensorlayer/layers/convolution/super_resolution.py @@ -53,7 +53,7 @@ def __init__( in_channels=None, name=None # 'subpixel_conv1d' ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.scale = scale self.in_channels = in_channels self.out_channels = int(self.in_channels / self.scale) @@ -150,7 +150,7 @@ def __init__( in_channels=None, name=None # 'subpixel_conv2d' ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.scale = scale self.n_out_channels = n_out_channels self.in_channels = in_channels diff --git a/tensorlayer/layers/convolution/ternary_conv.py b/tensorlayer/layers/convolution/ternary_conv.py index 33e01507c..52ec7b6f0 100644 --- a/tensorlayer/layers/convolution/ternary_conv.py +++ b/tensorlayer/layers/convolution/ternary_conv.py @@ -75,7 +75,7 @@ def __init__( in_channels=None, name=None # 'ternary_cnn2d', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides diff --git a/tensorlayer/layers/core.py b/tensorlayer/layers/core.py index 2f2fa16d1..ff76302f0 100644 --- a/tensorlayer/layers/core.py +++ b/tensorlayer/layers/core.py @@ -56,7 +56,7 @@ class Layer(object): """ - def __init__(self, name=None, act=None, *args, **kwargs): + def __init__(self, name=None, act=None, haveact=False, *args, **kwargs): """ Initializing the Layer. @@ -94,7 +94,7 @@ def __init__(self, name=None, act=None, *args, **kwargs): _global_layer_name_dict[name] = 0 self.name = name - if act is not None: + if haveact: if isinstance(act, str): if act not in _act_dict.keys(): raise RuntimeError("No activation function named {}.".format(act)) diff --git a/tensorlayer/layers/dense/base_dense.py b/tensorlayer/layers/dense/base_dense.py index e4c96af50..0a613add0 100644 --- a/tensorlayer/layers/dense/base_dense.py +++ b/tensorlayer/layers/dense/base_dense.py @@ -63,7 +63,7 @@ def __init__( name=None, # 'dense', ): - super(Dense, self).__init__(name, act=act) + super(Dense, self).__init__(name, act=act, haveact=True) self.n_units = n_units self.W_init = W_init diff --git a/tensorlayer/layers/dense/binary_dense.py b/tensorlayer/layers/dense/binary_dense.py index 77adde3b0..4dcc041eb 100644 --- a/tensorlayer/layers/dense/binary_dense.py +++ b/tensorlayer/layers/dense/binary_dense.py @@ -49,7 +49,7 @@ def __init__( in_channels=None, name=None, #'binary_dense', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_units = n_units self.use_gemm = use_gemm self.W_init = W_init diff --git a/tensorlayer/layers/dense/dorefa_dense.py b/tensorlayer/layers/dense/dorefa_dense.py index 07e9c339e..ae823e8b5 100644 --- a/tensorlayer/layers/dense/dorefa_dense.py +++ b/tensorlayer/layers/dense/dorefa_dense.py @@ -56,7 +56,7 @@ def __init__( in_channels=None, name=None, #'dorefa_dense', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.bitW = bitW self.bitA = bitA self.n_units = n_units diff --git a/tensorlayer/layers/dense/dropconnect.py b/tensorlayer/layers/dense/dropconnect.py index 449221f42..2a65050a7 100644 --- a/tensorlayer/layers/dense/dropconnect.py +++ b/tensorlayer/layers/dense/dropconnect.py @@ -65,7 +65,7 @@ def __init__( in_channels=None, name=None, # 'dropconnect', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) if isinstance(keep, numbers.Real) and not (keep > 0 and keep <= 1): raise ValueError("keep must be a scalar tensor or a float in the " "range (0, 1], got %g" % keep) diff --git a/tensorlayer/layers/dense/quan_dense.py b/tensorlayer/layers/dense/quan_dense.py index f5c42492f..b64112de5 100644 --- a/tensorlayer/layers/dense/quan_dense.py +++ b/tensorlayer/layers/dense/quan_dense.py @@ -54,7 +54,7 @@ def __init__( in_channels=None, name=None, #'quan_dense', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_units = n_units self.bitW = bitW self.bitA = bitA diff --git a/tensorlayer/layers/dense/quan_dense_bn.py b/tensorlayer/layers/dense/quan_dense_bn.py index e647a7e6e..c69324b1b 100644 --- a/tensorlayer/layers/dense/quan_dense_bn.py +++ b/tensorlayer/layers/dense/quan_dense_bn.py @@ -84,7 +84,7 @@ def __init__( W_init_args=None, name=None, #'quan_dense_with_bn', ): - super(QuanDenseLayerWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name) + super(QuanDenseLayerWithBN, self).__init__(prev_layer=prev_layer, act=act, haveact=True, W_init_args=W_init_args, name=name) logging.info( "QuanDenseLayerWithBN %s: %d %s" % diff --git a/tensorlayer/layers/dense/ternary_dense.py b/tensorlayer/layers/dense/ternary_dense.py index 778636155..0cd636693 100644 --- a/tensorlayer/layers/dense/ternary_dense.py +++ b/tensorlayer/layers/dense/ternary_dense.py @@ -49,7 +49,7 @@ def __init__( in_channels=None, name=None, #'ternary_dense', ): - super().__init__(name, act=act) + super().__init__(name, act=act, haveact=True) self.n_units = n_units self.use_gemm = use_gemm self.W_init = W_init diff --git a/tensorlayer/layers/normalization.py b/tensorlayer/layers/normalization.py index 1f2b25f81..ee90045d3 100644 --- a/tensorlayer/layers/normalization.py +++ b/tensorlayer/layers/normalization.py @@ -195,7 +195,7 @@ def __init__( data_format='channels_last', name=None, ): - super(BatchNorm, self).__init__(name=name, act=act) + super(BatchNorm, self).__init__(name=name, act=act, haveact=True) self.decay = decay self.epsilon = epsilon self.data_format = data_format @@ -440,7 +440,7 @@ def __init__( gamma_init=tl.initializers.random_normal(mean=1.0, stddev=0.002), num_features=None, data_format='channels_last', name=None ): - super(InstanceNorm, self).__init__(name=name, act=act) + super(InstanceNorm, self).__init__(name=name, act=act, haveact=True) self.epsilon = epsilon self.beta_init = beta_init self.gamma_init = gamma_init @@ -648,8 +648,8 @@ def __init__( name=None, ): - # super(LayerNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) - super(LayerNorm, self).__init__(name, act=act) + # super(LayerNorm, self).__init__(prev_layer=prev_layer, act=act, haveact=True, name=name) + super(LayerNorm, self).__init__(name, act=act, haveact=True) self.center = center self.scale = scale self.epsilon = epsilon @@ -726,8 +726,8 @@ class GroupNorm(Layer): """ def __init__(self, groups=32, epsilon=1e-06, act=None, data_format='channels_last', name=None): #'groupnorm'): - # super(GroupNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) - super().__init__(name, act=act) + # super(GroupNorm, self).__init__(prev_layer=prev_layer, act=act, haveact=True, name=name) + super().__init__(name, act=act, haveact=True) self.groups = groups self.epsilon = epsilon self.data_format = data_format @@ -842,8 +842,8 @@ def __init__( data_format='channels_last', name=None, #'switchnorm', ): - # super(SwitchNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) - super().__init__(name, act=act) + # super(SwitchNorm, self).__init__(prev_layer=prev_layer, act=act, haveact=True, name=name) + super().__init__(name, act=act, haveact=True) self.epsilon = epsilon self.beta_init = beta_init self.gamma_init = gamma_init From 5549234233714467dbd6e246dcb445ad27531cc1 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 20:39:08 +0800 Subject: [PATCH 16/19] fix bug --- tensorlayer/layers/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorlayer/layers/merge.py b/tensorlayer/layers/merge.py index 8791578cc..613ef1f90 100644 --- a/tensorlayer/layers/merge.py +++ b/tensorlayer/layers/merge.py @@ -111,7 +111,7 @@ def __init__( name=None, #'elementwise', ): - super(Elementwise, self).__init__(name, act) + super(Elementwise, self).__init__(name, act=act, haveact=True) self.combine_fn = combine_fn self.build(None) From 0f1ae82de2771da4b4332a5bfc0c94ae310214eb Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Sat, 25 May 2019 21:05:35 +0800 Subject: [PATCH 17/19] fix bugs --- docs/user/get_start_model.rst | 2 +- tensorlayer/layers/convolution/binary_conv.py | 2 +- .../layers/convolution/deformable_conv.py | 2 +- .../layers/convolution/depthwise_conv.py | 2 +- tensorlayer/layers/convolution/dorefa_conv.py | 2 +- tensorlayer/layers/convolution/expert_conv.py | 6 +++--- .../layers/convolution/expert_deconv.py | 6 +++--- tensorlayer/layers/convolution/group_conv.py | 2 +- tensorlayer/layers/convolution/quan_conv.py | 2 +- .../layers/convolution/quan_conv_bn.py | 2 +- .../layers/convolution/separable_conv.py | 4 ++-- .../layers/convolution/simplified_conv.py | 6 +++--- .../layers/convolution/simplified_deconv.py | 4 ++-- .../layers/convolution/super_resolution.py | 4 ++-- .../layers/convolution/ternary_conv.py | 2 +- tensorlayer/layers/core.py | 20 +++++++++++-------- tensorlayer/layers/dense/base_dense.py | 2 +- tensorlayer/layers/dense/binary_dense.py | 2 +- tensorlayer/layers/dense/dorefa_dense.py | 2 +- tensorlayer/layers/dense/dropconnect.py | 2 +- tensorlayer/layers/dense/quan_dense.py | 2 +- tensorlayer/layers/dense/quan_dense_bn.py | 2 +- tensorlayer/layers/dense/ternary_dense.py | 2 +- tensorlayer/layers/merge.py | 2 +- tensorlayer/layers/normalization.py | 16 +++++++-------- tests/layers/test_layers_convolution.py | 4 ++-- 26 files changed, 54 insertions(+), 50 deletions(-) diff --git a/docs/user/get_start_model.rst b/docs/user/get_start_model.rst index 261f1d880..3026c5198 100644 --- a/docs/user/get_start_model.rst +++ b/docs/user/get_start_model.rst @@ -208,7 +208,7 @@ z = f(x*W+b) class Dense(Layer): def __init__(self, n_units, act=None, in_channels=None, name=None): - super(Dense, self).__init__(name, act=act, haveact=True) + super(Dense, self).__init__(name, act=act) self.n_units = n_units self.in_channels = in_channels diff --git a/tensorlayer/layers/convolution/binary_conv.py b/tensorlayer/layers/convolution/binary_conv.py index 355eea0ad..cf55127d5 100644 --- a/tensorlayer/layers/convolution/binary_conv.py +++ b/tensorlayer/layers/convolution/binary_conv.py @@ -75,7 +75,7 @@ def __init__( in_channels=None, name=None # 'binary_cnn2d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides diff --git a/tensorlayer/layers/convolution/deformable_conv.py b/tensorlayer/layers/convolution/deformable_conv.py index 48520fe15..616803ba1 100644 --- a/tensorlayer/layers/convolution/deformable_conv.py +++ b/tensorlayer/layers/convolution/deformable_conv.py @@ -83,7 +83,7 @@ def __init__( in_channels=None, name=None # 'deformable_conv_2d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.offset_layer = offset_layer self.n_filter = n_filter diff --git a/tensorlayer/layers/convolution/depthwise_conv.py b/tensorlayer/layers/convolution/depthwise_conv.py index 782141926..b11233c27 100644 --- a/tensorlayer/layers/convolution/depthwise_conv.py +++ b/tensorlayer/layers/convolution/depthwise_conv.py @@ -82,7 +82,7 @@ def __init__( in_channels=None, name=None # 'depthwise_conv2d' ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.filter_size = filter_size self.strides = self._strides = strides self.padding = padding diff --git a/tensorlayer/layers/convolution/dorefa_conv.py b/tensorlayer/layers/convolution/dorefa_conv.py index 21f2f15d4..dc7979967 100644 --- a/tensorlayer/layers/convolution/dorefa_conv.py +++ b/tensorlayer/layers/convolution/dorefa_conv.py @@ -81,7 +81,7 @@ def __init__( in_channels=None, name=None # 'dorefa_cnn2d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.bitW = bitW self.bitA = bitA self.n_filter = n_filter diff --git a/tensorlayer/layers/convolution/expert_conv.py b/tensorlayer/layers/convolution/expert_conv.py index a4c241454..50ea12cb9 100644 --- a/tensorlayer/layers/convolution/expert_conv.py +++ b/tensorlayer/layers/convolution/expert_conv.py @@ -71,7 +71,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn1d_layer', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = shape[-1] self.filter_size = shape[0] self.shape = shape @@ -190,7 +190,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn2d_layer', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = shape[-1] self.filter_size = (shape[0], shape[1]) self.shape = shape @@ -308,7 +308,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'cnn3d_layer' ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = shape[-1] self.filter_size = (shape[0], shape[1], shape[2]) self.shape = shape diff --git a/tensorlayer/layers/convolution/expert_deconv.py b/tensorlayer/layers/convolution/expert_deconv.py index 7a5ef874e..f23c752ad 100644 --- a/tensorlayer/layers/convolution/expert_deconv.py +++ b/tensorlayer/layers/convolution/expert_deconv.py @@ -80,7 +80,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn1d_layer', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides @@ -214,7 +214,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn2d_layer', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides @@ -340,7 +340,7 @@ def __init__( b_init=tl.initializers.constant(value=0.0), name=None # 'decnn3d_layer', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.shape = shape self.outputs_shape = outputs_shape self.strides = strides diff --git a/tensorlayer/layers/convolution/group_conv.py b/tensorlayer/layers/convolution/group_conv.py index c9555df72..bc35d4e00 100644 --- a/tensorlayer/layers/convolution/group_conv.py +++ b/tensorlayer/layers/convolution/group_conv.py @@ -73,7 +73,7 @@ def __init__( in_channels=None, name=None # 'groupconv', ): # Windaway - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides diff --git a/tensorlayer/layers/convolution/quan_conv.py b/tensorlayer/layers/convolution/quan_conv.py index 12302e8b5..75ee3943c 100644 --- a/tensorlayer/layers/convolution/quan_conv.py +++ b/tensorlayer/layers/convolution/quan_conv.py @@ -82,7 +82,7 @@ def __init__( in_channels=None, name=None # 'quan_cnn2d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.bitW = bitW self.bitA = bitA self.n_filter = n_filter diff --git a/tensorlayer/layers/convolution/quan_conv_bn.py b/tensorlayer/layers/convolution/quan_conv_bn.py index ee3ce5c97..ef0f9bfda 100644 --- a/tensorlayer/layers/convolution/quan_conv_bn.py +++ b/tensorlayer/layers/convolution/quan_conv_bn.py @@ -113,7 +113,7 @@ def __init__( data_format=None, name='quan_cnn2d_bn', ): - super(QuanConv2dWithBN, self).__init__(prev_layer=prev_layer, act=act, haveact=True, W_init_args=W_init_args, name=name) + super(QuanConv2dWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name) logging.info( "QuanConv2dWithBN %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s " % ( diff --git a/tensorlayer/layers/convolution/separable_conv.py b/tensorlayer/layers/convolution/separable_conv.py index 71ebb6e41..ca1c66d49 100644 --- a/tensorlayer/layers/convolution/separable_conv.py +++ b/tensorlayer/layers/convolution/separable_conv.py @@ -84,7 +84,7 @@ def __init__( in_channels=None, name=None # 'seperable1d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides @@ -231,7 +231,7 @@ def __init__( in_channels=None, name=None # 'seperable2d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides diff --git a/tensorlayer/layers/convolution/simplified_conv.py b/tensorlayer/layers/convolution/simplified_conv.py index 6a6d378b1..536d4e52e 100644 --- a/tensorlayer/layers/convolution/simplified_conv.py +++ b/tensorlayer/layers/convolution/simplified_conv.py @@ -70,7 +70,7 @@ def __init__( in_channels=None, name=None # 'conv1d' ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.stride = stride @@ -200,7 +200,7 @@ def __init__( in_channels=None, name=None # 'conv2d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self._strides = self.strides = strides @@ -333,7 +333,7 @@ def __init__( in_channels=None, name=None # 'conv3d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self._strides = self.strides = strides diff --git a/tensorlayer/layers/convolution/simplified_deconv.py b/tensorlayer/layers/convolution/simplified_deconv.py index eb3c72fec..57beff0f4 100644 --- a/tensorlayer/layers/convolution/simplified_deconv.py +++ b/tensorlayer/layers/convolution/simplified_deconv.py @@ -71,7 +71,7 @@ def __init__( in_channels=None, name=None # 'decnn2d' ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides @@ -198,7 +198,7 @@ def __init__( in_channels=None, name=None # 'decnn3d' ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = strides diff --git a/tensorlayer/layers/convolution/super_resolution.py b/tensorlayer/layers/convolution/super_resolution.py index 837a61b36..21d765e03 100644 --- a/tensorlayer/layers/convolution/super_resolution.py +++ b/tensorlayer/layers/convolution/super_resolution.py @@ -53,7 +53,7 @@ def __init__( in_channels=None, name=None # 'subpixel_conv1d' ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.scale = scale self.in_channels = in_channels self.out_channels = int(self.in_channels / self.scale) @@ -150,7 +150,7 @@ def __init__( in_channels=None, name=None # 'subpixel_conv2d' ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.scale = scale self.n_out_channels = n_out_channels self.in_channels = in_channels diff --git a/tensorlayer/layers/convolution/ternary_conv.py b/tensorlayer/layers/convolution/ternary_conv.py index 52ec7b6f0..33e01507c 100644 --- a/tensorlayer/layers/convolution/ternary_conv.py +++ b/tensorlayer/layers/convolution/ternary_conv.py @@ -75,7 +75,7 @@ def __init__( in_channels=None, name=None # 'ternary_cnn2d', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_filter = n_filter self.filter_size = filter_size self.strides = self._strides = strides diff --git a/tensorlayer/layers/core.py b/tensorlayer/layers/core.py index ff76302f0..dce3c34dd 100644 --- a/tensorlayer/layers/core.py +++ b/tensorlayer/layers/core.py @@ -21,12 +21,19 @@ "relu": tf.nn.relu, "relu6": tf.nn.relu6, "leaky_relu": tf.nn.leaky_relu, + "lrelu0.2": tf.nn.leaky_relu, "softplus": tf.nn.softplus, "tanh": tf.nn.tanh, "sigmoid": tf.nn.sigmoid, } +def str2act(act): + if act not in _act_dict.keys(): + raise Exception("Unsupported act {}".format(act)) + return _act_dict[act] + + class Layer(object): """The basic :class:`Layer` class represents a single layer of a neural network. @@ -56,7 +63,7 @@ class Layer(object): """ - def __init__(self, name=None, act=None, haveact=False, *args, **kwargs): + def __init__(self, name=None, act=None, *args, **kwargs): """ Initializing the Layer. @@ -94,13 +101,10 @@ def __init__(self, name=None, act=None, haveact=False, *args, **kwargs): _global_layer_name_dict[name] = 0 self.name = name - if haveact: - if isinstance(act, str): - if act not in _act_dict.keys(): - raise RuntimeError("No activation function named {}.".format(act)) - self.act = _act_dict[act] - else: - self.act = act + if isinstance(act, str): + self.act = str2act(act) + else: + self.act = act # Layer building state self._built = False diff --git a/tensorlayer/layers/dense/base_dense.py b/tensorlayer/layers/dense/base_dense.py index 0a613add0..e4c96af50 100644 --- a/tensorlayer/layers/dense/base_dense.py +++ b/tensorlayer/layers/dense/base_dense.py @@ -63,7 +63,7 @@ def __init__( name=None, # 'dense', ): - super(Dense, self).__init__(name, act=act, haveact=True) + super(Dense, self).__init__(name, act=act) self.n_units = n_units self.W_init = W_init diff --git a/tensorlayer/layers/dense/binary_dense.py b/tensorlayer/layers/dense/binary_dense.py index 4dcc041eb..77adde3b0 100644 --- a/tensorlayer/layers/dense/binary_dense.py +++ b/tensorlayer/layers/dense/binary_dense.py @@ -49,7 +49,7 @@ def __init__( in_channels=None, name=None, #'binary_dense', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_units = n_units self.use_gemm = use_gemm self.W_init = W_init diff --git a/tensorlayer/layers/dense/dorefa_dense.py b/tensorlayer/layers/dense/dorefa_dense.py index ae823e8b5..07e9c339e 100644 --- a/tensorlayer/layers/dense/dorefa_dense.py +++ b/tensorlayer/layers/dense/dorefa_dense.py @@ -56,7 +56,7 @@ def __init__( in_channels=None, name=None, #'dorefa_dense', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.bitW = bitW self.bitA = bitA self.n_units = n_units diff --git a/tensorlayer/layers/dense/dropconnect.py b/tensorlayer/layers/dense/dropconnect.py index 2a65050a7..449221f42 100644 --- a/tensorlayer/layers/dense/dropconnect.py +++ b/tensorlayer/layers/dense/dropconnect.py @@ -65,7 +65,7 @@ def __init__( in_channels=None, name=None, # 'dropconnect', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) if isinstance(keep, numbers.Real) and not (keep > 0 and keep <= 1): raise ValueError("keep must be a scalar tensor or a float in the " "range (0, 1], got %g" % keep) diff --git a/tensorlayer/layers/dense/quan_dense.py b/tensorlayer/layers/dense/quan_dense.py index b64112de5..f5c42492f 100644 --- a/tensorlayer/layers/dense/quan_dense.py +++ b/tensorlayer/layers/dense/quan_dense.py @@ -54,7 +54,7 @@ def __init__( in_channels=None, name=None, #'quan_dense', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_units = n_units self.bitW = bitW self.bitA = bitA diff --git a/tensorlayer/layers/dense/quan_dense_bn.py b/tensorlayer/layers/dense/quan_dense_bn.py index c69324b1b..e647a7e6e 100644 --- a/tensorlayer/layers/dense/quan_dense_bn.py +++ b/tensorlayer/layers/dense/quan_dense_bn.py @@ -84,7 +84,7 @@ def __init__( W_init_args=None, name=None, #'quan_dense_with_bn', ): - super(QuanDenseLayerWithBN, self).__init__(prev_layer=prev_layer, act=act, haveact=True, W_init_args=W_init_args, name=name) + super(QuanDenseLayerWithBN, self).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, name=name) logging.info( "QuanDenseLayerWithBN %s: %d %s" % diff --git a/tensorlayer/layers/dense/ternary_dense.py b/tensorlayer/layers/dense/ternary_dense.py index 0cd636693..778636155 100644 --- a/tensorlayer/layers/dense/ternary_dense.py +++ b/tensorlayer/layers/dense/ternary_dense.py @@ -49,7 +49,7 @@ def __init__( in_channels=None, name=None, #'ternary_dense', ): - super().__init__(name, act=act, haveact=True) + super().__init__(name, act=act) self.n_units = n_units self.use_gemm = use_gemm self.W_init = W_init diff --git a/tensorlayer/layers/merge.py b/tensorlayer/layers/merge.py index 613ef1f90..6f49374ca 100644 --- a/tensorlayer/layers/merge.py +++ b/tensorlayer/layers/merge.py @@ -111,7 +111,7 @@ def __init__( name=None, #'elementwise', ): - super(Elementwise, self).__init__(name, act=act, haveact=True) + super(Elementwise, self).__init__(name, act=act) self.combine_fn = combine_fn self.build(None) diff --git a/tensorlayer/layers/normalization.py b/tensorlayer/layers/normalization.py index ee90045d3..1f2b25f81 100644 --- a/tensorlayer/layers/normalization.py +++ b/tensorlayer/layers/normalization.py @@ -195,7 +195,7 @@ def __init__( data_format='channels_last', name=None, ): - super(BatchNorm, self).__init__(name=name, act=act, haveact=True) + super(BatchNorm, self).__init__(name=name, act=act) self.decay = decay self.epsilon = epsilon self.data_format = data_format @@ -440,7 +440,7 @@ def __init__( gamma_init=tl.initializers.random_normal(mean=1.0, stddev=0.002), num_features=None, data_format='channels_last', name=None ): - super(InstanceNorm, self).__init__(name=name, act=act, haveact=True) + super(InstanceNorm, self).__init__(name=name, act=act) self.epsilon = epsilon self.beta_init = beta_init self.gamma_init = gamma_init @@ -648,8 +648,8 @@ def __init__( name=None, ): - # super(LayerNorm, self).__init__(prev_layer=prev_layer, act=act, haveact=True, name=name) - super(LayerNorm, self).__init__(name, act=act, haveact=True) + # super(LayerNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) + super(LayerNorm, self).__init__(name, act=act) self.center = center self.scale = scale self.epsilon = epsilon @@ -726,8 +726,8 @@ class GroupNorm(Layer): """ def __init__(self, groups=32, epsilon=1e-06, act=None, data_format='channels_last', name=None): #'groupnorm'): - # super(GroupNorm, self).__init__(prev_layer=prev_layer, act=act, haveact=True, name=name) - super().__init__(name, act=act, haveact=True) + # super(GroupNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) + super().__init__(name, act=act) self.groups = groups self.epsilon = epsilon self.data_format = data_format @@ -842,8 +842,8 @@ def __init__( data_format='channels_last', name=None, #'switchnorm', ): - # super(SwitchNorm, self).__init__(prev_layer=prev_layer, act=act, haveact=True, name=name) - super().__init__(name, act=act, haveact=True) + # super(SwitchNorm, self).__init__(prev_layer=prev_layer, act=act, name=name) + super().__init__(name, act=act) self.epsilon = epsilon self.beta_init = beta_init self.gamma_init = gamma_init diff --git a/tests/layers/test_layers_convolution.py b/tests/layers/test_layers_convolution.py index 5c9ca755e..b768600de 100644 --- a/tests/layers/test_layers_convolution.py +++ b/tests/layers/test_layers_convolution.py @@ -467,7 +467,7 @@ class Exception_test(CustomTestCase): @classmethod def setUpClass(cls): - print("##### begin testing exception in activation #####") + print("##### begin testing exception in activation #####") def test_exception(cls): @@ -481,7 +481,7 @@ def test_exception(cls): b_init=tf.constant_initializer(value=0.0), name='conv2dlayer' )(cls.input_layer) except Exception as e: - cls.assertIsInstance(e, RuntimeError) + cls.assertIsInstance(e, Exception) print(e) From 65ee5a6f9130b09bf8e5cc1903b566f1ed7ea5eb Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Wed, 29 May 2019 20:03:25 +0800 Subject: [PATCH 18/19] parse float in lrelu --- tensorlayer/layers/core.py | 18 +++- tests/layers/test_layers_core_act.py | 122 +++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 tests/layers/test_layers_core_act.py diff --git a/tensorlayer/layers/core.py b/tensorlayer/layers/core.py index dce3c34dd..d98ea312e 100644 --- a/tensorlayer/layers/core.py +++ b/tensorlayer/layers/core.py @@ -21,7 +21,7 @@ "relu": tf.nn.relu, "relu6": tf.nn.relu6, "leaky_relu": tf.nn.leaky_relu, - "lrelu0.2": tf.nn.leaky_relu, + "lrelu": tf.nn.leaky_relu, "softplus": tf.nn.softplus, "tanh": tf.nn.tanh, "sigmoid": tf.nn.sigmoid, @@ -29,8 +29,22 @@ def str2act(act): + if len(act) > 5 and act[0:5] == "lrelu": + try: + alpha = float(act[5:]) + return lambda x: tf.nn.leaky_relu(x, alpha=alpha) + except Exception as e: + raise Exception("{} can not be parsed as a float".format(act[5:])) + + if len(act) > 10 and act[0:10] == "leaky_relu": + try: + alpha = float(act[10:]) + return lambda x: tf.nn.leaky_relu(x, alpha=alpha) + except Exception as e: + raise Exception("{} can not be parsed as a float".format(act[10:])) + if act not in _act_dict.keys(): - raise Exception("Unsupported act {}".format(act)) + raise Exception("Unsupported act: {}".format(act)) return _act_dict[act] diff --git a/tests/layers/test_layers_core_act.py b/tests/layers/test_layers_core_act.py new file mode 100644 index 000000000..74cfb6441 --- /dev/null +++ b/tests/layers/test_layers_core_act.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import unittest + +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' + +import tensorflow as tf +import tensorlayer as tl +from tensorlayer.layers import * +from tensorlayer.models import * + +from tests.utils import CustomTestCase + + +class Layer_Convolution_2D_Test(CustomTestCase): + + @classmethod + def setUpClass(cls): + print("##### begin testing activation #####") + + + @classmethod + def tearDownClass(cls): + pass + # tf.reset_default_graph() + + def test_layer_core_act(cls): + + cls.batch_size = 5 + cls.inputs_shape = [cls.batch_size, 400, 400, 3] + cls.input_layer = Input(cls.inputs_shape, name='input_layer') + + cls.n1 = tl.layers.Conv2dLayer( + act=tf.nn.relu, shape=(5, 5, 3, 32), strides=(1, 2, 2, 1), padding='SAME', + b_init=tf.constant_initializer(value=0.0), name='conv2dlayer' + )(cls.input_layer) + + cls.n2 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="relu", name='conv2d' + )(cls.n1) + + cls.n3 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="leaky_relu", b_init=None + )(cls.n2) + + cls.n4 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="lrelu", b_init=None + )(cls.n2) + + cls.n5 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="sigmoid", in_channels=32 + )(cls.n4) + + cls.n6 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="tanh", in_channels=32 + )(cls.n5) + + cls.n7 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="leaky_relu0.22", in_channels=32 + )(cls.n6) + + cls.n8 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="lrelu0.22", in_channels=32 + )(cls.n7) + + cls.n9 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="softplus", in_channels=32 + )(cls.n8) + + cls.n10 = tl.layers.Conv2d( + n_filter=32, filter_size=(3, 3), strides=(2, 2), act="relu6", in_channels=32 + )(cls.n9) + + cls.model = Model(cls.input_layer, cls.n8) + + +class Exception_test(CustomTestCase): + + @classmethod + def setUpClass(cls): + print("##### begin testing exception in activation #####") + + def test_exception(cls): + + cls.batch_size = 5 + cls.inputs_shape = [cls.batch_size, 400, 400, 3] + cls.input_layer = Input(cls.inputs_shape, name='input_layer') + + try: + cls.n1 = tl.layers.Conv2dLayer( + act='activation', shape=(5, 5, 3, 32), strides=(1, 2, 2, 1), padding='SAME', + b_init=tf.constant_initializer(value=0.0), name='conv2dlayer' + )(cls.input_layer) + except Exception as e: + cls.assertIsInstance(e, Exception) + print(e) + + try: + cls.n2 = tl.layers.Conv2dLayer( + act='leaky_relu0.2x', shape=(5, 5, 3, 32), strides=(1, 2, 2, 1), padding='SAME', + b_init=tf.constant_initializer(value=0.0), name='conv2dlayer' + )(cls.input_layer) + except Exception as e: + cls.assertIsInstance(e, Exception) + print(e) + + try: + cls.n3 = tl.layers.Conv2dLayer( + act='lrelu0.2x', shape=(5, 5, 3, 32), strides=(1, 2, 2, 1), padding='SAME', + b_init=tf.constant_initializer(value=0.0), name='conv2dlayer' + )(cls.input_layer) + except Exception as e: + cls.assertIsInstance(e, Exception) + print(e) + + +if __name__ == '__main__': + + tl.logging.set_verbosity(tl.logging.DEBUG) + + unittest.main() From d7b8e7d76348f6e5db65ca548c127ed7f610e7f8 Mon Sep 17 00:00:00 2001 From: RuihaiWu <37874862+warshallrho@users.noreply.github.com> Date: Wed, 29 May 2019 20:17:23 +0800 Subject: [PATCH 19/19] yapf --- tests/layers/test_layers_core_act.py | 37 +++++++++------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/tests/layers/test_layers_core_act.py b/tests/layers/test_layers_core_act.py index 74cfb6441..0da41fea0 100644 --- a/tests/layers/test_layers_core_act.py +++ b/tests/layers/test_layers_core_act.py @@ -19,7 +19,6 @@ class Layer_Convolution_2D_Test(CustomTestCase): def setUpClass(cls): print("##### begin testing activation #####") - @classmethod def tearDownClass(cls): pass @@ -36,41 +35,29 @@ def test_layer_core_act(cls): b_init=tf.constant_initializer(value=0.0), name='conv2dlayer' )(cls.input_layer) - cls.n2 = tl.layers.Conv2d( - n_filter=32, filter_size=(3, 3), strides=(2, 2), act="relu", name='conv2d' - )(cls.n1) + cls.n2 = tl.layers.Conv2d(n_filter=32, filter_size=(3, 3), strides=(2, 2), act="relu", name='conv2d')(cls.n1) - cls.n3 = tl.layers.Conv2d( - n_filter=32, filter_size=(3, 3), strides=(2, 2), act="leaky_relu", b_init=None - )(cls.n2) + cls.n3 = tl.layers.Conv2d(n_filter=32, filter_size=(3, 3), strides=(2, 2), act="leaky_relu", + b_init=None)(cls.n2) - cls.n4 = tl.layers.Conv2d( - n_filter=32, filter_size=(3, 3), strides=(2, 2), act="lrelu", b_init=None - )(cls.n2) + cls.n4 = tl.layers.Conv2d(n_filter=32, filter_size=(3, 3), strides=(2, 2), act="lrelu", b_init=None)(cls.n2) - cls.n5 = tl.layers.Conv2d( - n_filter=32, filter_size=(3, 3), strides=(2, 2), act="sigmoid", in_channels=32 - )(cls.n4) + cls.n5 = tl.layers.Conv2d(n_filter=32, filter_size=(3, 3), strides=(2, 2), act="sigmoid", + in_channels=32)(cls.n4) - cls.n6 = tl.layers.Conv2d( - n_filter=32, filter_size=(3, 3), strides=(2, 2), act="tanh", in_channels=32 - )(cls.n5) + cls.n6 = tl.layers.Conv2d(n_filter=32, filter_size=(3, 3), strides=(2, 2), act="tanh", in_channels=32)(cls.n5) cls.n7 = tl.layers.Conv2d( n_filter=32, filter_size=(3, 3), strides=(2, 2), act="leaky_relu0.22", in_channels=32 )(cls.n6) - cls.n8 = tl.layers.Conv2d( - n_filter=32, filter_size=(3, 3), strides=(2, 2), act="lrelu0.22", in_channels=32 - )(cls.n7) + cls.n8 = tl.layers.Conv2d(n_filter=32, filter_size=(3, 3), strides=(2, 2), act="lrelu0.22", + in_channels=32)(cls.n7) - cls.n9 = tl.layers.Conv2d( - n_filter=32, filter_size=(3, 3), strides=(2, 2), act="softplus", in_channels=32 - )(cls.n8) + cls.n9 = tl.layers.Conv2d(n_filter=32, filter_size=(3, 3), strides=(2, 2), act="softplus", + in_channels=32)(cls.n8) - cls.n10 = tl.layers.Conv2d( - n_filter=32, filter_size=(3, 3), strides=(2, 2), act="relu6", in_channels=32 - )(cls.n9) + cls.n10 = tl.layers.Conv2d(n_filter=32, filter_size=(3, 3), strides=(2, 2), act="relu6", in_channels=32)(cls.n9) cls.model = Model(cls.input_layer, cls.n8)