1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Core Keras layers.""" 16 17import copy 18import functools 19import operator 20import sys 21import textwrap 22import types as python_types 23import warnings 24 25import numpy as np 26 27from tensorflow.python.eager import backprop 28from tensorflow.python.eager import context 29from tensorflow.python.framework import constant_op 30from tensorflow.python.framework import dtypes 31from tensorflow.python.framework import ops 32from tensorflow.python.framework import sparse_tensor 33from tensorflow.python.framework import tensor_shape 34from tensorflow.python.keras import activations 35from tensorflow.python.keras import backend as K 36from tensorflow.python.keras import constraints 37from tensorflow.python.keras import initializers 38from tensorflow.python.keras import regularizers 39from tensorflow.python.keras.engine import keras_tensor 40from tensorflow.python.keras.engine.base_layer import Layer 41from tensorflow.python.keras.engine.input_spec import InputSpec 42from tensorflow.python.keras.utils import control_flow_util 43from tensorflow.python.keras.utils import conv_utils 44from tensorflow.python.keras.utils import generic_utils 45from tensorflow.python.keras.utils import tf_inspect 46from tensorflow.python.keras.utils import tf_utils 47from tensorflow.python.ops import array_ops 48from tensorflow.python.ops import embedding_ops 49from tensorflow.python.ops import gen_math_ops 50from tensorflow.python.ops import math_ops 51from tensorflow.python.ops import nn 52from tensorflow.python.ops import nn_ops 53from tensorflow.python.ops import sparse_ops 54from tensorflow.python.ops import standard_ops 55from tensorflow.python.ops import variable_scope 56from tensorflow.python.ops.ragged import ragged_getitem 57from tensorflow.python.ops.ragged import ragged_tensor 58from tensorflow.python.platform import tf_logging 59from tensorflow.python.trackable import base as trackable 60from tensorflow.python.util import dispatch 61from tensorflow.python.util import nest 62from tensorflow.python.util import tf_decorator 63from tensorflow.python.util.tf_export import get_canonical_name_for_symbol 64from tensorflow.python.util.tf_export import get_symbol_from_name 65from tensorflow.python.util.tf_export import keras_export 66 67 68# pylint: disable=g-classes-have-attributes 69@keras_export('keras.layers.Masking') 70class Masking(Layer): 71 """Masks a sequence by using a mask value to skip timesteps. 72 73 For each timestep in the input tensor (dimension #1 in the tensor), 74 if all values in the input tensor at that timestep 75 are equal to `mask_value`, then the timestep will be masked (skipped) 76 in all downstream layers (as long as they support masking). 77 78 If any downstream layer does not support masking yet receives such 79 an input mask, an exception will be raised. 80 81 Example: 82 83 Consider a Numpy data array `x` of shape `(samples, timesteps, features)`, 84 to be fed to an LSTM layer. You want to mask timestep #3 and #5 because you 85 lack data for these timesteps. You can: 86 87 - Set `x[:, 3, :] = 0.` and `x[:, 5, :] = 0.` 88 - Insert a `Masking` layer with `mask_value=0.` before the LSTM layer: 89 90 ```python 91 samples, timesteps, features = 32, 10, 8 92 inputs = np.random.random([samples, timesteps, features]).astype(np.float32) 93 inputs[:, 3, :] = 0. 94 inputs[:, 5, :] = 0. 95 96 model = tf.keras.models.Sequential() 97 model.add(tf.keras.layers.Masking(mask_value=0., 98 input_shape=(timesteps, features))) 99 model.add(tf.keras.layers.LSTM(32)) 100 101 output = model(inputs) 102 # The time step 3 and 5 will be skipped from LSTM calculation. 103 ``` 104 105 See [the masking and padding guide]( 106 https://www.tensorflow.org/guide/keras/masking_and_padding) 107 for more details. 108 """ 109 110 def __init__(self, mask_value=0., **kwargs): 111 super(Masking, self).__init__(**kwargs) 112 self.supports_masking = True 113 self.mask_value = mask_value 114 self._compute_output_and_mask_jointly = True 115 116 def compute_mask(self, inputs, mask=None): 117 return K.any(math_ops.not_equal(inputs, self.mask_value), axis=-1) 118 119 def call(self, inputs): 120 boolean_mask = K.any( 121 math_ops.not_equal(inputs, self.mask_value), axis=-1, keepdims=True) 122 outputs = inputs * math_ops.cast(boolean_mask, inputs.dtype) 123 # Compute the mask and outputs simultaneously. 124 outputs._keras_mask = array_ops.squeeze(boolean_mask, axis=-1) # pylint: disable=protected-access 125 return outputs 126 127 def compute_output_shape(self, input_shape): 128 return input_shape 129 130 def get_config(self): 131 config = {'mask_value': self.mask_value} 132 base_config = super(Masking, self).get_config() 133 return dict(list(base_config.items()) + list(config.items())) 134 135 136@keras_export('keras.layers.Dropout') 137class Dropout(Layer): 138 """Applies Dropout to the input. 139 140 The Dropout layer randomly sets input units to 0 with a frequency of `rate` 141 at each step during training time, which helps prevent overfitting. 142 Inputs not set to 0 are scaled up by 1/(1 - rate) such that the sum over 143 all inputs is unchanged. 144 145 Note that the Dropout layer only applies when `training` is set to True 146 such that no values are dropped during inference. When using `model.fit`, 147 `training` will be appropriately set to True automatically, and in other 148 contexts, you can set the kwarg explicitly to True when calling the layer. 149 150 (This is in contrast to setting `trainable=False` for a Dropout layer. 151 `trainable` does not affect the layer's behavior, as Dropout does 152 not have any variables/weights that can be frozen during training.) 153 154 >>> tf.random.set_seed(0) 155 >>> layer = tf.keras.layers.Dropout(.2, input_shape=(2,)) 156 >>> data = np.arange(10).reshape(5, 2).astype(np.float32) 157 >>> print(data) 158 [[0. 1.] 159 [2. 3.] 160 [4. 5.] 161 [6. 7.] 162 [8. 9.]] 163 >>> outputs = layer(data, training=True) 164 >>> print(outputs) 165 tf.Tensor( 166 [[ 0. 1.25] 167 [ 2.5 3.75] 168 [ 5. 6.25] 169 [ 7.5 8.75] 170 [10. 0. ]], shape=(5, 2), dtype=float32) 171 172 Args: 173 rate: Float between 0 and 1. Fraction of the input units to drop. 174 noise_shape: 1D integer tensor representing the shape of the 175 binary dropout mask that will be multiplied with the input. 176 For instance, if your inputs have shape 177 `(batch_size, timesteps, features)` and 178 you want the dropout mask to be the same for all timesteps, 179 you can use `noise_shape=(batch_size, 1, features)`. 180 seed: A Python integer to use as random seed. 181 182 Call arguments: 183 inputs: Input tensor (of any rank). 184 training: Python boolean indicating whether the layer should behave in 185 training mode (adding dropout) or in inference mode (doing nothing). 186 """ 187 188 def __init__(self, rate, noise_shape=None, seed=None, **kwargs): 189 super(Dropout, self).__init__(**kwargs) 190 if isinstance(rate, (int, float)) and not 0 <= rate <= 1: 191 raise ValueError(f'Invalid value {rate} received for ' 192 f'`rate`, expected a value between 0 and 1.') 193 self.rate = rate 194 self.noise_shape = noise_shape 195 self.seed = seed 196 self.supports_masking = True 197 198 def _get_noise_shape(self, inputs): 199 # Subclasses of `Dropout` may implement `_get_noise_shape(self, inputs)`, 200 # which will override `self.noise_shape`, and allows for custom noise 201 # shapes with dynamically sized inputs. 202 if self.noise_shape is None: 203 return None 204 205 concrete_inputs_shape = array_ops.shape(inputs) 206 noise_shape = [] 207 for i, value in enumerate(self.noise_shape): 208 noise_shape.append(concrete_inputs_shape[i] if value is None else value) 209 return ops.convert_to_tensor_v2_with_dispatch(noise_shape) 210 211 def call(self, inputs, training=None): 212 if training is None: 213 training = K.learning_phase() 214 215 def dropped_inputs(): 216 return nn.dropout( 217 inputs, 218 noise_shape=self._get_noise_shape(inputs), 219 seed=self.seed, 220 rate=self.rate) 221 222 output = control_flow_util.smart_cond(training, dropped_inputs, 223 lambda: array_ops.identity(inputs)) 224 return output 225 226 def compute_output_shape(self, input_shape): 227 return input_shape 228 229 def get_config(self): 230 config = { 231 'rate': self.rate, 232 'noise_shape': self.noise_shape, 233 'seed': self.seed 234 } 235 base_config = super(Dropout, self).get_config() 236 return dict(list(base_config.items()) + list(config.items())) 237 238 239@keras_export('keras.layers.SpatialDropout1D') 240class SpatialDropout1D(Dropout): 241 """Spatial 1D version of Dropout. 242 243 This version performs the same function as Dropout, however, it drops 244 entire 1D feature maps instead of individual elements. If adjacent frames 245 within feature maps are strongly correlated (as is normally the case in 246 early convolution layers) then regular dropout will not regularize the 247 activations and will otherwise just result in an effective learning rate 248 decrease. In this case, SpatialDropout1D will help promote independence 249 between feature maps and should be used instead. 250 251 Args: 252 rate: Float between 0 and 1. Fraction of the input units to drop. 253 254 Call arguments: 255 inputs: A 3D tensor. 256 training: Python boolean indicating whether the layer should behave in 257 training mode (adding dropout) or in inference mode (doing nothing). 258 259 Input shape: 260 3D tensor with shape: 261 `(samples, timesteps, channels)` 262 263 Output shape: 264 Same as input. 265 266 References: 267 - [Efficient Object Localization Using Convolutional 268 Networks](https://arxiv.org/abs/1411.4280) 269 """ 270 271 def __init__(self, rate, **kwargs): 272 super(SpatialDropout1D, self).__init__(rate, **kwargs) 273 self.input_spec = InputSpec(ndim=3) 274 275 def _get_noise_shape(self, inputs): 276 input_shape = array_ops.shape(inputs) 277 noise_shape = (input_shape[0], 1, input_shape[2]) 278 return noise_shape 279 280 281@keras_export('keras.layers.SpatialDropout2D') 282class SpatialDropout2D(Dropout): 283 """Spatial 2D version of Dropout. 284 285 This version performs the same function as Dropout, however, it drops 286 entire 2D feature maps instead of individual elements. If adjacent pixels 287 within feature maps are strongly correlated (as is normally the case in 288 early convolution layers) then regular dropout will not regularize the 289 activations and will otherwise just result in an effective learning rate 290 decrease. In this case, SpatialDropout2D will help promote independence 291 between feature maps and should be used instead. 292 293 Args: 294 rate: Float between 0 and 1. Fraction of the input units to drop. 295 data_format: 'channels_first' or 'channels_last'. 296 In 'channels_first' mode, the channels dimension 297 (the depth) is at index 1, 298 in 'channels_last' mode is it at index 3. 299 It defaults to the `image_data_format` value found in your 300 Keras config file at `~/.keras/keras.json`. 301 If you never set it, then it will be "channels_last". 302 303 Call arguments: 304 inputs: A 4D tensor. 305 training: Python boolean indicating whether the layer should behave in 306 training mode (adding dropout) or in inference mode (doing nothing). 307 308 Input shape: 309 4D tensor with shape: 310 `(samples, channels, rows, cols)` if data_format='channels_first' 311 or 4D tensor with shape: 312 `(samples, rows, cols, channels)` if data_format='channels_last'. 313 314 Output shape: 315 Same as input. 316 317 References: 318 - [Efficient Object Localization Using Convolutional 319 Networks](https://arxiv.org/abs/1411.4280) 320 """ 321 322 def __init__(self, rate, data_format=None, **kwargs): 323 super(SpatialDropout2D, self).__init__(rate, **kwargs) 324 if data_format is None: 325 data_format = K.image_data_format() 326 if data_format not in {'channels_last', 'channels_first'}: 327 raise ValueError('data_format must be in ' 328 '{"channels_last", "channels_first"}') 329 self.data_format = data_format 330 self.input_spec = InputSpec(ndim=4) 331 332 def _get_noise_shape(self, inputs): 333 input_shape = array_ops.shape(inputs) 334 if self.data_format == 'channels_first': 335 return (input_shape[0], input_shape[1], 1, 1) 336 elif self.data_format == 'channels_last': 337 return (input_shape[0], 1, 1, input_shape[3]) 338 339 340@keras_export('keras.layers.SpatialDropout3D') 341class SpatialDropout3D(Dropout): 342 """Spatial 3D version of Dropout. 343 344 This version performs the same function as Dropout, however, it drops 345 entire 3D feature maps instead of individual elements. If adjacent voxels 346 within feature maps are strongly correlated (as is normally the case in 347 early convolution layers) then regular dropout will not regularize the 348 activations and will otherwise just result in an effective learning rate 349 decrease. In this case, SpatialDropout3D will help promote independence 350 between feature maps and should be used instead. 351 352 Args: 353 rate: Float between 0 and 1. Fraction of the input units to drop. 354 data_format: 'channels_first' or 'channels_last'. 355 In 'channels_first' mode, the channels dimension (the depth) 356 is at index 1, in 'channels_last' mode is it at index 4. 357 It defaults to the `image_data_format` value found in your 358 Keras config file at `~/.keras/keras.json`. 359 If you never set it, then it will be "channels_last". 360 361 Call arguments: 362 inputs: A 5D tensor. 363 training: Python boolean indicating whether the layer should behave in 364 training mode (adding dropout) or in inference mode (doing nothing). 365 366 Input shape: 367 5D tensor with shape: 368 `(samples, channels, dim1, dim2, dim3)` if data_format='channels_first' 369 or 5D tensor with shape: 370 `(samples, dim1, dim2, dim3, channels)` if data_format='channels_last'. 371 372 Output shape: 373 Same as input. 374 375 References: 376 - [Efficient Object Localization Using Convolutional 377 Networks](https://arxiv.org/abs/1411.4280) 378 """ 379 380 def __init__(self, rate, data_format=None, **kwargs): 381 super(SpatialDropout3D, self).__init__(rate, **kwargs) 382 if data_format is None: 383 data_format = K.image_data_format() 384 if data_format not in {'channels_last', 'channels_first'}: 385 raise ValueError('data_format must be in ' 386 '{"channels_last", "channels_first"}') 387 self.data_format = data_format 388 self.input_spec = InputSpec(ndim=5) 389 390 def _get_noise_shape(self, inputs): 391 input_shape = array_ops.shape(inputs) 392 if self.data_format == 'channels_first': 393 return (input_shape[0], input_shape[1], 1, 1, 1) 394 elif self.data_format == 'channels_last': 395 return (input_shape[0], 1, 1, 1, input_shape[4]) 396 397 398@keras_export('keras.layers.Activation') 399class Activation(Layer): 400 """Applies an activation function to an output. 401 402 Args: 403 activation: Activation function, such as `tf.nn.relu`, or string name of 404 built-in activation function, such as "relu". 405 406 Usage: 407 408 >>> layer = tf.keras.layers.Activation('relu') 409 >>> output = layer([-3.0, -1.0, 0.0, 2.0]) 410 >>> list(output.numpy()) 411 [0.0, 0.0, 0.0, 2.0] 412 >>> layer = tf.keras.layers.Activation(tf.nn.relu) 413 >>> output = layer([-3.0, -1.0, 0.0, 2.0]) 414 >>> list(output.numpy()) 415 [0.0, 0.0, 0.0, 2.0] 416 417 Input shape: 418 Arbitrary. Use the keyword argument `input_shape` 419 (tuple of integers, does not include the batch axis) 420 when using this layer as the first layer in a model. 421 422 Output shape: 423 Same shape as input. 424 """ 425 426 def __init__(self, activation, **kwargs): 427 super(Activation, self).__init__(**kwargs) 428 self.supports_masking = True 429 self.activation = activations.get(activation) 430 431 def call(self, inputs): 432 return self.activation(inputs) 433 434 def compute_output_shape(self, input_shape): 435 return input_shape 436 437 def get_config(self): 438 config = {'activation': activations.serialize(self.activation)} 439 base_config = super(Activation, self).get_config() 440 return dict(list(base_config.items()) + list(config.items())) 441 442 443@keras_export('keras.layers.Reshape') 444class Reshape(Layer): 445 """Layer that reshapes inputs into the given shape. 446 447 Input shape: 448 Arbitrary, although all dimensions in the input shape must be known/fixed. 449 Use the keyword argument `input_shape` (tuple of integers, does not include 450 the samples/batch size axis) when using this layer as the first layer 451 in a model. 452 453 Output shape: 454 `(batch_size,) + target_shape` 455 456 Example: 457 458 >>> # as first layer in a Sequential model 459 >>> model = tf.keras.Sequential() 460 >>> model.add(tf.keras.layers.Reshape((3, 4), input_shape=(12,))) 461 >>> # model.output_shape == (None, 3, 4), `None` is the batch size. 462 >>> model.output_shape 463 (None, 3, 4) 464 465 >>> # as intermediate layer in a Sequential model 466 >>> model.add(tf.keras.layers.Reshape((6, 2))) 467 >>> model.output_shape 468 (None, 6, 2) 469 470 >>> # also supports shape inference using `-1` as dimension 471 >>> model.add(tf.keras.layers.Reshape((-1, 2, 2))) 472 >>> model.output_shape 473 (None, 3, 2, 2) 474 """ 475 476 def __init__(self, target_shape, **kwargs): 477 """Creates a `tf.keras.layers.Reshape` layer instance. 478 479 Args: 480 target_shape: Target shape. Tuple of integers, does not include the 481 samples dimension (batch size). 482 **kwargs: Any additional layer keyword arguments. 483 """ 484 super(Reshape, self).__init__(**kwargs) 485 self.target_shape = tuple(target_shape) 486 487 def _fix_unknown_dimension(self, input_shape, output_shape): 488 """Find and replace a missing dimension in an output shape. 489 490 This is a near direct port of the internal Numpy function 491 `_fix_unknown_dimension` in `numpy/core/src/multiarray/shape.c` 492 493 Args: 494 input_shape: Shape of array being reshaped 495 output_shape: Desired shape of the array with at most 496 a single -1 which indicates a dimension that should be 497 derived from the input shape. 498 499 Returns: 500 The new output shape with a -1 replaced with its computed value. 501 502 Raises: 503 ValueError: If the total array size of the output_shape is 504 different than the input_shape, or more than one unknown dimension 505 is specified. 506 """ 507 output_shape = list(output_shape) 508 msg = ('total size of new array must be unchanged, ' 509 'input_shape = {}, output_shape = {}' 510 .format(input_shape, output_shape)) 511 512 known, unknown = 1, None 513 for index, dim in enumerate(output_shape): 514 if dim < 0: 515 if unknown is None: 516 unknown = index 517 else: 518 raise ValueError('Can only specify one unknown dimension.') 519 else: 520 known *= dim 521 522 original = np.prod(input_shape, dtype=int) 523 if unknown is not None: 524 if known == 0 or original % known != 0: 525 raise ValueError(msg) 526 output_shape[unknown] = original // known 527 elif original != known: 528 raise ValueError(msg) 529 return output_shape 530 531 def compute_output_shape(self, input_shape): 532 input_shape = tensor_shape.TensorShape(input_shape).as_list() 533 if None in input_shape[1:]: 534 output_shape = [input_shape[0]] 535 # input shape (partially) unknown? replace -1's with None's 536 output_shape += tuple(s if s != -1 else None for s in self.target_shape) 537 else: 538 output_shape = [input_shape[0]] 539 output_shape += self._fix_unknown_dimension(input_shape[1:], 540 self.target_shape) 541 return tensor_shape.TensorShape(output_shape) 542 543 def call(self, inputs): 544 result = array_ops.reshape( 545 inputs, (array_ops.shape(inputs)[0],) + self.target_shape) 546 if not context.executing_eagerly(): 547 # Set the static shape for the result since it might lost during array_ops 548 # reshape, eg, some `None` dim in the result could be inferred. 549 result.set_shape(self.compute_output_shape(inputs.shape)) 550 return result 551 552 def get_config(self): 553 config = {'target_shape': self.target_shape} 554 base_config = super(Reshape, self).get_config() 555 return dict(list(base_config.items()) + list(config.items())) 556 557 558@keras_export('keras.layers.Permute') 559class Permute(Layer): 560 """Permutes the dimensions of the input according to a given pattern. 561 562 Useful e.g. connecting RNNs and convnets. 563 564 Example: 565 566 ```python 567 model = Sequential() 568 model.add(Permute((2, 1), input_shape=(10, 64))) 569 # now: model.output_shape == (None, 64, 10) 570 # note: `None` is the batch dimension 571 ``` 572 573 Args: 574 dims: Tuple of integers. Permutation pattern does not include the 575 samples dimension. Indexing starts at 1. 576 For instance, `(2, 1)` permutes the first and second dimensions 577 of the input. 578 579 Input shape: 580 Arbitrary. Use the keyword argument `input_shape` 581 (tuple of integers, does not include the samples axis) 582 when using this layer as the first layer in a model. 583 584 Output shape: 585 Same as the input shape, but with the dimensions re-ordered according 586 to the specified pattern. 587 """ 588 589 def __init__(self, dims, **kwargs): 590 super(Permute, self).__init__(**kwargs) 591 self.dims = tuple(dims) 592 if sorted(dims) != list(range(1, len(dims) + 1)): 593 raise ValueError( 594 'Invalid permutation `dims` for Permute Layer: %s. ' 595 'The set of indices in `dims` must be consecutive and start from 1.' % 596 (dims,)) 597 self.input_spec = InputSpec(ndim=len(self.dims) + 1) 598 599 def compute_output_shape(self, input_shape): 600 input_shape = tensor_shape.TensorShape(input_shape).as_list() 601 output_shape = copy.copy(input_shape) 602 for i, dim in enumerate(self.dims): 603 target_dim = input_shape[dim] 604 output_shape[i + 1] = target_dim 605 return tensor_shape.TensorShape(output_shape) 606 607 def call(self, inputs): 608 return array_ops.transpose(inputs, perm=(0,) + self.dims) 609 610 def get_config(self): 611 config = {'dims': self.dims} 612 base_config = super(Permute, self).get_config() 613 return dict(list(base_config.items()) + list(config.items())) 614 615 616@keras_export('keras.layers.Flatten') 617class Flatten(Layer): 618 """Flattens the input. Does not affect the batch size. 619 620 Note: If inputs are shaped `(batch,)` without a feature axis, then 621 flattening adds an extra channel dimension and output shape is `(batch, 1)`. 622 623 Args: 624 data_format: A string, 625 one of `channels_last` (default) or `channels_first`. 626 The ordering of the dimensions in the inputs. 627 `channels_last` corresponds to inputs with shape 628 `(batch, ..., channels)` while `channels_first` corresponds to 629 inputs with shape `(batch, channels, ...)`. 630 It defaults to the `image_data_format` value found in your 631 Keras config file at `~/.keras/keras.json`. 632 If you never set it, then it will be "channels_last". 633 634 Example: 635 636 >>> model = tf.keras.Sequential() 637 >>> model.add(tf.keras.layers.Conv2D(64, 3, 3, input_shape=(3, 32, 32))) 638 >>> model.output_shape 639 (None, 1, 10, 64) 640 641 >>> model.add(Flatten()) 642 >>> model.output_shape 643 (None, 640) 644 645 """ 646 647 def __init__(self, data_format=None, **kwargs): 648 super(Flatten, self).__init__(**kwargs) 649 self.data_format = conv_utils.normalize_data_format(data_format) 650 self.input_spec = InputSpec(min_ndim=1) 651 self._channels_first = self.data_format == 'channels_first' 652 653 def call(self, inputs): 654 if self._channels_first: 655 rank = inputs.shape.rank 656 if rank and rank > 1: 657 # Switch to channels-last format. 658 permutation = [0] 659 permutation.extend(range(2, rank)) 660 permutation.append(1) 661 inputs = array_ops.transpose(inputs, perm=permutation) 662 663 if context.executing_eagerly(): 664 # Full static shape is guaranteed to be available. 665 # Performance: Using `constant_op` is much faster than passing a list. 666 flattened_shape = constant_op.constant([inputs.shape[0], -1]) 667 return array_ops.reshape(inputs, flattened_shape) 668 else: 669 input_shape = inputs.shape 670 rank = input_shape.rank 671 if rank == 1: 672 return array_ops.expand_dims_v2(inputs, axis=1) 673 else: 674 batch_dim = tensor_shape.dimension_value(input_shape[0]) 675 non_batch_dims = input_shape[1:] 676 # Reshape in a way that preserves as much shape info as possible. 677 if non_batch_dims.is_fully_defined(): 678 last_dim = int(functools.reduce(operator.mul, non_batch_dims)) 679 flattened_shape = constant_op.constant([-1, last_dim]) 680 elif batch_dim is not None: 681 flattened_shape = constant_op.constant([int(batch_dim), -1]) 682 else: 683 flattened_shape = [array_ops.shape_v2(inputs)[0], -1] 684 return array_ops.reshape(inputs, flattened_shape) 685 686 def compute_output_shape(self, input_shape): 687 input_shape = tensor_shape.TensorShape(input_shape).as_list() 688 if not input_shape: 689 output_shape = tensor_shape.TensorShape([1]) 690 else: 691 output_shape = [input_shape[0]] 692 if np.all(input_shape[1:]): 693 output_shape += [np.prod(input_shape[1:], dtype=int)] 694 else: 695 output_shape += [None] 696 return tensor_shape.TensorShape(output_shape) 697 698 def get_config(self): 699 config = super(Flatten, self).get_config() 700 config.update({'data_format': self.data_format}) 701 return config 702 703 704@keras_export('keras.layers.RepeatVector') 705class RepeatVector(Layer): 706 """Repeats the input n times. 707 708 Example: 709 710 ```python 711 model = Sequential() 712 model.add(Dense(32, input_dim=32)) 713 # now: model.output_shape == (None, 32) 714 # note: `None` is the batch dimension 715 716 model.add(RepeatVector(3)) 717 # now: model.output_shape == (None, 3, 32) 718 ``` 719 720 Args: 721 n: Integer, repetition factor. 722 723 Input shape: 724 2D tensor of shape `(num_samples, features)`. 725 726 Output shape: 727 3D tensor of shape `(num_samples, n, features)`. 728 """ 729 730 def __init__(self, n, **kwargs): 731 super(RepeatVector, self).__init__(**kwargs) 732 self.n = n 733 if not isinstance(n, int): 734 raise TypeError(f'Expected an integer value for `n`, got {type(n)}.') 735 self.input_spec = InputSpec(ndim=2) 736 737 def compute_output_shape(self, input_shape): 738 input_shape = tensor_shape.TensorShape(input_shape).as_list() 739 return tensor_shape.TensorShape([input_shape[0], self.n, input_shape[1]]) 740 741 def call(self, inputs): 742 return K.repeat(inputs, self.n) 743 744 def get_config(self): 745 config = {'n': self.n} 746 base_config = super(RepeatVector, self).get_config() 747 return dict(list(base_config.items()) + list(config.items())) 748 749 750@keras_export('keras.layers.Lambda') 751class Lambda(Layer): 752 """Wraps arbitrary expressions as a `Layer` object. 753 754 The `Lambda` layer exists so that arbitrary expressions can be used 755 as a `Layer` when constructing `Sequential` 756 and Functional API models. `Lambda` layers are best suited for simple 757 operations or quick experimentation. For more advanced use cases, follow 758 [this guide](https://www.tensorflow.org/guide/keras/custom_layers_and_models) 759 for subclassing `tf.keras.layers.Layer`. 760 761 WARNING: `tf.keras.layers.Lambda` layers have (de)serialization limitations! 762 763 The main reason to subclass `tf.keras.layers.Layer` instead of using a 764 `Lambda` layer is saving and inspecting a Model. `Lambda` layers 765 are saved by serializing the Python bytecode, which is fundamentally 766 non-portable. They should only be loaded in the same environment where 767 they were saved. Subclassed layers can be saved in a more portable way 768 by overriding their `get_config` method. Models that rely on 769 subclassed Layers are also often easier to visualize and reason about. 770 771 Examples: 772 773 ```python 774 # add a x -> x^2 layer 775 model.add(Lambda(lambda x: x ** 2)) 776 ``` 777 ```python 778 # add a layer that returns the concatenation 779 # of the positive part of the input and 780 # the opposite of the negative part 781 782 def antirectifier(x): 783 x -= K.mean(x, axis=1, keepdims=True) 784 x = K.l2_normalize(x, axis=1) 785 pos = K.relu(x) 786 neg = K.relu(-x) 787 return K.concatenate([pos, neg], axis=1) 788 789 model.add(Lambda(antirectifier)) 790 ``` 791 792 Variables: 793 While it is possible to use Variables with Lambda layers, this practice is 794 discouraged as it can easily lead to bugs. For instance, consider the 795 following layer: 796 797 ```python 798 scale = tf.Variable(1.) 799 scale_layer = tf.keras.layers.Lambda(lambda x: x * scale) 800 ``` 801 802 Because scale_layer does not directly track the `scale` variable, it will 803 not appear in `scale_layer.trainable_weights` and will therefore not be 804 trained if `scale_layer` is used in a Model. 805 806 A better pattern is to write a subclassed Layer: 807 808 ```python 809 class ScaleLayer(tf.keras.layers.Layer): 810 def __init__(self): 811 super(ScaleLayer, self).__init__() 812 self.scale = tf.Variable(1.) 813 814 def call(self, inputs): 815 return inputs * self.scale 816 ``` 817 818 In general, Lambda layers can be convenient for simple stateless 819 computation, but anything more complex should use a subclass Layer instead. 820 821 Args: 822 function: The function to be evaluated. Takes input tensor as first 823 argument. 824 output_shape: Expected output shape from function. This argument can be 825 inferred if not explicitly provided. Can be a tuple or function. If a 826 tuple, it only specifies the first dimension onward; 827 sample dimension is assumed either the same as the input: `output_shape = 828 (input_shape[0], ) + output_shape` or, the input is `None` and 829 the sample dimension is also `None`: `output_shape = (None, ) + 830 output_shape` If a function, it specifies the entire shape as a function 831 of the 832 input shape: `output_shape = f(input_shape)` 833 mask: Either None (indicating no masking) or a callable with the same 834 signature as the `compute_mask` layer method, or a tensor that will be 835 returned as output mask regardless of what the input is. 836 arguments: Optional dictionary of keyword arguments to be passed to the 837 function. 838 839 Input shape: 840 Arbitrary. Use the keyword argument input_shape (tuple of 841 integers, does not include the samples axis) when using this layer as the 842 first layer in a model. 843 844 Output shape: 845 Specified by `output_shape` argument 846 """ 847 848 @trackable.no_automatic_dependency_tracking 849 def __init__(self, function, output_shape=None, mask=None, arguments=None, 850 **kwargs): 851 super(Lambda, self).__init__(**kwargs) 852 853 self.arguments = arguments or {} 854 self.function = function 855 856 if mask is not None: 857 self.supports_masking = True 858 self.mask = mask 859 self._output_shape = output_shape 860 861 # Warning on every invocation will be quite irksome in Eager mode. 862 self._already_warned = False 863 864 function_args = tf_inspect.getfullargspec(function).args 865 self._fn_expects_training_arg = 'training' in function_args 866 self._fn_expects_mask_arg = 'mask' in function_args 867 868 @tf_utils.shape_type_conversion 869 def compute_output_shape(self, input_shape): 870 if self._output_shape is None: 871 # Make use of existing autocomputation but provide Lambda-specific 872 # error message. This is always safe to run even when the outer context 873 # is Graph mode because Lambda layers don't have side effects such as 874 # `add_loss`. 875 with context.eager_mode(): 876 try: 877 return super(Lambda, self).compute_output_shape(input_shape) 878 except NotImplementedError: 879 raise NotImplementedError( 880 'We could not automatically infer the shape of the Lambda\'s ' 881 'output. Please specify `output_shape` for this Lambda.') 882 883 if callable(self._output_shape): 884 output_shapes = self._output_shape(input_shape) 885 return tf_utils.convert_shapes(output_shapes, to_tuples=False) 886 887 # Output shapes are passed directly and don't include batch dimension. 888 input_tensor_shape = tf_utils.convert_shapes(input_shape, to_tuples=False) 889 batch_size = nest.flatten(input_tensor_shape)[0][0] if input_shape else None 890 891 def _add_batch(shape): 892 return tensor_shape.TensorShape([batch_size] + shape.as_list()) 893 894 output_shapes = tf_utils.convert_shapes(self._output_shape, to_tuples=False) 895 return nest.map_structure(_add_batch, output_shapes) 896 897 def call(self, inputs, mask=None, training=None): 898 # We must copy for thread safety, but it only needs to be a shallow copy. 899 kwargs = {k: v for k, v in self.arguments.items()} 900 if self._fn_expects_mask_arg: 901 kwargs['mask'] = mask 902 if self._fn_expects_training_arg: 903 kwargs['training'] = training 904 905 created_variables = [] 906 def _variable_creator(next_creator, **kwargs): 907 var = next_creator(**kwargs) 908 created_variables.append(var) 909 return var 910 911 with backprop.GradientTape(watch_accessed_variables=True) as tape,\ 912 variable_scope.variable_creator_scope(_variable_creator): 913 result = self.function(inputs, **kwargs) 914 self._check_variables(created_variables, tape.watched_variables()) 915 return result 916 917 def _check_variables(self, created_variables, accessed_variables): 918 if not created_variables and not accessed_variables: 919 # In the common case that a Lambda layer does not touch a Variable, we 920 # don't want to incur the runtime cost of assembling any state used for 921 # checking only to immediately discard it. 922 return 923 924 tracked_weights = set(v.ref() for v in self.weights) 925 untracked_new_vars = [ 926 v for v in created_variables if v.ref() not in tracked_weights 927 ] 928 if untracked_new_vars: 929 variable_str = '\n'.join(' {}'.format(i) for i in untracked_new_vars) 930 error_str = textwrap.dedent( 931 ''' 932 The following Variables were created within a Lambda layer ({name}) 933 but are not tracked by said layer: 934 {variable_str} 935 The layer cannot safely ensure proper Variable reuse across multiple 936 calls, and consquently this behavior is disallowed for safety. Lambda 937 layers are not well suited to stateful computation; instead, writing a 938 subclassed Layer is the recommend way to define layers with 939 Variables.''' 940 ).format(name=self.name, variable_str=variable_str) 941 raise ValueError(error_str) 942 943 untracked_used_vars = [ 944 v for v in accessed_variables if v.ref() not in tracked_weights 945 ] 946 if untracked_used_vars and not self._already_warned: 947 variable_str = '\n'.join(' {}'.format(i) for i in untracked_used_vars) 948 self._warn(textwrap.dedent( 949 ''' 950 The following Variables were used a Lambda layer's call ({name}), but 951 are not present in its tracked objects: 952 {variable_str} 953 It is possible that this is intended behavior, but it is more likely 954 an omission. This is a strong indication that this layer should be 955 formulated as a subclassed Layer rather than a Lambda layer.''' 956 ).format(name=self.name, variable_str=variable_str)) 957 self._already_warned = True 958 959 def _warn(self, msg): 960 # This method will be overridden in a unit test to raise an error, because 961 # self.assertWarns is not universally implemented. 962 return tf_logging.warning(msg) 963 964 def compute_mask(self, inputs, mask=None): 965 if callable(self.mask): 966 return self.mask(inputs, mask) 967 return self.mask 968 969 def get_config(self): 970 function_config = self._serialize_function_to_config(self.function) 971 output_shape_config = self._serialize_function_to_config(self._output_shape, 972 allow_raw=True) 973 config = { 974 'function': function_config[0], 975 'function_type': function_config[1], 976 'module': function_config[2], 977 'output_shape': output_shape_config[0], 978 'output_shape_type': output_shape_config[1], 979 'output_shape_module': output_shape_config[2], 980 } 981 if self.mask is not None: 982 mask_config = self._serialize_function_to_config(self.mask) 983 config.update({ 984 'mask': mask_config[0], 985 'mask_type': mask_config[1], 986 'mask_module': mask_config[2] 987 }) 988 config['arguments'] = self.arguments 989 990 base_config = super(Lambda, self).get_config() 991 return dict(list(base_config.items()) + list(config.items())) 992 993 def _serialize_function_to_config(self, inputs, allow_raw=False): 994 if isinstance(inputs, python_types.LambdaType): 995 output = generic_utils.func_dump(inputs) 996 output_type = 'lambda' 997 module = inputs.__module__ 998 elif callable(inputs): 999 output = inputs.__name__ 1000 output_type = 'function' 1001 module = inputs.__module__ 1002 elif allow_raw: 1003 output = inputs 1004 output_type = 'raw' 1005 module = None 1006 else: 1007 raise ValueError( 1008 'Invalid input for serialization, type: %s ' % type(inputs)) 1009 1010 return output, output_type, module 1011 1012 @classmethod 1013 def from_config(cls, config, custom_objects=None): 1014 config = config.copy() 1015 function = cls._parse_function_from_config( 1016 config, custom_objects, 'function', 'module', 'function_type') 1017 1018 output_shape = cls._parse_function_from_config( 1019 config, custom_objects, 'output_shape', 'output_shape_module', 1020 'output_shape_type') 1021 if 'mask' in config: 1022 mask = cls._parse_function_from_config( 1023 config, custom_objects, 'mask', 'mask_module', 'mask_type') 1024 else: 1025 mask = None 1026 1027 config['function'] = function 1028 config['output_shape'] = output_shape 1029 config['mask'] = mask 1030 1031 # If arguments were numpy array, they have been saved as 1032 # list. We need to recover the ndarray 1033 if 'arguments' in config: 1034 for key in config['arguments']: 1035 if isinstance(config['arguments'][key], dict): 1036 arg_dict = config['arguments'][key] 1037 if 'type' in arg_dict and arg_dict['type'] == 'ndarray': 1038 # Overwrite the argument with its numpy translation 1039 config['arguments'][key] = np.array(arg_dict['value']) 1040 1041 return cls(**config) 1042 1043 @classmethod 1044 def _parse_function_from_config( 1045 cls, config, custom_objects, func_attr_name, module_attr_name, 1046 func_type_attr_name): 1047 globs = globals().copy() 1048 module = config.pop(module_attr_name, None) 1049 if module in sys.modules: 1050 globs.update(sys.modules[module].__dict__) 1051 elif module is not None: 1052 # Note: we don't know the name of the function if it's a lambda. 1053 warnings.warn('{} is not loaded, but a Lambda layer uses it. ' 1054 'It may cause errors.'.format(module) 1055 , UserWarning) 1056 if custom_objects: 1057 globs.update(custom_objects) 1058 function_type = config.pop(func_type_attr_name) 1059 if function_type == 'function': 1060 # Simple lookup in custom objects 1061 function = generic_utils.deserialize_keras_object( 1062 config[func_attr_name], 1063 custom_objects=custom_objects, 1064 printable_module_name='function in Lambda layer') 1065 elif function_type == 'lambda': 1066 # Unsafe deserialization from bytecode 1067 function = generic_utils.func_load( 1068 config[func_attr_name], globs=globs) 1069 elif function_type == 'raw': 1070 function = config[func_attr_name] 1071 else: 1072 raise TypeError('Unknown function type:', function_type) 1073 return function 1074 1075 1076@keras_export('keras.layers.Dense') 1077class Dense(Layer): 1078 """Just your regular densely-connected NN layer. 1079 1080 `Dense` implements the operation: 1081 `output = activation(dot(input, kernel) + bias)` 1082 where `activation` is the element-wise activation function 1083 passed as the `activation` argument, `kernel` is a weights matrix 1084 created by the layer, and `bias` is a bias vector created by the layer 1085 (only applicable if `use_bias` is `True`). These are all attributes of 1086 `Dense`. 1087 1088 Note: If the input to the layer has a rank greater than 2, then `Dense` 1089 computes the dot product between the `inputs` and the `kernel` along the 1090 last axis of the `inputs` and axis 0 of the `kernel` (using `tf.tensordot`). 1091 For example, if input has dimensions `(batch_size, d0, d1)`, 1092 then we create a `kernel` with shape `(d1, units)`, and the `kernel` operates 1093 along axis 2 of the `input`, on every sub-tensor of shape `(1, 1, d1)` 1094 (there are `batch_size * d0` such sub-tensors). 1095 The output in this case will have shape `(batch_size, d0, units)`. 1096 1097 Besides, layer attributes cannot be modified after the layer has been called 1098 once (except the `trainable` attribute). 1099 When a popular kwarg `input_shape` is passed, then keras will create 1100 an input layer to insert before the current layer. This can be treated 1101 equivalent to explicitly defining an `InputLayer`. 1102 1103 Example: 1104 1105 >>> # Create a `Sequential` model and add a Dense layer as the first layer. 1106 >>> model = tf.keras.models.Sequential() 1107 >>> model.add(tf.keras.Input(shape=(16,))) 1108 >>> model.add(tf.keras.layers.Dense(32, activation='relu')) 1109 >>> # Now the model will take as input arrays of shape (None, 16) 1110 >>> # and output arrays of shape (None, 32). 1111 >>> # Note that after the first layer, you don't need to specify 1112 >>> # the size of the input anymore: 1113 >>> model.add(tf.keras.layers.Dense(32)) 1114 >>> model.output_shape 1115 (None, 32) 1116 1117 Args: 1118 units: Positive integer, dimensionality of the output space. 1119 activation: Activation function to use. 1120 If you don't specify anything, no activation is applied 1121 (ie. "linear" activation: `a(x) = x`). 1122 use_bias: Boolean, whether the layer uses a bias vector. 1123 kernel_initializer: Initializer for the `kernel` weights matrix. 1124 bias_initializer: Initializer for the bias vector. 1125 kernel_regularizer: Regularizer function applied to 1126 the `kernel` weights matrix. 1127 bias_regularizer: Regularizer function applied to the bias vector. 1128 activity_regularizer: Regularizer function applied to 1129 the output of the layer (its "activation"). 1130 kernel_constraint: Constraint function applied to 1131 the `kernel` weights matrix. 1132 bias_constraint: Constraint function applied to the bias vector. 1133 1134 Input shape: 1135 N-D tensor with shape: `(batch_size, ..., input_dim)`. 1136 The most common situation would be 1137 a 2D input with shape `(batch_size, input_dim)`. 1138 1139 Output shape: 1140 N-D tensor with shape: `(batch_size, ..., units)`. 1141 For instance, for a 2D input with shape `(batch_size, input_dim)`, 1142 the output would have shape `(batch_size, units)`. 1143 """ 1144 1145 def __init__(self, 1146 units, 1147 activation=None, 1148 use_bias=True, 1149 kernel_initializer='glorot_uniform', 1150 bias_initializer='zeros', 1151 kernel_regularizer=None, 1152 bias_regularizer=None, 1153 activity_regularizer=None, 1154 kernel_constraint=None, 1155 bias_constraint=None, 1156 **kwargs): 1157 super(Dense, self).__init__( 1158 activity_regularizer=activity_regularizer, **kwargs) 1159 1160 self.units = int(units) if not isinstance(units, int) else units 1161 if self.units < 0: 1162 raise ValueError(f'Received an invalid value for `units`, expected ' 1163 f'a positive integer, got {units}.') 1164 self.activation = activations.get(activation) 1165 self.use_bias = use_bias 1166 self.kernel_initializer = initializers.get(kernel_initializer) 1167 self.bias_initializer = initializers.get(bias_initializer) 1168 self.kernel_regularizer = regularizers.get(kernel_regularizer) 1169 self.bias_regularizer = regularizers.get(bias_regularizer) 1170 self.kernel_constraint = constraints.get(kernel_constraint) 1171 self.bias_constraint = constraints.get(bias_constraint) 1172 1173 self.input_spec = InputSpec(min_ndim=2) 1174 self.supports_masking = True 1175 1176 def build(self, input_shape): 1177 dtype = dtypes.as_dtype(self.dtype or K.floatx()) 1178 if not (dtype.is_floating or dtype.is_complex): 1179 raise TypeError('Unable to build `Dense` layer with non-floating point ' 1180 'dtype %s' % (dtype,)) 1181 1182 input_shape = tensor_shape.TensorShape(input_shape) 1183 last_dim = tensor_shape.dimension_value(input_shape[-1]) 1184 if last_dim is None: 1185 raise ValueError('The last dimension of the inputs to `Dense` ' 1186 'should be defined. Found `None`.') 1187 self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim}) 1188 self.kernel = self.add_weight( 1189 'kernel', 1190 shape=[last_dim, self.units], 1191 initializer=self.kernel_initializer, 1192 regularizer=self.kernel_regularizer, 1193 constraint=self.kernel_constraint, 1194 dtype=self.dtype, 1195 trainable=True) 1196 if self.use_bias: 1197 self.bias = self.add_weight( 1198 'bias', 1199 shape=[self.units,], 1200 initializer=self.bias_initializer, 1201 regularizer=self.bias_regularizer, 1202 constraint=self.bias_constraint, 1203 dtype=self.dtype, 1204 trainable=True) 1205 else: 1206 self.bias = None 1207 self.built = True 1208 1209 def call(self, inputs): 1210 if inputs.dtype.base_dtype != self._compute_dtype_object.base_dtype: 1211 inputs = math_ops.cast(inputs, dtype=self._compute_dtype_object) 1212 1213 rank = inputs.shape.rank 1214 if rank == 2 or rank is None: 1215 # We use embedding_lookup_sparse as a more efficient matmul operation for 1216 # large sparse input tensors. The op will result in a sparse gradient, as 1217 # opposed to sparse_ops.sparse_tensor_dense_matmul which results in dense 1218 # gradients. This can lead to sigfinicant speedups, see b/171762937. 1219 if isinstance(inputs, sparse_tensor.SparseTensor): 1220 # We need to fill empty rows, as the op assumes at least one id per row. 1221 inputs, _ = sparse_ops.sparse_fill_empty_rows(inputs, 0) 1222 # We need to do some munging of our input to use the embedding lookup as 1223 # a matrix multiply. We split our input matrix into separate ids and 1224 # weights tensors. The values of the ids tensor should be the column 1225 # indices of our input matrix and the values of the weights tensor 1226 # can continue to the actual matrix weights. 1227 # The column arrangement of ids and weights 1228 # will be summed over and does not matter. See the documentation for 1229 # sparse_ops.sparse_tensor_dense_matmul a more detailed explanation 1230 # of the inputs to both ops. 1231 ids = sparse_tensor.SparseTensor( 1232 indices=inputs.indices, 1233 values=inputs.indices[:, 1], 1234 dense_shape=inputs.dense_shape) 1235 weights = inputs 1236 outputs = embedding_ops.embedding_lookup_sparse_v2( 1237 self.kernel, ids, weights, combiner='sum') 1238 else: 1239 outputs = gen_math_ops.MatMul(a=inputs, b=self.kernel) 1240 # Broadcast kernel to inputs. 1241 else: 1242 outputs = standard_ops.tensordot(inputs, self.kernel, [[rank - 1], [0]]) 1243 # Reshape the output back to the original ndim of the input. 1244 if not context.executing_eagerly(): 1245 shape = inputs.shape.as_list() 1246 output_shape = shape[:-1] + [self.kernel.shape[-1]] 1247 outputs.set_shape(output_shape) 1248 1249 if self.use_bias: 1250 outputs = nn_ops.bias_add(outputs, self.bias) 1251 1252 if self.activation is not None: 1253 outputs = self.activation(outputs) 1254 return outputs 1255 1256 def compute_output_shape(self, input_shape): 1257 input_shape = tensor_shape.TensorShape(input_shape) 1258 input_shape = input_shape.with_rank_at_least(2) 1259 if tensor_shape.dimension_value(input_shape[-1]) is None: 1260 raise ValueError( 1261 'The innermost dimension of input_shape must be defined, but saw: %s' 1262 % (input_shape,)) 1263 return input_shape[:-1].concatenate(self.units) 1264 1265 def get_config(self): 1266 config = super(Dense, self).get_config() 1267 config.update({ 1268 'units': self.units, 1269 'activation': activations.serialize(self.activation), 1270 'use_bias': self.use_bias, 1271 'kernel_initializer': initializers.serialize(self.kernel_initializer), 1272 'bias_initializer': initializers.serialize(self.bias_initializer), 1273 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), 1274 'bias_regularizer': regularizers.serialize(self.bias_regularizer), 1275 'activity_regularizer': 1276 regularizers.serialize(self.activity_regularizer), 1277 'kernel_constraint': constraints.serialize(self.kernel_constraint), 1278 'bias_constraint': constraints.serialize(self.bias_constraint) 1279 }) 1280 return config 1281 1282 1283@keras_export('keras.layers.ActivityRegularization') 1284class ActivityRegularization(Layer): 1285 """Layer that applies an update to the cost function based input activity. 1286 1287 Args: 1288 l1: L1 regularization factor (positive float). 1289 l2: L2 regularization factor (positive float). 1290 1291 Input shape: 1292 Arbitrary. Use the keyword argument `input_shape` 1293 (tuple of integers, does not include the samples axis) 1294 when using this layer as the first layer in a model. 1295 1296 Output shape: 1297 Same shape as input. 1298 """ 1299 1300 def __init__(self, l1=0., l2=0., **kwargs): 1301 super(ActivityRegularization, self).__init__( 1302 activity_regularizer=regularizers.L1L2(l1=l1, l2=l2), **kwargs) 1303 self.supports_masking = True 1304 self.l1 = l1 1305 self.l2 = l2 1306 1307 def compute_output_shape(self, input_shape): 1308 return input_shape 1309 1310 def get_config(self): 1311 config = {'l1': self.l1, 'l2': self.l2} 1312 base_config = super(ActivityRegularization, self).get_config() 1313 return dict(list(base_config.items()) + list(config.items())) 1314 1315 1316class TFOpLambda(Layer): 1317 """Wraps TF API symbols in a `Layer` object. 1318 1319 It is inserted by the Functional API construction whenever users call 1320 a supported TF symbol on KerasTensors. 1321 1322 Like Lambda layers, this layer tries to raise warnings when it detects users 1323 explicitly use variables in the call. (To let them know 1324 that the layer will not capture the variables). 1325 1326 This is useful in the case where users do something like: 1327 x = keras.Input(...) 1328 y = tf.Variable(...) 1329 out = x * tf_variable 1330 """ 1331 1332 @trackable.no_automatic_dependency_tracking 1333 def __init__(self, function, **kwargs): 1334 self.function = function 1335 self.symbol = ( 1336 get_canonical_name_for_symbol(self.function, 1337 add_prefix_to_v1_names=True) or 1338 get_canonical_name_for_symbol(self.function, 1339 api_name='keras', 1340 add_prefix_to_v1_names=True)) 1341 if 'name' not in kwargs: 1342 # Generate a name. 1343 # TFOpLambda layers avoid already-observed names, 1344 # because users cannot easily control the generated names. 1345 # Without this avoidance, users would be more likely to run 1346 # into unavoidable duplicate layer name collisions. 1347 # (For standard layers users could just set `name` when creating the 1348 # layer to work around a collision, but they can't do that for 1349 # auto-generated layers) 1350 if self.symbol: 1351 name = 'tf.' + self.symbol 1352 else: 1353 name = self.function.__name__ 1354 kwargs['name'] = K.unique_object_name( 1355 name, zero_based=True, avoid_observed_names=True) 1356 kwargs['autocast'] = False 1357 1358 # Decorate the function to produce this layer's call method 1359 def _call_wrapper(*args, **kwargs): 1360 return self._call_wrapper(*args, **kwargs) 1361 self.call = tf_decorator.make_decorator(function, _call_wrapper) 1362 1363 # Do not individually trace op layers in the SavedModel. 1364 self._must_restore_from_config = True 1365 1366 super(TFOpLambda, self).__init__(**kwargs) 1367 1368 # Preserve all argument data structures when saving/loading a config 1369 # (e.g., don't unnest lists that contain one element) 1370 self._preserve_input_structure_in_config = True 1371 1372 # Warning on every invocation will be quite irksome in Eager mode. 1373 self._already_warned = False 1374 1375 self._expects_training_arg = False 1376 self._expects_mask_arg = False 1377 1378 def _call_wrapper(self, *args, **kwargs): 1379 created_variables = [] 1380 def _variable_creator(next_creator, **creator_kwargs): 1381 var = next_creator(**creator_kwargs) 1382 created_variables.append(var) 1383 return var 1384 1385 with backprop.GradientTape(watch_accessed_variables=True) as tape, \ 1386 variable_scope.variable_creator_scope(_variable_creator): 1387 # We explicitly drop `name` arguments here, 1388 # to guard against the case where an op explicitly has a 1389 # `name` passed (which is susceptible to producing 1390 # multiple ops w/ the same name when the layer is reused) 1391 kwargs.pop('name', None) 1392 result = self.function(*args, **kwargs) 1393 self._check_variables(created_variables, tape.watched_variables()) 1394 return result 1395 1396 def _check_variables(self, created_variables, accessed_variables): 1397 if not created_variables and not accessed_variables: 1398 # In the common case that a Lambda layer does not touch a Variable, we 1399 # don't want to incur the runtime cost of assembling any state used for 1400 # checking only to immediately discard it. 1401 return 1402 1403 tracked_weights = set(v.ref() for v in self.weights) 1404 untracked_new_vars = [ 1405 v for v in created_variables if v.ref() not in tracked_weights 1406 ] 1407 if untracked_new_vars: 1408 variable_str = '\n'.join(' {}'.format(i) for i in untracked_new_vars) 1409 error_str = textwrap.dedent( 1410 ''' 1411 The following Variables were created within a Lambda layer ({name}) 1412 but are not tracked by said layer: 1413 {variable_str} 1414 The layer cannot safely ensure proper Variable reuse across multiple 1415 calls, and consquently this behavior is disallowed for safety. Lambda 1416 layers are not well suited to stateful computation; instead, writing a 1417 subclassed Layer is the recommend way to define layers with 1418 Variables.''' 1419 ).format(name=self.name, variable_str=variable_str) 1420 raise ValueError(error_str) 1421 1422 untracked_used_vars = [ 1423 v for v in accessed_variables if v.ref() not in tracked_weights 1424 ] 1425 if untracked_used_vars and not self._already_warned: 1426 variable_str = '\n'.join(' {}'.format(i) for i in untracked_used_vars) 1427 self._warn(textwrap.dedent( 1428 ''' 1429 The following Variables were used a Lambda layer's call ({name}), but 1430 are not present in its tracked objects: 1431 {variable_str} 1432 It is possible that this is intended behavior, but it is more likely 1433 an omission. This is a strong indication that this layer should be 1434 formulated as a subclassed Layer rather than a Lambda layer.''' 1435 ).format(name=self.name, variable_str=variable_str)) 1436 self._already_warned = True 1437 1438 def _warn(self, msg): 1439 # This method will be overridden in a unit test to raise an error, because 1440 # self.assertWarns is not universally implemented. 1441 return tf_logging.warning(msg) 1442 1443 def get_config(self): 1444 if not self.symbol: 1445 raise ValueError('This Keras op layer was generated from %s, a method ' 1446 'that is not an exposed in the TensorFlow API. This ' 1447 'may have happened if the method was explicitly ' 1448 'decorated to add dispatching support, and it was used ' 1449 'during Functional model construction. ' 1450 'To ensure cross-version compatibility of Keras models ' 1451 'that use op layers, only op layers produced from ' 1452 'exported TF API symbols can be serialized.' 1453 % self.function) 1454 config = { 1455 'function': self.symbol 1456 } 1457 1458 base_config = super(TFOpLambda, self).get_config() 1459 return dict(list(base_config.items()) + list(config.items())) 1460 1461 @classmethod 1462 def from_config(cls, config, custom_objects=None): 1463 config = config.copy() 1464 symbol_name = config['function'] 1465 function = get_symbol_from_name(symbol_name) 1466 if not function: 1467 raise ValueError( 1468 'TF symbol `tf.%s` could not be found.' % symbol_name) 1469 1470 config['function'] = function 1471 1472 return cls(**config) 1473 1474 1475class KerasOpDispatcher(dispatch.GlobalOpDispatcher): 1476 """A global dispatcher that allows building a functional model with TF Ops.""" 1477 1478 def handle(self, op, args, kwargs): 1479 """Handle the specified operation with the specified arguments.""" 1480 if any( 1481 isinstance(x, keras_tensor.KerasTensor) 1482 for x in nest.flatten([args, kwargs])): 1483 return TFOpLambda(op)(*args, **kwargs) 1484 else: 1485 return self.NOT_SUPPORTED 1486 1487KerasOpDispatcher().register() 1488 1489 1490def _slice_to_dict(x): 1491 if isinstance(x, slice): 1492 return {'start': x.start, 'stop': x.stop, 'step': x.step} 1493 return x 1494 1495 1496def _dict_to_slice(x): 1497 if isinstance(x, dict): 1498 return slice(x['start'], x['stop'], x['step']) 1499 return x 1500 1501 1502class SlicingOpLambda(TFOpLambda): 1503 """Wraps TF API symbols in a `Layer` object. 1504 1505 It is inserted by the Functional API construction whenever users call 1506 a supported TF symbol on KerasTensors. 1507 1508 Like Lambda layers, this layer tries to raise warnings when it detects users 1509 explicitly use variables in the call. (To let them know 1510 that the layer will not capture the variables). 1511 1512 This is useful in the case where users do something like: 1513 x = keras.Input(...) 1514 y = tf.Variable(...) 1515 out = x * tf_variable 1516 """ 1517 1518 @trackable.no_automatic_dependency_tracking 1519 def __init__(self, function, **kwargs): 1520 super(SlicingOpLambda, self).__init__(function, **kwargs) 1521 1522 original_call = self.call 1523 # Decorate the function to produce this layer's call method 1524 def _call_wrapper(*args, **kwargs): 1525 # Turn any slice dicts in the args back into `slice` objects. 1526 # This conversion cannot use nest.flatten/map_structure, 1527 # because dicts are flattened by nest while slices aren't. 1528 # So, map_structure would only see the individual elements in the 1529 # dict. 1530 # This can't use map_structure_up_to either because the 'shallowness' of 1531 # the shallow tree would have to vary depending on if only one dim or 1532 # multiple are being sliced. 1533 new_args = [] 1534 for arg in args: 1535 arg = _dict_to_slice(arg) 1536 if isinstance(arg, (list, tuple)): 1537 new_arg = [] 1538 for sub_arg in arg: 1539 new_arg.append(_dict_to_slice(sub_arg)) 1540 arg = new_arg 1541 new_args.append(arg) 1542 1543 # Handle the kwargs too. 1544 new_kwargs = {} 1545 for key, value in kwargs.items(): 1546 value = _dict_to_slice(value) 1547 if isinstance(value, (list, tuple)): 1548 new_value = [] 1549 for v in value: 1550 new_value.append(_dict_to_slice(v)) 1551 value = new_value 1552 new_kwargs[key] = value 1553 1554 return original_call(*new_args, **new_kwargs) 1555 self.call = tf_decorator.make_decorator(original_call, _call_wrapper) 1556 1557 1558class TFSlicingOpDispatcher(dispatch.OpDispatcher): 1559 """A global dispatcher that allows building a functional model with TF Ops.""" 1560 1561 def __init__(self, op): 1562 self.op = op 1563 1564 def handle(self, args, kwargs): 1565 """Handle the specified operation with the specified arguments.""" 1566 args = nest.map_structure(_slice_to_dict, args) 1567 kwargs = nest.map_structure(_slice_to_dict, kwargs) 1568 if any( 1569 isinstance(x, keras_tensor.KerasTensor) 1570 for x in nest.flatten([args, kwargs])): 1571 return SlicingOpLambda(self.op)(*args, **kwargs) 1572 else: 1573 return self.NOT_SUPPORTED 1574 1575for slicing_op in [ 1576 array_ops._slice_helper, # pylint: disable=protected-access 1577 array_ops.boolean_mask, 1578 array_ops.boolean_mask_v2, 1579 ragged_getitem.ragged_tensor_getitem 1580]: 1581 TFSlicingOpDispatcher(slicing_op).register(slicing_op) 1582 1583 1584class InstanceProperty(Layer): 1585 """Wraps an instance property access (e.g. `x.foo`) in a Keras Layer. 1586 1587 This layer takes an attribute name `attr_name` in the constructor and, 1588 when called on input tensor `obj` returns `obj.attr_name`. 1589 1590 KerasTensors specialized for specific extension types use it to 1591 represent instance property accesses on the represented object in the 1592 case where the property needs to be dynamically accessed as opposed to 1593 being statically computed from the typespec, e.g. 1594 1595 x = keras.Input(..., ragged=True) 1596 out = x.flat_values 1597 """ 1598 1599 @trackable.no_automatic_dependency_tracking 1600 def __init__(self, attr_name, **kwargs): 1601 self.attr_name = attr_name 1602 1603 if 'name' not in kwargs: 1604 kwargs['name'] = K.unique_object_name( 1605 'input.' + self.attr_name, zero_based=True, avoid_observed_names=True) 1606 kwargs['autocast'] = False 1607 1608 # Do not individually trace op layers in the SavedModel. 1609 self._must_restore_from_config = True 1610 1611 super(InstanceProperty, self).__init__(**kwargs) 1612 1613 # Preserve all argument data structures when saving/loading a config 1614 # (e.g., don't unnest lists that contain one element) 1615 self._preserve_input_structure_in_config = True 1616 1617 def call(self, obj): 1618 return getattr(obj, self.attr_name) 1619 1620 def get_config(self): 1621 config = { 1622 'attr_name': self.attr_name 1623 } 1624 base_config = super(InstanceProperty, self).get_config() 1625 return dict(list(base_config.items()) + list(config.items())) 1626 1627 @classmethod 1628 def from_config(cls, config, custom_objects=None): 1629 return cls(**config) 1630 1631 1632class InstanceMethod(InstanceProperty): 1633 """Wraps an instance method access (e.g. `x.foo(arg)` in a Keras Layer. 1634 1635 This layer takes an attribute name `attr_name` in the constructor and, 1636 when called on input tensor `obj` with additional arguments `args` and 1637 `kwargs` returns `obj.attr_name(*args, **kwargs)`. 1638 1639 KerasTensors specialized for specific extension types use it to 1640 represent dynamic instance method calls on the represented object, e.g. 1641 1642 x = keras.Input(..., ragged=True) 1643 new_values = keras.Input(...) 1644 out = x.with_values(new_values) 1645 """ 1646 1647 def call(self, obj, args, kwargs): 1648 method = getattr(obj, self.attr_name) 1649 return method(*args, **kwargs) 1650 1651 1652def _delegate_property(keras_tensor_cls, property_name): # pylint: disable=invalid-name 1653 """Register property on a KerasTensor class. 1654 1655 Calling this multiple times with the same arguments should be a no-op. 1656 1657 This method exposes a property on the KerasTensor class that will use an 1658 `InstanceProperty` layer to access the property on the represented 1659 intermediate values in the model. 1660 1661 Args: 1662 keras_tensor_cls: The KerasTensor subclass that should expose the property. 1663 property_name: The name of the property to expose and delegate to the 1664 represented (Composite)Tensor. 1665 """ 1666 # We use a lambda because we can't create a Keras layer at import time 1667 # due to dynamic layer class versioning. 1668 property_access = property(lambda self: InstanceProperty(property_name)(self)) # pylint: disable=unnecessary-lambda 1669 setattr(keras_tensor_cls, property_name, property_access) 1670 1671 1672def _delegate_method(keras_tensor_cls, method_name): # pylint: disable=invalid-name 1673 """Register method on a KerasTensor class. 1674 1675 Calling this function times with the same arguments should be a no-op. 1676 1677 This method exposes an instance method on the KerasTensor class that will use 1678 an `InstanceMethod` layer to run the desired method on the represented 1679 intermediate values in the model. 1680 1681 Args: 1682 keras_tensor_cls: The KerasTensor subclass that should expose the property. 1683 method_name: The name of the method to expose and delegate to the 1684 represented (Composite)Tensor. 1685 """ 1686 def delegate(self, *args, **kwargs): 1687 return InstanceMethod(method_name)(self, args, kwargs) 1688 setattr(keras_tensor_cls, method_name, delegate) 1689 1690# We do not support the `uniform_row_length` property because it 1691# returns either `None` or an int tensor, and code that relies on it tends 1692# to check `is None` directly. Delegating it here would always return a 1693# `KerasTensor`, regardless of what can be statically inferred. This would 1694# never equal `None`, breaking code that expects it to be partially-static 1695# in unpredictable ways. 1696for ragged_property in [ 1697 'values', 1698 'flat_values', 1699 'row_splits', 1700 'nested_row_splits' 1701]: 1702 _delegate_property(keras_tensor.RaggedKerasTensor, ragged_property) 1703 1704for ragged_method_name in [ 1705 'value_rowids', 1706 'nested_value_rowids', 1707 'nrows', 1708 'row_starts', 1709 'row_limits', 1710 'row_lengths', 1711 'nested_row_lengths', 1712 'bounding_shape', 1713 'with_values', 1714 'with_flat_values', 1715 'with_row_splits_dtype', 1716 'merge_dims', 1717 'to_tensor', 1718 'to_sparse', 1719]: 1720 _delegate_method(keras_tensor.RaggedKerasTensor, ragged_method_name) 1721 1722for sparse_property in [ 1723 'indices', 1724 'values', 1725]: 1726 _delegate_property(keras_tensor.SparseKerasTensor, sparse_property) 1727 1728for sparse_method in [ 1729 'with_values', 1730]: 1731 _delegate_method(keras_tensor.SparseKerasTensor, sparse_method) 1732 1733 1734class ClassMethod(Layer): 1735 """Wraps a TF API Class's class method in a `Layer` object. 1736 1737 It is inserted by the Functional API construction whenever users call 1738 a supported TF Class's class method on KerasTensors. 1739 1740 This is useful in the case where users do something like: 1741 x = keras.Input(...) 1742 y = keras.Input(...) 1743 out = tf.RaggedTensor.from_row_splits(x, y) 1744 """ 1745 1746 @trackable.no_automatic_dependency_tracking 1747 def __init__(self, cls_ref, method_name, **kwargs): 1748 self.cls_ref = cls_ref 1749 self.method_name = method_name 1750 self.cls_symbol = ( 1751 get_canonical_name_for_symbol(self.cls_ref, 1752 add_prefix_to_v1_names=True) or 1753 get_canonical_name_for_symbol(self.cls_ref, 1754 api_name='keras', 1755 add_prefix_to_v1_names=True)) 1756 if 'name' not in kwargs: 1757 kwargs['name'] = K.unique_object_name( 1758 'tf.' + self.cls_symbol + '.' + self.method_name, zero_based=True, 1759 avoid_observed_names=True) 1760 kwargs['autocast'] = False 1761 1762 # Do not individually trace op layers in the SavedModel. 1763 self._must_restore_from_config = True 1764 1765 super(ClassMethod, self).__init__(**kwargs) 1766 1767 # Preserve all argument data structures when saving/loading a config 1768 # (e.g., don't unnest lists that contain one element) 1769 self._preserve_input_structure_in_config = True 1770 1771 self._expects_training_arg = False 1772 self._expects_mask_arg = False 1773 1774 def call(self, args, kwargs): 1775 return getattr(self.cls_ref, self.method_name)(*args, **kwargs) 1776 1777 def get_config(self): 1778 if not self.cls_symbol: 1779 raise ValueError('This Keras class method conversion tried to convert ' 1780 'a method belonging to class %s, a class ' 1781 'that is not an exposed in the TensorFlow API. ' 1782 'To ensure cross-version compatibility of Keras models ' 1783 'that use op layers, only op layers produced from ' 1784 'exported TF API symbols can be serialized.' 1785 % self.cls_symbol) 1786 config = { 1787 'cls_symbol': self.cls_symbol, 1788 'method_name': self.method_name 1789 } 1790 1791 base_config = super(ClassMethod, self).get_config() 1792 return dict(list(base_config.items()) + list(config.items())) 1793 1794 @classmethod 1795 def from_config(cls, config, custom_objects=None): 1796 config = config.copy() 1797 symbol_name = config.pop('cls_symbol') 1798 cls_ref = get_symbol_from_name(symbol_name) 1799 if not cls_ref: 1800 raise ValueError( 1801 'TF symbol `tf.%s` could not be found.' % symbol_name) 1802 1803 config['cls_ref'] = cls_ref 1804 1805 return cls(**config) 1806 1807 1808class TFClassMethodDispatcher(dispatch.OpDispatcher): 1809 """A class method dispatcher that allows building a functional model with TF class methods.""" 1810 1811 def __init__(self, cls, method_name): 1812 self.cls = cls 1813 self.method_name = method_name 1814 1815 def handle(self, args, kwargs): 1816 """Handle the specified operation with the specified arguments.""" 1817 if any( 1818 isinstance(x, keras_tensor.KerasTensor) 1819 for x in nest.flatten([args, kwargs])): 1820 return ClassMethod(self.cls, self.method_name)(args[1:], kwargs) 1821 else: 1822 return self.NOT_SUPPORTED 1823 1824for ragged_class_method in [ 1825 'from_value_rowids', 1826 'from_row_splits', 1827 'from_row_lengths', 1828 'from_row_starts', 1829 'from_row_limits', 1830 'from_uniform_row_length', 1831 'from_nested_value_rowids', 1832 'from_nested_row_splits', 1833 'from_nested_row_lengths', 1834 'from_tensor', 1835 'from_sparse', 1836]: 1837 TFClassMethodDispatcher( 1838 ragged_tensor.RaggedTensor, ragged_class_method).register( 1839 getattr(ragged_tensor.RaggedTensor, ragged_class_method)) 1840