Skip to content

lipdp.layers module

AddBias

Bases: tf.keras.layers.Layer

Adds a bias to the input.

Remark: the euclidean norm of the bias must be bounded in advance. Note that this is the euclidean norm of the whole bias vector, not the norm of each element of the bias vector.

Warning: beware zero gradients outside the ball of norm norm_max. In the future, we might choose a smoother projection on the ball to ensure that the gradient remains non zero outside the ball.

Source code in lipdp/layers.py
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
class AddBias(tf.keras.layers.Layer):
    """Adds a bias to the input.

    Remark: the euclidean norm of the bias must be bounded in advance.
    Note that this is the euclidean norm of the whole bias vector, not
    the norm of each element of the bias vector.

    Warning: beware zero gradients outside the ball of norm norm_max.
    In the future, we might choose a smoother projection on the ball to ensure
    that the gradient remains non zero outside the ball.
    """

    def __init__(self, norm_max, **kwargs):
        super().__init__(**kwargs)
        self.norm_max = norm_max

    def build(self, input_shape):
        self.bias = self.add_weight(
            name="bias",
            shape=(input_shape[-1],),
            initializer="zeros",
            trainable=True,
        )

    def call(self, inputs, **kwargs):
        # parametrize the bias so it belongs to a ball of norm norm_max.
        bias = tf.clip_by_norm(self.bias, self.norm_max)  # 1-Lipschitz operation.
        return inputs + bias

DPLayer

Wrapper for created differentially private layers, instanciates abstract methods use for computing the bounds of the gradient relatively to the parameters and to the input.

Source code in lipdp/layers.py
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
class DPLayer:
    """
    Wrapper for created differentially private layers, instanciates abstract methods
    use for computing the bounds of the gradient relatively to the parameters and to the
    input.
    """

    @abstractmethod
    def backpropagate_params(self, input_bound, gradient_bound):
        """Corresponds to the Lipschitz constant of the output wrt the parameters,
            i.e. the norm of the Jacobian of the output wrt the parameters times the norm of the cotangeant vector.

        Args:
            input_bound: Maximum norm of input.
            gradient_bound: Maximum norm of gradients (co-tangent vector)

        Returns:
            Maximum norm of tangent vector."""
        pass

    @abstractmethod
    def backpropagate_inputs(self, input_bound, gradient_bound):
        """Applies the dilatation of the cotangeant vector norm (upstream gradient) by the Jacobian,
            i.e. multiply by the Lipschitz constant of the output wrt input.

        Args:
            input_bound: Maximum norm of input.
            gradient_bound: Maximum norm of gradients (co-tangent vector)

        Returns:
            Maximum norm of tangent vector.
        """
        pass

    @abstractmethod
    def propagate_inputs(self, input_bound):
        """Maximum norm of output of element.

        Remark: when the layer is linear, this coincides with its Lipschitz constant * input_bound.
        """
        pass

    @abstractmethod
    def has_parameters(self):
        pass

backpropagate_inputs(input_bound, gradient_bound) abstractmethod

Applies the dilatation of the cotangeant vector norm (upstream gradient) by the Jacobian, i.e. multiply by the Lipschitz constant of the output wrt input.

Parameters:

Name Type Description Default
input_bound

Maximum norm of input.

required
gradient_bound

Maximum norm of gradients (co-tangent vector)

required

Returns:

Type Description

Maximum norm of tangent vector.

Source code in lipdp/layers.py
53
54
55
56
57
58
59
60
61
62
63
64
65
@abstractmethod
def backpropagate_inputs(self, input_bound, gradient_bound):
    """Applies the dilatation of the cotangeant vector norm (upstream gradient) by the Jacobian,
        i.e. multiply by the Lipschitz constant of the output wrt input.

    Args:
        input_bound: Maximum norm of input.
        gradient_bound: Maximum norm of gradients (co-tangent vector)

    Returns:
        Maximum norm of tangent vector.
    """
    pass

backpropagate_params(input_bound, gradient_bound) abstractmethod

Corresponds to the Lipschitz constant of the output wrt the parameters, i.e. the norm of the Jacobian of the output wrt the parameters times the norm of the cotangeant vector.

Parameters:

Name Type Description Default
input_bound

Maximum norm of input.

required
gradient_bound

Maximum norm of gradients (co-tangent vector)

required

Returns:

Type Description

Maximum norm of tangent vector.

Source code in lipdp/layers.py
40
41
42
43
44
45
46
47
48
49
50
51
@abstractmethod
def backpropagate_params(self, input_bound, gradient_bound):
    """Corresponds to the Lipschitz constant of the output wrt the parameters,
        i.e. the norm of the Jacobian of the output wrt the parameters times the norm of the cotangeant vector.

    Args:
        input_bound: Maximum norm of input.
        gradient_bound: Maximum norm of gradients (co-tangent vector)

    Returns:
        Maximum norm of tangent vector."""
    pass

propagate_inputs(input_bound) abstractmethod

Maximum norm of output of element.

Remark: when the layer is linear, this coincides with its Lipschitz constant * input_bound.

Source code in lipdp/layers.py
67
68
69
70
71
72
73
@abstractmethod
def propagate_inputs(self, input_bound):
    """Maximum norm of output of element.

    Remark: when the layer is linear, this coincides with its Lipschitz constant * input_bound.
    """
    pass

DP_AddBias

Bases: AddBias, DPLayer

Adds a bias to the input.

The bias is projected on the ball of norm norm_max during training. The projection on the ball is a 1-Lipschitz function, since the ball is convex.

Source code in lipdp/layers.py
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
class DP_AddBias(AddBias, DPLayer):
    """Adds a bias to the input.

    The bias is projected on the ball of norm `norm_max` during training.
    The projection on the ball is a 1-Lipschitz function, since the ball
    is convex.
    """

    def __init__(self, *args, nm_coef=1, **kwargs):
        super().__init__(*args, **kwargs)
        self.nm_coef = nm_coef

    def backpropagate_params(self, input_bound, gradient_bound):
        return gradient_bound  # clipping is a 1-Lipschitz operation.

    def backpropagate_inputs(self, input_bound, gradient_bound):
        return 1 * gradient_bound  # adding is a 1-Lipschitz operation.

    def propagate_inputs(self, input_bound):
        return input_bound + self.norm_max

    def has_parameters(self):
        return True

DP_BoundedInput

Bases: tf.keras.layers.Layer, DPLayer

Input layer that clips the input to a given norm.

Remark: every pipeline should start with this layer.

Attributes:

Name Type Description
upper_bound

Maximum norm of the input.

enforce_clipping

If True (default), the input is clipped to the given norm.

Source code in lipdp/layers.py
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
class DP_BoundedInput(tf.keras.layers.Layer, DPLayer):
    """Input layer that clips the input to a given norm.

    Remark: every pipeline should start with this layer.

    Attributes:
        upper_bound: Maximum norm of the input.
        enforce_clipping: If True (default), the input is clipped to the given norm.
    """

    def __init__(self, *args, upper_bound, enforce_clipping=True, **kwargs):
        super().__init__(*args, **kwargs)
        self.upper_bound = upper_bound
        self.enforce_clipping = enforce_clipping

    def call(self, x, *args, **kwargs):
        if self.enforce_clipping:
            axes = list(range(1, len(x.shape)))
            x = tf.clip_by_norm(x, self.upper_bound, axes=axes)
        return x

    def backpropagate_params(self, input_bound, gradient_bound):
        raise ValueError("InputLayer doesn't have parameters")

    def backpropagate_inputs(self, input_bound, gradient_bound):
        return 1 * gradient_bound

    def propagate_inputs(self, input_bound):
        if input_bound is None:
            return self.upper_bound
        return min(self.upper_bound, input_bound)

    def has_parameters(self):
        return False

DP_ClipGradient

Bases: tf.keras.layers.Layer, DPLayer

Clips the gradient during the backward pass.

Behave like identity function during the forward pass. The clipping is done automatically during the backward pass.

Attributes:

Name Type Description
clip_value float

The maximum norm of the gradient.

Source code in lipdp/layers.py
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
class DP_ClipGradient(tf.keras.layers.Layer, DPLayer):
    """Clips the gradient during the backward pass.

    Behave like identity function during the forward pass.
    The clipping is done automatically during the backward pass.

    Attributes:
        clip_value (float): The maximum norm of the gradient.
    """

    def __init__(self, clip_value, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.clip_value = clip_value

    def call(self, inputs, *args, **kwargs):
        batch_size = tf.cast(tf.shape(inputs)[0], tf.float32)
        # the clipping is done elementwise
        # since REDUCTION=SUM_OVER_BATCH_SIZE, we need to divide by batch_size
        # to get the correct norm.
        # this makes the clipping independent of the batch size.
        elementwise_clip_value = self.clip_value / batch_size
        return clip_gradient(inputs, elementwise_clip_value)

    def backpropagate_params(self, input_bound, gradient_bound):
        raise ValueError("ClipGradient doesn't have parameters")

    def backpropagate_inputs(self, input_bound, gradient_bound):
        return min(gradient_bound, self.clip_value)

    def propagate_inputs(self, input_bound):
        return input_bound

    def has_parameters(self):
        return False

DP_MaxPool2D

Bases: tf.keras.layers.MaxPool2D, DPLayer

Max pooling layer that preserves the gradient norm.

Parameters:

Name Type Description Default
layer_cls

Class of the layer to wrap.

required

Returns:

Type Description

A differentially private layer that doesn't have parameters.

Source code in lipdp/layers.py
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
class DP_MaxPool2D(tf.keras.layers.MaxPool2D, DPLayer):
    """Max pooling layer that preserves the gradient norm.

    Args:
        layer_cls: Class of the layer to wrap.

    Returns:
        A differentially private layer that doesn't have parameters.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        assert (
            self.strides is None or self.strides == self.pool_size
        ), "Ensure that strides == pool_size, otherwise it is not 1-Lipschitz."

    def backpropagate_params(self, input_bound, gradient_bound):
        raise ValueError("Layer doesn't have parameters")

    def backpropagate_inputs(self, input_bound, gradient_bound):
        return 1 * gradient_bound

    def propagate_inputs(self, input_bound):
        return input_bound

    def has_parameters(self):
        return False

DP_ScaledL2NormPooling2D

Bases: lip.layers.ScaledL2NormPooling2D, DPLayer

Max pooling layer that preserves the gradient norm.

Parameters:

Name Type Description Default
layer_cls

Class of the layer to wrap.

required

Returns:

Type Description

A differentially private layer that doesn't have parameters.

Source code in lipdp/layers.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
class DP_ScaledL2NormPooling2D(lip.layers.ScaledL2NormPooling2D, DPLayer):
    """Max pooling layer that preserves the gradient norm.

    Args:
        layer_cls: Class of the layer to wrap.

    Returns:
        A differentially private layer that doesn't have parameters.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        assert (
            self.strides is None or self.strides == self.pool_size
        ), "Ensure that strides == pool_size, otherwise it is not 1-Lipschitz."

    def backpropagate_params(self, input_bound, gradient_bound):
        raise ValueError("Layer doesn't have parameters")

    def backpropagate_inputs(self, input_bound, gradient_bound):
        return 1 * gradient_bound

    def propagate_inputs(self, input_bound):
        return input_bound

    def has_parameters(self):
        return False

DP_WrappedResidual

Bases: tf.keras.layers.Layer, DPLayer

Source code in lipdp/layers.py
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
class DP_WrappedResidual(tf.keras.layers.Layer, DPLayer):
    def __init__(self, block, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.block = block

    def call(self, inputs, *args, **kwargs):
        assert len(inputs) == 2
        i1, i2 = inputs
        i2 = self.block(i2, *args, **kwargs)
        return i1, i2

    def backpropagate_params(self, input_bound, gradient_bound):
        assert len(input_bound) == 2
        assert len(gradient_bound) == 2
        _, i2 = input_bound
        _, g2 = gradient_bound
        g2 = self.block.backpropagate_params(i2, g2)
        return g2

    def backpropagate_inputs(self, input_bound, gradient_bound):
        assert len(input_bound) == 2
        assert len(gradient_bound) == 2
        _, i2 = input_bound
        g1, g2 = gradient_bound
        g2 = self.block.backpropagate_inputs(i2, g2)
        return g1, g2

    def propagate_inputs(self, input_bound):
        assert len(input_bound) == 2
        i1, i2 = input_bound
        i2 = self.block.propagate_inputs(i2)
        return i1, i2

    def has_parameters(self):
        return self.block.has_parameters()

    @property
    def nm_coef(self):
        """Returns the norm multiplier coefficient of the layer.

        Remark: this is a property to mimic the behavior of an attribute.
        """
        return self.block.nm_coef

nm_coef property

Returns the norm multiplier coefficient of the layer.

Remark: this is a property to mimic the behavior of an attribute.

DP_GNP_Factory(layer_cls)

Factory for creating differentially private gradient norm preserving layers that don't have parameters.

Remark: the layer is assumed to be GNP. This means that the gradient norm is preserved by the layer (i.e its Jacobian norm is 1). Pllease ensure that the layer is GNP before using this factory.

Parameters:

Name Type Description Default
layer_cls

Class of the layer to wrap.

required

Returns:

Type Description

A differentially private layer that doesn't have parameters.

Source code in lipdp/layers.py
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def DP_GNP_Factory(layer_cls):
    """Factory for creating differentially private gradient norm preserving layers that don't have parameters.

    Remark: the layer is assumed to be GNP.
    This means that the gradient norm is preserved by the layer (i.e its Jacobian norm is 1).
    Pllease ensure that the layer is GNP before using this factory.

    Args:
        layer_cls: Class of the layer to wrap.

    Returns:
        A differentially private layer that doesn't have parameters.
    """

    class DP_GNP(layer_cls, DPLayer):
        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)

        def backpropagate_params(self, input_bound, gradient_bound):
            raise ValueError("Layer doesn't have parameters")

        def backpropagate_inputs(self, input_bound, gradient_bound):
            return 1 * gradient_bound

        def propagate_inputs(self, input_bound):
            return input_bound

        def has_parameters(self):
            return False

    DP_GNP.__name__ = f"DP_{layer_cls.__name__}"
    return DP_GNP

clip_gradient(x, clip_value)

Clips the gradient during the backward pass.

Behave like identity function during the forward pass.

Source code in lipdp/layers.py
483
484
485
486
487
488
489
490
491
492
493
494
495
496
@tf.custom_gradient
def clip_gradient(x, clip_value):
    """Clips the gradient during the backward pass.

    Behave like identity function during the forward pass.
    """

    def grad_fn(dy):
        # clip by norm each row
        axes = list(range(1, len(dy.shape)))
        clipped_dy = tf.clip_by_norm(dy, clip_value, axes=axes)
        return clipped_dy, None  # No gradient for clip_value

    return x, grad_fn

make_residuals(merge_policy, wrapped_layers)

Returns a list of layers that implement a residual block.

Parameters:

Name Type Description Default
merge_policy

either "add" or "1-lip-add".

required
wrapped_layers

a list of layers that will be wrapped in residual blocks.

required

Returns:

Type Description

A list of layers that implement a residual block.

Source code in lipdp/layers.py
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
def make_residuals(merge_policy, wrapped_layers):
    """Returns a list of layers that implement a residual block.

    Args:
        merge_policy: either "add" or "1-lip-add".
        wrapped_layers: a list of layers that will be wrapped in residual blocks.

    Returns:
        A list of layers that implement a residual block.
    """
    layers = [DP_SplitResidual()]

    for layer in wrapped_layers:
        residual_block = DP_WrappedResidual(layer)
        layers.append(residual_block)

    layers.append(DP_MergeResidual(merge_policy))

    return layers