Skip to content

Customs related to optimization

Custom loss functions

SoftmaxFocalCrossEntropy

Bases: Loss

Categorical Focal Loss for multiclass classification. Inherited from tf.keras.losses.Loss

Focal loss applies a modulating term to the cross entropy loss to focus learning on hard misclassified examples. This results in both better performance in tasks with imbalanced data and a better calibrated outputs.

.. math:: FocalLoss(p_t) = \alpha_t * (1-p_t)^\gamma * log(p_t)

Where

.. math:: p_t = p; \hspace{1.2} a_t = a when y=1, and

.. math:: p_t = 1-p; \hspace{1.2} a_t = 1-a otherwise

Parameters:

Name Type Description Default
gamma float

modulating factor

 2.0
alpha float

weighting factor

 0.75
Source code in conftrainer/optimization/losses.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
class SoftmaxFocalCrossEntropy(Loss):
    """
    Categorical Focal Loss for multiclass classification. Inherited from tf.keras.losses.Loss

    Focal loss applies a modulating term to the cross entropy loss to focus learning on hard
    misclassified examples. This results in both better performance in tasks with imbalanced
    data and a better calibrated outputs.

    .. math::
        FocalLoss(p_t) = \\alpha_t * (1-p_t)^\\gamma * log(p_t)

    Where

    .. math::
        p_t = p; \\hspace{1.2} a_t = a
    when y=1, and

    .. math::
        p_t = 1-p; \\hspace{1.2} a_t = 1-a
    otherwise

    Parameters
    ----------
    gamma : float
        modulating factor
    alpha : float
        weighting factor
    """

    def __init__(self, gamma: float = 2., alpha: float = 0.75,
                 name: str = "softmax_focal_crossentropy", **kwargs) -> None:
        super().__init__(name=name, **kwargs)
        self.gamma = gamma
        self.alpha = alpha

    def call(self, y_true: tf.Tensor, y_pred: tf.Tensor, from_logits: bool = False) -> tf.Tensor:
        """
        Compute the loss for given inputs

        Returns
        -------
        out: tf.Tensor
            Categorical focal loss
        """
        if from_logits:
            y_pred = tf.nn.softmax(y_pred)
        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))

        result = -reduce_mean(self.alpha * pow(1. - pt_1, self.gamma) * log(pt_1 + epsilon())) - \
                 reduce_mean((1 - self.alpha) * pow(pt_0, self.gamma) * log(1. - pt_0 + epsilon()))

        return result

call(y_true, y_pred, from_logits=False)

Compute the loss for given inputs

Returns:

Name Type Description
out tf.Tensor

Categorical focal loss
Source code in conftrainer/optimization/losses.py 
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def call(self, y_true: tf.Tensor, y_pred: tf.Tensor, from_logits: bool = False) -> tf.Tensor:
    """
    Compute the loss for given inputs

    Returns
    -------
    out: tf.Tensor
        Categorical focal loss
    """
    if from_logits:
        y_pred = tf.nn.softmax(y_pred)
    pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
    pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))

    result = -reduce_mean(self.alpha * pow(1. - pt_1, self.gamma) * log(pt_1 + epsilon())) - \
             reduce_mean((1 - self.alpha) * pow(pt_0, self.gamma) * log(1. - pt_0 + epsilon()))

    return result

BYOLLoss

Bases: Loss

Implementation of BYOL loss from the BYOL paper.

Normalizes both online and target networks outputs and returns their cosine distance.

Inherits from tf.keras.losses.Loss

Parameters:

name: str name of the instance. Used during logging and in callbacks Methods:

get_serialized_dict: returns a dict with name of the loss as key and the serialized config as value

Source code in conftrainer/optimization/losses.py 
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
class BYOLLoss(Loss):
    """
    Implementation of BYOL loss from the BYOL paper.

    Normalizes both online and target networks outputs and returns their cosine distance.

    Inherits from tf.keras.losses.Loss

    Parameters:
    ----------
    name: str
        name of the instance. Used during logging and in callbacks
    Methods:
    --------
    get_serialized_dict:
        returns a dict with name of the loss as key and the serialized config as value
    """

    def __init__(self, name="byol_loss", **kwargs):
        super().__init__(name=name, **kwargs)

    @staticmethod
    def call(y_true: tf.Tensor, y_pred: tf.Tensor) -> tf.Tensor:
        """
        BYOL loss
        This is a simple cosine similarity.

        Parameters
        ----------
        y_true, y_pred: tf.Tensor
            Tensors of equal shape to calculate cosine similarity of
        """
        normed_true, normed_pred = l2_normalize(y_true, axis=-1), l2_normalize(y_pred, axis=-1)
        return reduce_sum((normed_true - normed_pred) ** 2, axis=-1)

call(y_true, y_pred) staticmethod

BYOL loss This is a simple cosine similarity.

Parameters:

Name Type Description Default
y_true tf.Tensor

Tensors of equal shape to calculate cosine similarity of

 required
y_pred tf.Tensor

Tensors of equal shape to calculate cosine similarity of

 required
Source code in conftrainer/optimization/losses.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
@staticmethod
def call(y_true: tf.Tensor, y_pred: tf.Tensor) -> tf.Tensor:
    """
    BYOL loss
    This is a simple cosine similarity.

    Parameters
    ----------
    y_true, y_pred: tf.Tensor
        Tensors of equal shape to calculate cosine similarity of
    """
    normed_true, normed_pred = l2_normalize(y_true, axis=-1), l2_normalize(y_pred, axis=-1)
    return reduce_sum((normed_true - normed_pred) ** 2, axis=-1)

SigmoidFocalCrossEntropy

Bases: Loss

Implements the focal loss function.

Focal loss was first introduced in the RetinaNet paper (https://arxiv.org/pdf/1708.02002.pdf). Focal loss is extremely useful for classification when you have highly imbalanced classes. It down-weights well-classified examples and focuses on hard examples. The loss value is much higher for a sample which is misclassified by the classifier as compared to the loss value corresponding to a well-classified example. One of the best use-cases of focal loss is its usage in object detection where the imbalance between the background class and other classes is extremely high.

Usage:

fl = tfa.losses.SigmoidFocalCrossEntropy() loss = fl( ... y_true = [[1.0], [1.0], [0.0]],y_pred = [[0.97], [0.91], [0.03]]) loss

Usage with tf.keras API:

model = tf.keras.Model() model.compile('sgd', loss=tfa.losses.SigmoidFocalCrossEntropy())

Args: alpha: balancing factor, default value is 0.25. gamma: modulating factor, default value is 2.0.

Returns: Weighted loss float Tensor. If reduction is NONE, this has the same shape as y_true; otherwise, it is scalar.

Raises: ValueError: If the shape of sample_weight is invalid or value of gamma is less than zero.

Source code in conftrainer/optimization/losses.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
class SigmoidFocalCrossEntropy(Loss):
    """Implements the focal loss function.

    Focal loss was first introduced in the RetinaNet paper
    (https://arxiv.org/pdf/1708.02002.pdf). Focal loss is extremely useful for
    classification when you have highly imbalanced classes. It down-weights
    well-classified examples and focuses on hard examples. The loss value is
    much higher for a sample which is misclassified by the classifier as compared
    to the loss value corresponding to a well-classified example. One of the
    best use-cases of focal loss is its usage in object detection where the
    imbalance between the background class and other classes is extremely high.

    Usage:

    >>> fl = tfa.losses.SigmoidFocalCrossEntropy()
    >>> loss = fl(
    ...     y_true = [[1.0], [1.0], [0.0]],y_pred = [[0.97], [0.91], [0.03]])
    >>> loss
    <tf.Tensor: shape=(3,), dtype=float32, numpy=array([6.8532745e-06, 1.9097870e-04, 2.0559824e-05],
    dtype=float32)>

    Usage with `tf.keras` API:

    >>> model = tf.keras.Model()
    >>> model.compile('sgd', loss=tfa.losses.SigmoidFocalCrossEntropy())

    Args:
      alpha: balancing factor, default value is 0.25.
      gamma: modulating factor, default value is 2.0.

    Returns:
      Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same
          shape as `y_true`; otherwise, it is scalar.

    Raises:
        ValueError: If the shape of `sample_weight` is invalid or value of
          `gamma` is less than zero.
    """

    def __init__(
            self,
            from_logits: bool = False,
            alpha=0.25,
            gamma=2.0,
            reduction: str = tf.keras.losses.Reduction.AUTO,
            name: str = "sigmoid_focal_crossentropy",
            **kwargs
    ):
        super().__init__(reduction=reduction, name=name, **kwargs)

        self.from_logits = from_logits
        self.alpha = alpha
        self.gamma = gamma
        self.reduction = reduction
        self.name = name

    @tf.function
    def call(self, y_true, y_pred, from_logits: bool = False) -> tf.Tensor:
        """Implements the focal loss function.

        Focal loss was first introduced in the RetinaNet paper
        (https://arxiv.org/pdf/1708.02002.pdf). Focal loss is extremely useful for
        classification when you have highly imbalanced classes. It down-weights
        well-classified examples and focuses on hard examples. The loss value is
        much higher for a sample which is misclassified by the classifier as compared
        to the loss value corresponding to a well-classified example. One of the
        best use-cases of focal loss is its usage in object detection where the
        imbalance between the background class and other classes is extremely high.

        Args:
            y_true: true targets tensor.
            y_pred: predictions tensor.
            alpha: balancing factor.
            gamma: modulating factor.

        Returns:
            Weighted loss float `Tensor`. If `reduction` is `NONE`,this has the
            same shape as `y_true`; otherwise, it is scalar.
        """
        y_pred = tf.convert_to_tensor(y_pred)
        y_true = tf.cast(y_true, dtype=y_pred.dtype)

        # Get the cross_entropy for each entry
        ce = K.binary_crossentropy(y_true, y_pred, from_logits=self.from_logits)

        # If logits are provided then convert the predictions into probabilities
        if from_logits:
            pred_prob = tf.sigmoid(y_pred)
        else:
            pred_prob = y_pred

        p_t = (y_true * pred_prob) + ((1 - y_true) * (1 - pred_prob))

        self.alpha = tf.cast(self.alpha, dtype=y_true.dtype)
        alpha_factor = y_true * self.alpha + (1 - y_true) * (1 - self.alpha)

        self.gamma = tf.cast(self.gamma, dtype=y_true.dtype)
        modulating_factor = tf.pow((1.0 - p_t), self.gamma)

        # compute the final loss and return
        return tf.reduce_sum(alpha_factor * modulating_factor * ce, axis=-1)

call(y_true, y_pred, from_logits=False)

Implements the focal loss function.

Focal loss was first introduced in the RetinaNet paper (https://arxiv.org/pdf/1708.02002.pdf). Focal loss is extremely useful for classification when you have highly imbalanced classes. It down-weights well-classified examples and focuses on hard examples. The loss value is much higher for a sample which is misclassified by the classifier as compared to the loss value corresponding to a well-classified example. One of the best use-cases of focal loss is its usage in object detection where the imbalance between the background class and other classes is extremely high.

Args: y_true: true targets tensor. y_pred: predictions tensor. alpha: balancing factor. gamma: modulating factor.

Returns: Weighted loss float Tensor. If reduction is NONE,this has the same shape as y_true; otherwise, it is scalar.

Source code in conftrainer/optimization/losses.py 
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
@tf.function
def call(self, y_true, y_pred, from_logits: bool = False) -> tf.Tensor:
    """Implements the focal loss function.

    Focal loss was first introduced in the RetinaNet paper
    (https://arxiv.org/pdf/1708.02002.pdf). Focal loss is extremely useful for
    classification when you have highly imbalanced classes. It down-weights
    well-classified examples and focuses on hard examples. The loss value is
    much higher for a sample which is misclassified by the classifier as compared
    to the loss value corresponding to a well-classified example. One of the
    best use-cases of focal loss is its usage in object detection where the
    imbalance between the background class and other classes is extremely high.

    Args:
        y_true: true targets tensor.
        y_pred: predictions tensor.
        alpha: balancing factor.
        gamma: modulating factor.

    Returns:
        Weighted loss float `Tensor`. If `reduction` is `NONE`,this has the
        same shape as `y_true`; otherwise, it is scalar.
    """
    y_pred = tf.convert_to_tensor(y_pred)
    y_true = tf.cast(y_true, dtype=y_pred.dtype)

    # Get the cross_entropy for each entry
    ce = K.binary_crossentropy(y_true, y_pred, from_logits=self.from_logits)

    # If logits are provided then convert the predictions into probabilities
    if from_logits:
        pred_prob = tf.sigmoid(y_pred)
    else:
        pred_prob = y_pred

    p_t = (y_true * pred_prob) + ((1 - y_true) * (1 - pred_prob))

    self.alpha = tf.cast(self.alpha, dtype=y_true.dtype)
    alpha_factor = y_true * self.alpha + (1 - y_true) * (1 - self.alpha)

    self.gamma = tf.cast(self.gamma, dtype=y_true.dtype)
    modulating_factor = tf.pow((1.0 - p_t), self.gamma)

    # compute the final loss and return
    return tf.reduce_sum(alpha_factor * modulating_factor * ce, axis=-1)

Optimization related utils

get_metrics(metric_config_list)

Import and initialize metrics from tensorflow and tensorflow addons

Parameters:

Name Type Description Default
metric_config_list List[Dict[str, dict]]

dicts with names and arguments of metrics to import

 required

Returns:

Name Type Description
out Dict[str, tf.keras.metrics.Metric]

names of metrics as keys and initialized metrics
Source code in conftrainer/optimization/utils.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
def get_metrics(metric_config_list: List[ObjInitConfig]) -> Dict[str, tf_metrics.Metric]:
    """
    Import and initialize metrics from tensorflow and tensorflow addons

    Parameters
    ----------
    metric_config_list : List[Dict[str, dict]]
        dicts with names and arguments of metrics to import

    Returns
    -------
    out : Dict[str, tf.keras.metrics.Metric]
        names of metrics as keys and initialized metrics
    """

    modules = [tf_metrics, custom_metrics]
    imported_metrics = {}
    for metric_config in metric_config_list:
        name = metric_config.args.get("name", metric_config.name)
        imported_metrics[name] = create_object(config=metric_config, modules=modules)
    return imported_metrics