import numpy as np

from scipy.special import expit

class SiLU():
    """
    smooth, y-range [-inf, inf], no vanishing gradient (no decay), moderate computationally expensive
    """
    @staticmethod
    def f(x):
        # sig_x = 1 / (1 + np.exp(-x))
        sig_x = expit(x)
        return x * sig_x
    @staticmethod
    def df(x, order=1):
        sig_x = 1 / (1 + np.exp(-x))
        match order:
            case 1:
                return sig_x + x * sig_x * (1 - sig_x)
            case 2:
                return sig_x * (1 - sig_x) + sig_x * (1 - 2 * sig_x) * x + (1 - sig_x) * x * sig_x
            case _:
                raise NotImplementedError(f"SiLU gradient order {order} is not implemented yet")
