Kerasのhard_sigmoidが max(0, min(1, (0.2 * x) + 0.5)) である話

g(x) = \left\{
\begin{array}{ll}
0 & (x < -2.5) \\
0.2x + 0.5 & (-2.5 \leq x \leq 2.5) \\
1 & (2.5 < x)
\end{array}
\right.

def hard_sigmoid(x):
    """Segment-wise linear approximation of sigmoid.

    Faster than sigmoid.
    Returns `0.` if `x < -2.5`, `1.` if `x > 2.5`.
    In `-2.5 <= x <= 2.5`, returns `0.2 * x + 0.5`.

    # Arguments
        x: A tensor or variable.

    # Returns
        A tensor.

    {{np_implementation}}
    """
    x = (0.2 * x) + 0.5
    zero = _to_tensor(0., x.dtype.base_dtype)
    one = _to_tensor(1., x.dtype.base_dtype)
    x = tf.clip_by_value(x, zero, one)
    return x

def hard_sigmoid(x):
    """
    An approximation of sigmoid.

    More approximate and faster than ultra_fast_sigmoid.

    Approx in 3 parts: 0, scaled linear, 1.

    Removing the slope and shift does not make it faster.

    """
    # Use the same dtype as determined by "upgrade_to_float",
    # and perform computation in that dtype.
    out_dtype = scalar.upgrade_to_float(scalar.Scalar(dtype=x.dtype))[0].dtype
    slope = tensor.constant(0.2, dtype=out_dtype)
    shift = tensor.constant(0.5, dtype=out_dtype)
    x = (x * slope) + shift
    x = tensor.clip(x, 0, 1)
    return x

g(x) = \left\{
\begin{array}{ll}
0 & (x < -a/2) \\
\frac{x}{a} + \frac{1}{2} & (-a/2 \leq x \leq a/2) \\
1 & (a/2 < x)
\end{array}
\right.

\begin{align}
\int_{-\infty}^0 \left(f(x) - g(x) \right)^2 dx &= \int_{-\infty}^0 \left(f(x)\right)^2 dx - 2\int_{-\infty}^0 f(x)g(x) dx + \int_{-\infty}^0 \left( g(x) \right)^2 dx \\
&= \int_{-\infty}^0 \left(f(x)\right)^2 dx - 2\int_{-a/2}^0 f(x)g(x) dx + \int_{-a/2}^0 \left( g(x) \right)^2 dx \\
&= \int_{-\infty}^0 \left(f(x)\right)^2 dx - 2\int_{-a/2}^0 \frac{e^x}{e^x+1} \left(\frac{x}{a} + \frac{1}{2}\right)dx + \int_{-a/2}^0 \left(\frac{x}{a} + \frac{1}{2}\right)^2 dx 
\end{align}

\begin{align}
\int_{-a/2}^0 \left(\frac{x}{a} + \frac{1}{2}\right)^2 dx &= \int_{0}^{a/2} \left(\frac{y}{a} \right)^2 dy = \int_{0}^{a/2} \frac{y^2}{a^2} dy = \frac{1}{a^2} \frac{1}{3}\left(\frac{a}{2}\right)^3 = \frac{a}{24}
\end{align}

\begin{align}
\frac{d}{da}\left( - 2\int_{-a/2}^0 \frac{e^x}{e^x+1} \left(\frac{x}{a} + \frac{1}{2}\right)dx\right) + \frac{1}{24}
\end{align}

\begin{align}
\frac{d}{da} \left (\int_{-a/2}^{0} M(a,x) dx \right) &= \int_{-a/2}^{0}\frac{\partial}{\partial a} M(a,x) dx \\ 
&= \int_{-a/2}^{0} 2 \left(-\frac{x}{a^2} \right) \frac{e^x}{e^x+1} dx
\end{align}

\begin{align}
0 &= \frac{1}{24} + 2 \left. \frac{x}{a^2}  \log(1+e^x)\right|_{x=-a/2}^{x=0} - 2 \int_{-a/2}^{0}  \frac{1}{a^2}  \log(1+e^x) dx \\
&= \frac{1}{24} + \frac{1}{a}  \log(1+e^{-a/2}) - \frac{2}{a^2} \int_{-a/2}^{0}  \log(1+e^x) dx
\end{align}

\begin{align}
\int_{-a/2}^{0} \log(1+e^x) dx = \int_{-e^{-a/2}}^{-1} \log(1-u) \frac{du}{u}
\end{align}

\operatorname{Li}_2(z) = -\lim_{h \to 0^{-}} \int_h^z \frac{\log(1-u)}{u} du

0 = \frac{1}{24} + \frac{1}{a}  \log(1+e^{-a/2}) + \frac{2}{a^2} \left( \operatorname{Li}_2(-1)- \operatorname{Li}_2(-e^{-a/2}) \right)

g(x) = \left\{
\begin{array}{ll}
0 & (x < -a) \\
\frac{(x+a)^2}{2a^2} & (-a \leq x < 0) \\
1 - \frac{(x-a)^2}{2a^2} & (0 \leq x \leq a) \\
1 & (a < x)
\end{array}
\right.

\frac{17}{60} = \frac{4}{a^3}  \left(-\frac{a}{2} \operatorname{Li}_2(-e^a) - \frac{a}{2} \operatorname{Li}_2(-1) + \operatorname{Li}_3(-e^a) -  \operatorname{Li}_3(-1)\right)