\documentclass[border=3pt,tikz]{standalone}
\usepackage{amsfonts,amsmath,amssymb,amsthm}       % blackboard math symbols
%\usepackage{amssymb} % for \mathbb
\usepackage{tikz}
%\usepackage{etoolbox} % for \ifthen
\usepackage{listofitems} % for \readlist to create arrays
\usetikzlibrary{arrows.meta} % for arrow size
\usepackage[outline]{contour} % glow around text
\contourlength{1.4pt}
\usetikzlibrary{calc}
\usetikzlibrary{positioning}
\usetikzlibrary{positioning}
\tikzset{>=latex} % for LaTeX arrow head
\usepackage{xcolor}
\colorlet{myred}{red!80!black}
\colorlet{myblue}{blue!80!black}
\colorlet{mygreen}{green!60!black}
\colorlet{myorange}{orange!70!red!60!black}
\colorlet{mydarkred}{red!30!black}
\colorlet{mydarkblue}{blue!40!black}
\colorlet{mydarkgreen}{green!30!black}
\tikzstyle{node}=[thick,circle,draw=myblue,minimum size=25,inner sep=0.5,outer sep=0.6]
\tikzstyle{node in}=[node,green!20!black,draw=mygreen!30!black,fill=mygreen!25]
\tikzstyle{node hidden}=[node,blue!20!black,draw=myblue!30!black,fill=myblue!20]
\tikzstyle{node convol}=[node,orange!20!black,draw=myorange!30!black,fill=myorange!20]
\tikzstyle{node out}=[node,red!20!black,draw=myred!30!black,fill=myred!20]
\tikzstyle{connect}=[thick,mydarkblue] %,line cap=round
\tikzstyle{connect arrow}=[-{Latex[length=4,width=3.5]},thick,mydarkblue,shorten <=0.5,shorten >=1]
\tikzset{ % node styles, numbered for easy mapping with \nstyle
  node 1/.style={node in},
  node 2/.style={node hidden},
  node 3/.style={node out},
}
\def\nstyle{int(\lay<\Nnodlen?min(2,\lay):3)} % map layer number onto 1, 2, or 3
\begin{document}
\fontsize{14}{12}\selectfont

\begin{tikzpicture}[x=2.7cm,y=1.6cm]
  \message{^^JNeural network activation}
  \def\NI{4} % number of nodes in input layers
  \def\NO{3} % number of nodes in output layers
  \def\NOE{4} % number of nodes in output layers after expansion
  \def\yshift{0.4} % shift last node for dots

  % INPUT LAYER
  \foreach \i [evaluate={\c=int(\i==\NI); \y=\NI/2-\i; \index=(\i<\NI?int(\i):"n");}]
              in {0,...,\NI}{ % loop over nodes
    \ifnum\i=0 %bias node
    \node[node in,outer sep=0.6] (NI-\i) at (0,\y) {1};
    \else
    \node[node in,outer sep=0.6] (NI-\i) at (0,\y) {};
    \fi
  }

  % OUTPUT LAYER
  \foreach \i [evaluate={\c=int(\i==\NO); \y=\NO/2-\i; \index=(\i<\NO?int(\i):int(\i));}]
    in {1,...,\NO}{ % loop over nodes
    \ifnum\i=3 % high-lighted node
      \node[node hidden]
        (NO-\i) at (1-0.2,\y) {$\zeta_3$};
      \foreach \j [evaluate={\index=(\j<\NI?int(\j+1):int(\j+1);}] in {0,...,\NI}{ % loop over nodes in previous layer
        \draw[connect] (NI-\j) -- (NO-\i)
          node[pos=0.50] {\contour{white}{$w_{3,\index}$}};
      }
    \else % other light-colored nodes
      \node[node hidden]
        (NO-\i) at (1-0.2,\y) {$\zeta_{\index}$};
      \foreach \j in {0,...,\NI}{ % loop over nodes in previous layer
        \draw[connect,myblue!20] (NI-\j) -- (NO-\i);
      }
    \fi
  }

  % Other LAYERS NOT VISIBLE
  \foreach \i [evaluate={\c=int(\i==\NO); \y=\NO/2-\i; \index=(\i<\NO?int(\i):int(\i));}]
    in {\NO,...,1}{ % loop over nodes
      \node (NR-\i) at (-0.7,\y) {};
      \foreach \j in {1,...,\NI}{ % loop over nodes in the next layer
        \draw[connect,myblue!20] ($(NR-\i)!0.3!(NI-\j)$) -- (NI-\j);
      }
  }

  % bit of math
  \node[right=0.1cm of NO-3]
    {$\Rightarrow e^{\zeta_3}\propto\mathbb{P}_{a_{t}}^{\zeta}(\mathcal{U}_{a_{t}})$};

  % Texts
  \node[above of=NI-0,align=center] {input layers};
  \node[above of=NO-1,align=center] {output layers};

  % INPUT LAYER
  \foreach \i [evaluate={\c=int(\i==\NI); \y=\NI/2-\i; \index=(\i<\NI?int(\i):"n");}]
              in {0,...,\NI}{ % loop over nodes
    \ifnum\i=0 %bias node
    \node[node in,outer sep=0.6] (NIE-\i) at (0+3,\y) {1};
    \else
    \node[node in,outer sep=0.6] (NIE-\i) at (0+3,\y) {};
    \fi
  }

  % OUTPUT LAYER
  \foreach \i [evaluate={\c=int(\i==\NOE); \y=\NOE/2-\i; \index=(\i<\NOE?int(\i):int(\i));}]
    in {1,...,\NOE}{ % loop over nodes
    \ifnum\i>2 % high-lighted node
      \node[node hidden]
        (NOE-\i) at (1+3.5,\y) {$\tilde{\zeta}_\index$};
      \foreach \j [evaluate={\index=(\j<\NI?int(\j+1):int(\j+1));}] in {0,...,\NI}{ % loop over nodes in previous layer
        \draw[connect] (NIE-\j) -- (NOE-\i)
          node[pos=-5/4+\i/2] {\contour{white}{$\tilde{w}_{\i,\index}$}};
      }
    \else % other light-colored nodes
      \node[node hidden]
        (NOE-\i) at (1+3.5,\y) {$\tilde{\zeta}_{\index}$};
      \foreach \j in {0,...,\NI}{ % loop over nodes in previous layer
        \draw[connect,myblue!20] (NIE-\j) -- (NOE-\i);
      }
    \fi
  }

  % Other LAYERS NOT VISIBLE
  \foreach \i [evaluate={\c=int(\i==\NO); \y=\NO/2-\i; \index=(\i<\NO?int(\i):int(\i));}]
    in {\NO,...,1}{ % loop over nodes
      \node (NRE-\i) at (-0.7+3.2,\y) {};
      \foreach \j in {1,...,\NI}{ % loop over nodes in the next layer
        \draw[connect,myblue!20] ($(NRE-\i)!0.3!(NIE-\j)$) -- (NIE-\j);
      }
  }

  % Texts
  \node[above of=NIE-0,align=center] {input layers};
  \node[above of=NOE-1,align=center] {output layers};

  % expansion
  \draw[black,->,thick] ($(NO-2)!0.2!(NRE-2)$) -- node [above] {Expansion} (NRE-2) ;

\end{tikzpicture}
\end{document}
