\documentclass[11pt]{article}

\usepackage{fullpage}
\usepackage{fleqn}

\parindent 0pt
%\newcommand{\myeqnarr}[1]{
%  \begin{equation}\begin{array}{@{\hspace*{.5in}}lll}#1\end{array}\end{equation}}
\newcommand{\myeqnarr}[1]{
   \begin{eqnarray}#1\end{eqnarray}}
\newcommand{\mypartial}[2]{\frac{\partial #1}{\partial #2}}
\newcommand{\mysection}[1]{\vspace*{.1in}\section*{#1}}

\begin{document}
\pagestyle{empty}

\begin{center}
\LARGE\bf
Derivation of the Backprop Learning Rule\\[.2in]

\large\bf
David S. Touretzky\\
15-486/782: Artificial Neural Networks\\[.2in]

\end{center}

\mysection{1. Unit Activation Equation}

\myeqnarr{
 net_k & = & \sum_{j} y_j\cdot w_{jk} \\[1em]
 y_k & = & f(net_k) 
 }

The transfer function $f(\cdot)$ can be any smooth, differentiable,
nonlinear function.  Originally the logistic function $(1+\exp(-x))^{-1}$
was used, but many today favor $\tanh$ because its range is $[-1,+1]$
instead of $[0,1]$, which gives better learning behavior.

\mysection{2. Error Measure}

Error $E$ is summed over all patterns and all output units.  The summation
over patterns is left implicit below.  $d_k$ is the desired output value for
unit $k$ on the present pattern, and $y_k$ is the actual output produced by
unit $k$.

\myeqnarr{
 E & = & \frac{1}{2} \sum_k \left(d_k - y_k \right)^2 
 }

\mysection{3. Error of the Output Layer}

$\delta_k$ is the gradient of the error with respect to unit $k$'s input.
It is backpropagated to the preceding layer to calculate $\delta_j$, and
also used to calculate the weight update $\Delta w_{jk}$.

\myeqnarr{
 \mypartial{E}{y_k} & = & (y_k - d_k) 
 }

\vspace*{.1in}

\myeqnarr{
 \delta_k & = & \mypartial{E}{net_k} \\[.2in]
 & = & \mypartial{E}{y_k} \cdot \mypartial{y_k}{net_k} \\[.2in]
 & = & (y_k - d_k) \cdot f^\prime(net_k)
 }

\newpage
\mysection{4. Backpropagated Error for Hidden Units}

We back-propagate the error through the $w_{jk}$ connections to calculate the
error signal for hidden unit $j$.

\myeqnarr{
 \mypartial{E}{y_j} & = & \sum_k \left( \mypartial{E}{net_k} \cdot \mypartial{net_k}{y_j} \right) \\[0.1in]
 & = & \sum_k \left( \delta_k \cdot w_{jk} \right)
 }

\myeqnarr{
 \delta_j & = & \mypartial{E}{net_j} \\[.2in]
 & = & \mypartial{E}{y_j} \cdot \mypartial{y_j}{net_j} \\[0.1in]
 & = & \mypartial{E}{y_j} \cdot f^\prime(net_j)
 }

\mysection{5. Weight Update}

We update the weights by the negative of the error gradient (because we
want error to decrease), scaled by a learning rate $\eta$.

\myeqnarr{
 \mypartial{E}{w_{jk}} & = & \mypartial{E}{net_k} \cdot \mypartial{net_k}{w_{jk}} \\[0.1in]
 & = & \delta_k \cdot y_j
 }

\myeqnarr{
 \mypartial{E}{w_{ij}} & = & \mypartial{E}{net_j} \cdot \mypartial{net_j}{w_{ij}} \\[0.1in]
 & = & \delta_j \cdot y_i
 }

\vspace*{0.05in}

\myeqnarr{
 \Delta w_{jk} & = & -\eta\cdot\mypartial{E}{w_{jk}} \\
 \Delta w_{ij} & = & -\eta\cdot\mypartial{E}{w_{ij}}
 }

\end{document}