深入解析bp反向传播算法的流程：从数学原理到高效实现

1次阅读

共计 2177 个字符，预计需要花费 6 分钟才能阅读完成。

反向传播算法的本质是链式法则的递归应用。考虑神经网络中第 $l$ 层的权重 $W^l$，其梯度计算可表示为：

$$\frac{\partial L}{\partial W^l} = \frac{\partial L}{\partial z^l} \frac{\partial z^l}{\partial W^l}$$

其中 $z^l$ 表示该层线性输出，$L$ 为损失函数。这个递推过程从输出层开始，逐层反向计算梯度。

def backward_naive(self, x, y):
    # 逐样本逐层计算梯度
    for i in range(len(x)):
        # 前向传播代码省略...
        for l in reversed(range(self.num_layers)):
            # 手工计算每层梯度
            delta = ...  # 误差项计算
            dW = np.outer(delta, self.activations[l-1])

def backward_vectorized(self, X, y):
    # 批量矩阵运算
    # 前向传播代码省略...
    delta = (self.output - y) * sigmoid_derivative(self.z_output)
    for l in reversed(range(self.num_layers)):
        dW = delta.T @ self.activations[l-1] / len(X)
        delta = (delta @ self.weights[l]) * sigmoid_derivative(self.zs[l-1])

性能测试数据（单次迭代耗时）：
| Batch Size | 循环计算 (ms) | 矩阵运算 (ms) |
|————|————-|————-|
| 32 | 45.2 | 3.1 |
| 128 | 178.6 | 5.8 |
| 512 | 702.4 | 15.3 |

import numpy as np
from typing import List, Tuple

class NeuralNetwork:
    def __init__(self, layer_sizes: List[int]):
        # He 初始化缓解梯度消失
        self.weights = [np.random.randn(y, x) * np.sqrt(2/x) 
                       for x, y in zip(layer_sizes[:-1], layer_sizes[1:])]

    def backprop(self, X: np.ndarray, y: np.ndarray) -> List[np.ndarray]:
        """完整反向传播实现"""
        # 保存各层激活值和线性输出
        activations = [X]
        zs = []

        # 前向传播
        for w in self.weights:
            z = activations[-1] @ w.T
            zs.append(z)
            activations.append(self._sigmoid(z))

        # 反向传播
        delta = (activations[-1] - y) * self._sigmoid_derivative(zs[-1])
        grads = [delta.T @ activations[-2] / len(X)]

        for i in reversed(range(len(self.weights)-1)):
            delta = (delta @ self.weights[i+1]) * self._sigmoid_derivative(zs[i])
            grads.insert(0, delta.T @ activations[i] / len(X))

        return grads

    def _sigmoid(self, z: np.ndarray) -> np.ndarray:
        return 1/(1+np.exp(-z))

    def _sigmoid_derivative(self, z: np.ndarray) -> np.ndarray:
        s = self._sigmoid(z)
        return s * (1-s)