code/08-neural-nets/backprop.py

"""手写反向传播：每个数记住"它是怎么算出来的"，从而能把梯度一路传回去。
最后用它训练一个小网络，学会异或（XOR）。

    python backprop.py
纯 Python，不依赖任何库。固定了随机种子，结果每次一样。
"""
import math
import random


class Num:
    """一个会记账的数：记下自己的值、梯度，以及"我对每个上游数的局部导数"。"""

    def __init__(self, value, parents=()):
        self.value = value
        self.grad = 0.0
        self.parents = parents  # [(上游的 Num, 局部导数), ...]

    def __add__(self, other):
        other = other if isinstance(other, Num) else Num(other)
        # a + b 对 a 的导数是 1，对 b 的导数也是 1
        return Num(self.value + other.value, [(self, 1.0), (other, 1.0)])

    def __mul__(self, other):
        other = other if isinstance(other, Num) else Num(other)
        # a × b 对 a 的导数是 b，对 b 的导数是 a
        return Num(self.value * other.value, [(self, other.value), (other, self.value)])

    def __sub__(self, other):
        return self + (other * -1.0 if isinstance(other, Num) else Num(-other))

    def __pow__(self, n):
        # x 的 n 次方，导数是 n × x 的 (n-1) 次方
        return Num(self.value ** n, [(self, n * self.value ** (n - 1))])

    def tanh(self):
        t = math.tanh(self.value)
        # tanh 的导数是 1 - tanh²
        return Num(t, [(self, 1 - t * t)])

    __radd__ = __add__
    __rmul__ = __mul__

    def backward(self):
        """从这个数（通常是损失）出发，把梯度传给所有上游的数。"""
        order, seen = [], set()

        def visit(node):  # 先访问完所有上游，再把自己放进列表：得到一个"从上游到下游"的顺序
            if id(node) not in seen:
                seen.add(id(node))
                for parent, _ in node.parents:
                    visit(parent)
                order.append(node)

        visit(self)
        self.grad = 1.0  # 损失对自己的导数是 1
        for node in reversed(order):  # 从下游往上游，链式法则：上游梯度 += 下游梯度 × 局部导数
            for parent, local in node.parents:
                parent.grad += node.grad * local


# ---------- 1. 在一个小算式上验证 ----------
print("== 1. f = (a × b + c)²，a=2, b=-3, c=10")
a, b, c = Num(2.0), Num(-3.0), Num(10.0)
f = (a * b + c) ** 2
f.backward()
print(f"  f = {f.value}")
print(f"  自动算出的梯度：df/da={a.grad}, df/db={b.grad}, df/dc={c.grad}")


def numeric(fn, x, h=1e-6):
    return (fn(x + h) - fn(x - h)) / (2 * h)


print("  数值求导核对：",
      f"df/da≈{numeric(lambda v: (v * -3 + 10) ** 2, 2.0):.4f},",
      f"df/db≈{numeric(lambda v: (2 * v + 10) ** 2, -3.0):.4f},",
      f"df/dc≈{numeric(lambda v: (2 * -3 + v) ** 2, 10.0):.4f}")


# ---------- 2. 一个小神经网络 ----------
class Neuron:
    def __init__(self, n_inputs):
        self.w = [Num(random.uniform(-1, 1)) for _ in range(n_inputs)]
        self.b = Num(0.0)

    def __call__(self, xs):
        total = self.b
        for w, x in zip(self.w, xs):
            total = total + w * x
        return total.tanh()

    def params(self):
        return self.w + [self.b]


class Layer:
    def __init__(self, n_inputs, n_outputs):
        self.neurons = [Neuron(n_inputs) for _ in range(n_outputs)]

    def __call__(self, xs):
        return [n(xs) for n in self.neurons]

    def params(self):
        return [p for n in self.neurons for p in n.params()]


random.seed(1)
hidden, output = Layer(2, 4), Layer(4, 1)
params = hidden.params() + output.params()
print(f"\n== 2. 网络：2 个输入 → 4 个隐藏神经元 → 1 个输出，共 {len(params)} 个参数")

# 异或：两个输入相同输出 -1，不同输出 1（tanh 的输出范围是 -1 到 1）
data = [([0, 0], -1), ([0, 1], 1), ([1, 0], 1), ([1, 1], -1)]
lr = 0.1
for epoch in range(1, 301):
    loss = Num(0.0)
    for xs, y in data:
        pred = output(hidden(xs))[0]
        loss = loss + (pred - y) ** 2
    for p in params:
        p.grad = 0.0  # 每一轮都要清零，否则梯度会一直累加
    loss.backward()
    for p in params:
        p.value -= lr * p.grad
    if epoch in (1, 10, 50, 100, 200, 300):
        print(f"  第 {epoch:3d} 轮  损失 {loss.value:.4f}")

print("  训练后的预测：")
for xs, y in data:
    print(f"    输入 {xs} → {output(hidden(xs))[0].value:+.3f}（目标 {y:+d}）")