Indexed text attachment
verify-pde-learning.py
Source for Partial Differential Equations in Machine Learning and AI: Naturality, Well-Posedness, and Spectral Rates
View paper · Download original file
Lines 664–763 of 1244. Source text is displayed literally.
664 665eps = 1e-3666x0p = [flow(pf(eps), [x], T, 0.0, 2000)[0] for x in pts]667sens = max(abs(a - b) for a, b in zip(x0s, x0p)) / eps668jac = max(p_h(b, T) / p_h(a, 0) for a, b in zip(x0s, pts))669amp = math.exp(10 ** 2 * T / 2)670check('S4 the backward heat equation amplifies mode 10 by e^50; the reverse flow with the score is Lipschitz in the score error',671 amp > 5e21 and sens < 5 and jac < 50,672 'backward amplification %.2e; sample displacement per unit score error %.3f; largest Jacobian of the reverse map %.2f' % (amp, sens, jac))673 674 675def sc2(x, t, a=1.0):676 return -x / t + (a / t) * math.tanh(a * x / t)677 678 679g1, g2 = d1(lambda y: sc2(y, 0.1), 0.0, 1e-5), d1(lambda y: sc2(y, 0.01), 0.0, 1e-6)680check('S5 for two-point data the slope of the score at the midpoint is -1/t + 1/t^2: the difficulty moves into small t',681 close(g1, 90, 1e-5) and close(g2, 9900, 1e-5), 'slope %.2f at t=0.1 and %.1f at t=0.01' % (g1, g2))682 683 684def ent(t):685 return simpson(lambda y: -p_h(y, t) * math.log(p_h(y, t)), -9, 9, 6000)686 687 688dh = (ent(0.301) - ent(0.299)) / 0.002689J = simpson(lambda y: p_h(y, 0.3) * sc_h(y, 0.3) ** 2, -9, 9, 6000)690check('S6 de Bruijn: under the heat flow the entropy grows at half the Fisher information', close(dh, J / 2, 1e-6),691 'dh/dt = %.6f, J/2 = %.6f' % (dh, J / 2))692 693 694def p_ou(x, t):695 e = math.exp(-t)696 return sum(w * gauss(x, m * e, s * s * e * e + 1 - e * e) for w, m, s in zip(W, MU, SD))697 698 699def kl_ou(t):700 return simpson(lambda y: p_ou(y, t) * math.log(p_ou(y, t) / gauss(y, 0, 1)), -9, 9, 6000)701 702 703k0 = kl_ou(0)704rat = {t: kl_ou(t) / (math.exp(-2 * t) * k0) for t in (0.25, 0.5, 1.0, 2.0)}705lim = kl_ou(4.0) * math.exp(8.0)706mean0 = sum(w * m for w, m in zip(W, MU))707check('S7 Ornstein-Uhlenbeck noising contracts relative entropy to the Gaussian at least at rate 2, and exactly so in the limit',708 all(v <= 1 + 1e-12 for v in rat.values()) and close(lim, mean0 ** 2 / 2, 0.02),709 'KL_0 = %.4f; KL_t / (e^{-2t} KL_0) = %s; e^{2t} KL_t at t=4 is %.5f (mean^2/2 = %.5f)' % (k0, ', '.join('%.3f' % rat[t] for t in sorted(rat)), lim, mean0 ** 2 / 2))710 711t = 0.5712 713 714def p2(x):715 return 0.5 * (gauss(x, 1, t) + gauss(x, -1, t))716 717 718def s_true(x):719 return sc2(x, t)720 721 722def dsm(s):723 f = lambda z: gauss(z, 0, 1) * 0.5 * sum((s(x0 + math.sqrt(t) * z) + z / math.sqrt(t)) ** 2 for x0 in (-1.0, 1.0))724 return simpson(f, -9, 9, 6000)725 726 727def esm(s):728 return simpson(lambda y: p2(y) * (s(y) - s_true(y)) ** 2, -10, 10, 6000)729 730 731J2 = simpson(lambda y: p2(y) * s_true(y) ** 2, -10, 10, 6000)732gaps = [dsm(s) - esm(s) for s in (lambda y: -math.tanh(y), lambda y: -y / 2 + math.sin(y))]733check('S8 denoising score matching differs from explicit score matching by the constant 1/t - J, so they have the same minimizer',734 close(gaps[0], gaps[1], 1e-8) and close(gaps[0], 1 / t - J2, 1e-8), 'gap %.6f for two test functions; 1/t - J = %.6f' % (gaps[0], 1 / t - J2))735 736sec('4. Characteristics: continuous normalizing flows and flow matching')737A_, B_, C_ = 1.3, 0.2, 0.8738 739 740def vf(t, y):741 th = math.tanh(A_ * y[0] + B_)742 return [C_ * th, C_ * A_ * (1 - th * th)]743 744 745r = 0.0746for x in (-1.5, -0.4, 0.3, 1.2):747 y, ld = flow(vf, [x, 0.0], 0.0, 1.0, 1000)748 yp = flow(vf, [x + 1e-4, 0.0], 0.0, 1.0, 1000)[0]749 ym = flow(vf, [x - 1e-4, 0.0], 0.0, 1.0, 1000)[0]750 r = max(r, abs(math.exp(ld) - (yp - ym) / 2e-4))751check('F1 along a characteristic the log-density changes at rate -div v (instantaneous change of variables)', r < 1e-6,752 'max difference between exp(int div v) and the derivative of the flow map %.1e' % r)753 754m_, s_ = 2.0, 0.5755 756 757def vfm(t, y):758 return [m_ + (t * s_ * s_ - (1 - t)) / ((1 - t) ** 2 + t * t * s_ * s_) * (y[0] - t * m_)]759 760 761r = max(abs(flow(vfm, [x], 0.0, 1.0, 2000)[0] - (m_ + s_ * x)) for x in (-2.0, -1.0, 0.0, 0.7, 2.0))762check('F2 flow matching between Gaussians with independent coupling integrates to the monotone transport map x -> m + s x',763 r < 1e-7, 'max error %.1e' % r)