Independent scholarship · Open accessThe monthly digest

Indexed text attachment

verify-pde-learning.py

Source for Partial Differential Equations in Machine Learning and AI: Naturality, Well-Posedness, and Spectral Rates

View paper · Download original file

Lines 641–740 of 1244. Source text is displayed literally.

641 642 643pts = [-2.0, -1.0, 0.0, 1.0, 2.0]644r1 = max(abs(d1(lambda t: p_h(x, t), 0.3) - 0.5 * d2(lambda y: p_h(y, 0.3), x)) for x in pts)645check('S1 the noised mixture density solves the heat equation p_t = p_xx / 2', r1 < 1e-6, 'max residual %.1e' % r1)646 647t = 0.5648r2 = 0.0649for x in pts:650    post = simpson(lambda y: y * p_h(y, 0) * gauss(x, y, t), -7, 7, 6000) / p_h(x, t)651    r2 = max(r2, abs(post - (x + t * sc_h(x, t))))652check('S2 Tweedie: the posterior mean of the clean sample is x + t * score', r2 < 1e-8, 'max difference %.1e' % r2)653 654 655def pf(eps=0.0):656    return lambda t, y: [-0.5 * (sc_h(y[0], t) + eps * math.sin(3 * y[0]))]657 658 659T = 1.0660x0s = [flow(pf(), [x], T, 0.0, 2000)[0] for x in pts]661r3 = max(abs(cdf_h(a, 0) - cdf_h(b, T)) for a, b in zip(x0s, pts))662check('S3 the probability-flow ODE dx/dt = -score/2 transports the noised law back to the data law',663      r3 < 1e-8, 'max change of the distribution function along five trajectories %.1e' % r3)664 665eps = 1e-3666x0p = [flow(pf(eps), [x], T, 0.0, 2000)[0] for x in pts]667sens = max(abs(a - b) for a, b in zip(x0s, x0p)) / eps668jac = max(p_h(b, T) / p_h(a, 0) for a, b in zip(x0s, pts))669amp = math.exp(10 ** 2 * T / 2)670check('S4 the backward heat equation amplifies mode 10 by e^50; the reverse flow with the score is Lipschitz in the score error',671      amp > 5e21 and sens < 5 and jac < 50,672      'backward amplification %.2e; sample displacement per unit score error %.3f; largest Jacobian of the reverse map %.2f' % (amp, sens, jac))673 674 675def sc2(x, t, a=1.0):676    return -x / t + (a / t) * math.tanh(a * x / t)677 678 679g1, g2 = d1(lambda y: sc2(y, 0.1), 0.0, 1e-5), d1(lambda y: sc2(y, 0.01), 0.0, 1e-6)680check('S5 for two-point data the slope of the score at the midpoint is -1/t + 1/t^2: the difficulty moves into small t',681      close(g1, 90, 1e-5) and close(g2, 9900, 1e-5), 'slope %.2f at t=0.1 and %.1f at t=0.01' % (g1, g2))682 683 684def ent(t):685    return simpson(lambda y: -p_h(y, t) * math.log(p_h(y, t)), -9, 9, 6000)686 687 688dh = (ent(0.301) - ent(0.299)) / 0.002689J = simpson(lambda y: p_h(y, 0.3) * sc_h(y, 0.3) ** 2, -9, 9, 6000)690check('S6 de Bruijn: under the heat flow the entropy grows at half the Fisher information', close(dh, J / 2, 1e-6),691      'dh/dt = %.6f, J/2 = %.6f' % (dh, J / 2))692 693 694def p_ou(x, t):695    e = math.exp(-t)696    return sum(w * gauss(x, m * e, s * s * e * e + 1 - e * e) for w, m, s in zip(W, MU, SD))697 698 699def kl_ou(t):700    return simpson(lambda y: p_ou(y, t) * math.log(p_ou(y, t) / gauss(y, 0, 1)), -9, 9, 6000)701 702 703k0 = kl_ou(0)704rat = {t: kl_ou(t) / (math.exp(-2 * t) * k0) for t in (0.25, 0.5, 1.0, 2.0)}705lim = kl_ou(4.0) * math.exp(8.0)706mean0 = sum(w * m for w, m in zip(W, MU))707check('S7 Ornstein-Uhlenbeck noising contracts relative entropy to the Gaussian at least at rate 2, and exactly so in the limit',708      all(v <= 1 + 1e-12 for v in rat.values()) and close(lim, mean0 ** 2 / 2, 0.02),709      'KL_0 = %.4f; KL_t / (e^{-2t} KL_0) = %s; e^{2t} KL_t at t=4 is %.5f (mean^2/2 = %.5f)' % (k0, ', '.join('%.3f' % rat[t] for t in sorted(rat)), lim, mean0 ** 2 / 2))710 711t = 0.5712 713 714def p2(x):715    return 0.5 * (gauss(x, 1, t) + gauss(x, -1, t))716 717 718def s_true(x):719    return sc2(x, t)720 721 722def dsm(s):723    f = lambda z: gauss(z, 0, 1) * 0.5 * sum((s(x0 + math.sqrt(t) * z) + z / math.sqrt(t)) ** 2 for x0 in (-1.0, 1.0))724    return simpson(f, -9, 9, 6000)725 726 727def esm(s):728    return simpson(lambda y: p2(y) * (s(y) - s_true(y)) ** 2, -10, 10, 6000)729 730 731J2 = simpson(lambda y: p2(y) * s_true(y) ** 2, -10, 10, 6000)732gaps = [dsm(s) - esm(s) for s in (lambda y: -math.tanh(y), lambda y: -y / 2 + math.sin(y))]733check('S8 denoising score matching differs from explicit score matching by the constant 1/t - J, so they have the same minimizer',734      close(gaps[0], gaps[1], 1e-8) and close(gaps[0], 1 / t - J2, 1e-8), 'gap %.6f for two test functions; 1/t - J = %.6f' % (gaps[0], 1 / t - J2))735 736sec('4. Characteristics: continuous normalizing flows and flow matching')737A_, B_, C_ = 1.3, 0.2, 0.8738 739 740def vf(t, y):