Independent scholarship · Open accessThe monthly digest

Indexed text attachment

verify-pde-learning.py

Source for Partial Differential Equations in Machine Learning and AI: Naturality, Well-Posedness, and Spectral Rates

View paper · Download original file

Lines 584–683 of 1244. Source text is displayed literally.

584 585 586def ub(x):587    return sum(c[k] * cmath.exp(1j * k * x) for k in c).real588 589 590exact = {k: sum(c[p] * c[k - p] for p in c if (k - p) in c) for k in range(-K, K + 1)}591 592 593def alias_err(N):594    xs = [2 * PI * j / N for j in range(N)]595    sq = [ub(t) ** 2 for t in xs]596    return max(abs(coef(sq, k) - exact[k]) for k in range(-K, K + 1))597 598 599e13, e12, e9 = alias_err(3 * K + 1), alias_err(3 * K), alias_err(2 * K + 1)600check('A2 squaring a signal of band K is alias-free in the band exactly when the grid has more than 3K points',601      e13 < 1e-12 and e12 > 1e-3 and e9 > 1e-3, 'K=4: error on 13 points %.0e, on 12 points %.3f, on 9 points %.3f' % (e13, e12, e9))602 603 604def relu_mean(N):605    return sum(max(math.sin(2 * PI * j / N), 0.0) for j in range(N)) / N606 607 608errs = {N: relu_mean(N) - 1 / PI for N in (8, 16, 32, 64)}609okf = all(close(relu_mean(N), 1 / (N * math.tan(PI / N)), 1e-13) for N in errs)610check('A3 a rectifier has unbounded band: the sampled mean of relu(sin) is cot(pi/N)/N, in error by -pi/(3 N^2)',611      okf and all(close(errs[N] * N * N, -PI / 3, 0.02) for N in errs) and close(errs[8] / errs[16], 4, 0.01),612      'error times N^2 = %s (limit %.4f)' % (', '.join('%.4f' % (errs[N] * N * N) for N in sorted(errs)), -PI / 3))613 614 615def relu_band(N):616    xs = [2 * PI * j / N for j in range(N)]617    v = [max(ub(t), 0.0) for t in xs]618    return [coef(v, k) for k in range(-K, K + 1)]619 620 621ref = relu_band(8192)622ea = {N: max(abs(p - q) for p, q in zip(relu_band(N), ref)) for N in (16, 32, 64, 128, 256)}623check('A4 the in-band aliasing error of a rectifier layer falls with oversampling at second order',624      all(ea[N] > ea[2 * N] for N in (16, 32, 64, 128)) and ea[256] < ea[16] / 100,625      'max coefficient error on 16..256 points: ' + ', '.join('%.1e' % ea[N] for N in sorted(ea)))626 627sec('3. Diffusion models and the heat equation')628W, MU, SD = [0.3, 0.7], [-1.5, 1.0], [0.4, 0.6]629 630 631def p_h(x, t):632    return sum(w * gauss(x, m, s * s + t) for w, m, s in zip(W, MU, SD))633 634 635def sc_h(x, t):636    return sum(w * gauss(x, m, s * s + t) * (-(x - m) / (s * s + t)) for w, m, s in zip(W, MU, SD)) / p_h(x, t)637 638 639def cdf_h(x, t):640    return sum(w * Phi((x - m) / math.sqrt(s * s + t)) for w, m, s in zip(W, MU, SD))641 642 643pts = [-2.0, -1.0, 0.0, 1.0, 2.0]644r1 = max(abs(d1(lambda t: p_h(x, t), 0.3) - 0.5 * d2(lambda y: p_h(y, 0.3), x)) for x in pts)645check('S1 the noised mixture density solves the heat equation p_t = p_xx / 2', r1 < 1e-6, 'max residual %.1e' % r1)646 647t = 0.5648r2 = 0.0649for x in pts:650    post = simpson(lambda y: y * p_h(y, 0) * gauss(x, y, t), -7, 7, 6000) / p_h(x, t)651    r2 = max(r2, abs(post - (x + t * sc_h(x, t))))652check('S2 Tweedie: the posterior mean of the clean sample is x + t * score', r2 < 1e-8, 'max difference %.1e' % r2)653 654 655def pf(eps=0.0):656    return lambda t, y: [-0.5 * (sc_h(y[0], t) + eps * math.sin(3 * y[0]))]657 658 659T = 1.0660x0s = [flow(pf(), [x], T, 0.0, 2000)[0] for x in pts]661r3 = max(abs(cdf_h(a, 0) - cdf_h(b, T)) for a, b in zip(x0s, pts))662check('S3 the probability-flow ODE dx/dt = -score/2 transports the noised law back to the data law',663      r3 < 1e-8, 'max change of the distribution function along five trajectories %.1e' % r3)664 665eps = 1e-3666x0p = [flow(pf(eps), [x], T, 0.0, 2000)[0] for x in pts]667sens = max(abs(a - b) for a, b in zip(x0s, x0p)) / eps668jac = max(p_h(b, T) / p_h(a, 0) for a, b in zip(x0s, pts))669amp = math.exp(10 ** 2 * T / 2)670check('S4 the backward heat equation amplifies mode 10 by e^50; the reverse flow with the score is Lipschitz in the score error',671      amp > 5e21 and sens < 5 and jac < 50,672      'backward amplification %.2e; sample displacement per unit score error %.3f; largest Jacobian of the reverse map %.2f' % (amp, sens, jac))673 674 675def sc2(x, t, a=1.0):676    return -x / t + (a / t) * math.tanh(a * x / t)677 678 679g1, g2 = d1(lambda y: sc2(y, 0.1), 0.0, 1e-5), d1(lambda y: sc2(y, 0.01), 0.0, 1e-6)680check('S5 for two-point data the slope of the score at the midpoint is -1/t + 1/t^2: the difficulty moves into small t',681      close(g1, 90, 1e-5) and close(g2, 9900, 1e-5), 'slope %.2f at t=0.1 and %.1f at t=0.01' % (g1, g2))682 683