Independent scholarship · Open accessThe monthly digest

Indexed text attachment

verify-pde-learning.py

Source for Partial Differential Equations in Machine Learning and AI: Naturality, Well-Posedness, and Spectral Rates

View paper · Download original file

Lines 561–660 of 1244. Source text is displayed literally.

561def apply_stencil(N):562    h = 2 * PI / N563    xs = [j * h for j in range(N)]564    us = [u(t) for t in xs]565    out = [(2 * us[j] - us[j - 1] - us[(j + 1) % N]) / (h * h) for j in range(N)]566    ex = [lap_exact(t) for t in xs]567    return norm(axpy(-1, out, ex)) / norm(ex)568 569 570em = max(apply_mult(16), apply_mult(64))571es16, es64 = apply_stencil(16), apply_stencil(64)572sy16 = (math.sin(5 * PI / 16) / (5 * PI / 16)) ** 2573sy64 = (math.sin(5 * PI / 64) / (5 * PI / 64)) ** 2574check('A1 a Fourier multiplier commutes with resampling of band-limited inputs; a stencil does so only to second order',575      em < 1e-12 and es16 > 5 * es64 > 0 and close(sy16, 0.7175, 1e-3) and close(sy64, 0.9801, 1e-3),576      'multiplier error %.0e on 16 and 64 points; stencil relative error %.3f and %.4f; its symbol at k=5 is %.4f and %.4f of k^2' % (em, es16, es64, sy16, sy64))577 578K = 4579c = {}580for k in range(1, K + 1):581    c[k] = complex(random.gauss(0, 1), random.gauss(0, 1))582    c[-k] = c[k].conjugate()583c[0] = complex(0.3, 0)584 585 586def ub(x):587    return sum(c[k] * cmath.exp(1j * k * x) for k in c).real588 589 590exact = {k: sum(c[p] * c[k - p] for p in c if (k - p) in c) for k in range(-K, K + 1)}591 592 593def alias_err(N):594    xs = [2 * PI * j / N for j in range(N)]595    sq = [ub(t) ** 2 for t in xs]596    return max(abs(coef(sq, k) - exact[k]) for k in range(-K, K + 1))597 598 599e13, e12, e9 = alias_err(3 * K + 1), alias_err(3 * K), alias_err(2 * K + 1)600check('A2 squaring a signal of band K is alias-free in the band exactly when the grid has more than 3K points',601      e13 < 1e-12 and e12 > 1e-3 and e9 > 1e-3, 'K=4: error on 13 points %.0e, on 12 points %.3f, on 9 points %.3f' % (e13, e12, e9))602 603 604def relu_mean(N):605    return sum(max(math.sin(2 * PI * j / N), 0.0) for j in range(N)) / N606 607 608errs = {N: relu_mean(N) - 1 / PI for N in (8, 16, 32, 64)}609okf = all(close(relu_mean(N), 1 / (N * math.tan(PI / N)), 1e-13) for N in errs)610check('A3 a rectifier has unbounded band: the sampled mean of relu(sin) is cot(pi/N)/N, in error by -pi/(3 N^2)',611      okf and all(close(errs[N] * N * N, -PI / 3, 0.02) for N in errs) and close(errs[8] / errs[16], 4, 0.01),612      'error times N^2 = %s (limit %.4f)' % (', '.join('%.4f' % (errs[N] * N * N) for N in sorted(errs)), -PI / 3))613 614 615def relu_band(N):616    xs = [2 * PI * j / N for j in range(N)]617    v = [max(ub(t), 0.0) for t in xs]618    return [coef(v, k) for k in range(-K, K + 1)]619 620 621ref = relu_band(8192)622ea = {N: max(abs(p - q) for p, q in zip(relu_band(N), ref)) for N in (16, 32, 64, 128, 256)}623check('A4 the in-band aliasing error of a rectifier layer falls with oversampling at second order',624      all(ea[N] > ea[2 * N] for N in (16, 32, 64, 128)) and ea[256] < ea[16] / 100,625      'max coefficient error on 16..256 points: ' + ', '.join('%.1e' % ea[N] for N in sorted(ea)))626 627sec('3. Diffusion models and the heat equation')628W, MU, SD = [0.3, 0.7], [-1.5, 1.0], [0.4, 0.6]629 630 631def p_h(x, t):632    return sum(w * gauss(x, m, s * s + t) for w, m, s in zip(W, MU, SD))633 634 635def sc_h(x, t):636    return sum(w * gauss(x, m, s * s + t) * (-(x - m) / (s * s + t)) for w, m, s in zip(W, MU, SD)) / p_h(x, t)637 638 639def cdf_h(x, t):640    return sum(w * Phi((x - m) / math.sqrt(s * s + t)) for w, m, s in zip(W, MU, SD))641 642 643pts = [-2.0, -1.0, 0.0, 1.0, 2.0]644r1 = max(abs(d1(lambda t: p_h(x, t), 0.3) - 0.5 * d2(lambda y: p_h(y, 0.3), x)) for x in pts)645check('S1 the noised mixture density solves the heat equation p_t = p_xx / 2', r1 < 1e-6, 'max residual %.1e' % r1)646 647t = 0.5648r2 = 0.0649for x in pts:650    post = simpson(lambda y: y * p_h(y, 0) * gauss(x, y, t), -7, 7, 6000) / p_h(x, t)651    r2 = max(r2, abs(post - (x + t * sc_h(x, t))))652check('S2 Tweedie: the posterior mean of the clean sample is x + t * score', r2 < 1e-8, 'max difference %.1e' % r2)653 654 655def pf(eps=0.0):656    return lambda t, y: [-0.5 * (sc_h(y[0], t) + eps * math.sin(3 * y[0]))]657 658 659T = 1.0660x0s = [flow(pf(), [x], T, 0.0, 2000)[0] for x in pts]