Independent scholarship · Open accessThe monthly digest

Indexed text attachment

verify-pde-learning.py

Source for Partial Differential Equations in Machine Learning and AI: Naturality, Well-Posedness, and Spectral Rates

View paper · Download original file

Lines 541–640 of 1244. Source text is displayed literally.

541    return sum((a[k] * math.cos(k * x) + b[k] * math.sin(k * x)) / (1 + k * k) for k in range(1, K + 1))542 543 544def lap_exact(x):545    return sum((a[k] * math.cos(k * x) + b[k] * math.sin(k * x)) * k * k for k in range(1, K + 1))546 547 548def apply_mult(N):549    xs = [2 * PI * j / N for j in range(N)]550    c = dft([u(t) for t in xs])551    out = []552    for j in range(N):553        s = 0554        for k in range(N):555            f = k if k <= N // 2 else k - N556            s += c[k] / (1 + f * f) * cmath.exp(2j * PI * k * j / N)557        out.append(s.real)558    return max(abs(out[j] - mult_exact(xs[j])) for j in range(N))559 560 561def apply_stencil(N):562    h = 2 * PI / N563    xs = [j * h for j in range(N)]564    us = [u(t) for t in xs]565    out = [(2 * us[j] - us[j - 1] - us[(j + 1) % N]) / (h * h) for j in range(N)]566    ex = [lap_exact(t) for t in xs]567    return norm(axpy(-1, out, ex)) / norm(ex)568 569 570em = max(apply_mult(16), apply_mult(64))571es16, es64 = apply_stencil(16), apply_stencil(64)572sy16 = (math.sin(5 * PI / 16) / (5 * PI / 16)) ** 2573sy64 = (math.sin(5 * PI / 64) / (5 * PI / 64)) ** 2574check('A1 a Fourier multiplier commutes with resampling of band-limited inputs; a stencil does so only to second order',575      em < 1e-12 and es16 > 5 * es64 > 0 and close(sy16, 0.7175, 1e-3) and close(sy64, 0.9801, 1e-3),576      'multiplier error %.0e on 16 and 64 points; stencil relative error %.3f and %.4f; its symbol at k=5 is %.4f and %.4f of k^2' % (em, es16, es64, sy16, sy64))577 578K = 4579c = {}580for k in range(1, K + 1):581    c[k] = complex(random.gauss(0, 1), random.gauss(0, 1))582    c[-k] = c[k].conjugate()583c[0] = complex(0.3, 0)584 585 586def ub(x):587    return sum(c[k] * cmath.exp(1j * k * x) for k in c).real588 589 590exact = {k: sum(c[p] * c[k - p] for p in c if (k - p) in c) for k in range(-K, K + 1)}591 592 593def alias_err(N):594    xs = [2 * PI * j / N for j in range(N)]595    sq = [ub(t) ** 2 for t in xs]596    return max(abs(coef(sq, k) - exact[k]) for k in range(-K, K + 1))597 598 599e13, e12, e9 = alias_err(3 * K + 1), alias_err(3 * K), alias_err(2 * K + 1)600check('A2 squaring a signal of band K is alias-free in the band exactly when the grid has more than 3K points',601      e13 < 1e-12 and e12 > 1e-3 and e9 > 1e-3, 'K=4: error on 13 points %.0e, on 12 points %.3f, on 9 points %.3f' % (e13, e12, e9))602 603 604def relu_mean(N):605    return sum(max(math.sin(2 * PI * j / N), 0.0) for j in range(N)) / N606 607 608errs = {N: relu_mean(N) - 1 / PI for N in (8, 16, 32, 64)}609okf = all(close(relu_mean(N), 1 / (N * math.tan(PI / N)), 1e-13) for N in errs)610check('A3 a rectifier has unbounded band: the sampled mean of relu(sin) is cot(pi/N)/N, in error by -pi/(3 N^2)',611      okf and all(close(errs[N] * N * N, -PI / 3, 0.02) for N in errs) and close(errs[8] / errs[16], 4, 0.01),612      'error times N^2 = %s (limit %.4f)' % (', '.join('%.4f' % (errs[N] * N * N) for N in sorted(errs)), -PI / 3))613 614 615def relu_band(N):616    xs = [2 * PI * j / N for j in range(N)]617    v = [max(ub(t), 0.0) for t in xs]618    return [coef(v, k) for k in range(-K, K + 1)]619 620 621ref = relu_band(8192)622ea = {N: max(abs(p - q) for p, q in zip(relu_band(N), ref)) for N in (16, 32, 64, 128, 256)}623check('A4 the in-band aliasing error of a rectifier layer falls with oversampling at second order',624      all(ea[N] > ea[2 * N] for N in (16, 32, 64, 128)) and ea[256] < ea[16] / 100,625      'max coefficient error on 16..256 points: ' + ', '.join('%.1e' % ea[N] for N in sorted(ea)))626 627sec('3. Diffusion models and the heat equation')628W, MU, SD = [0.3, 0.7], [-1.5, 1.0], [0.4, 0.6]629 630 631def p_h(x, t):632    return sum(w * gauss(x, m, s * s + t) for w, m, s in zip(W, MU, SD))633 634 635def sc_h(x, t):636    return sum(w * gauss(x, m, s * s + t) * (-(x - m) / (s * s + t)) for w, m, s in zip(W, MU, SD)) / p_h(x, t)637 638 639def cdf_h(x, t):640    return sum(w * Phi((x - m) / math.sqrt(s * s + t)) for w, m, s in zip(W, MU, SD))