Independent scholarship · Open accessThe monthly digest

Indexed text attachment

verify-pde-learning.py

Source for Partial Differential Equations in Machine Learning and AI: Naturality, Well-Posedness, and Spectral Rates

View paper · Download original file

Lines 1044–1143 of 1244. Source text is displayed literally.

1044check('K2 kernel gradient descent learns mode k at rate mu_k: after 200 steps mode 1 is fitted and 3.8 per cent of mode 8 remains',1045      r1_ < 1e-12 and close(r8_, (1 - mu_disc(8)) ** 200, 1e-9) and 0.03 < r8_ < 0.06,1046      'residual amplitudes %.0e (k=1), %.4f (k=8); steps to 1 per cent: %.1f, %.0f, %.0f for k = 1, 8, 16' % (r1_, r8_, st1, st8, st16))1047 1048n = 311049AD = [[v * 32 * 32 for v in row] for row in lap1(n, 'D')]1050ld, _ = jacobi_eig(AD)1051kap = ld[-1] / ld[0]1052AA = mm(tr(AD), AD)1053l2, _ = jacobi_eig(AA)1054kap2 = l2[-1] / l2[0]1055 1056 1057def its(k_):1058    return math.log(1e-6) / math.log((k_ - 1) / (k_ + 1))1059 1060 1061check('K3 the strong-form (collocation) least-squares problem has the square of the condition number of the energy (Ritz) problem',1062      close(kap, (math.sin(31 * PI / 64) / math.sin(PI / 64)) ** 2, 1e-8) and close(kap2, kap * kap, 1e-6),1063      '31 unknowns: condition %.1f (Ritz) and %.0f (strong form); optimal-step gradient iterations for 1e-6: %.0f and %.0f' % (kap, kap2, its(kap), its(kap2)))1064 1065AP = [[v / (hh * hh) for v in row] for row in lap1(N, 'P')]1066 1067 1068def rate_pinn(k):1069    v = [math.cos(k * i * hh) for i in range(N)]1070    w = mv(Kmat, mv(AP, mv(AP, v)))1071    return dot(v, w) / dot(v, v)1072 1073 1074rp1, rp16 = rate_pinn(1), rate_pinn(16)1075check('K4 composing the kernel with the strong-form loss reverses the bias: mode rates k^4/(1+k^2) grow with k',1076      close(rp1, mu_disc(1) * (4 / hh ** 2 * math.sin(hh / 2) ** 2) ** 2, 1e-9) and rp16 / rp1 > 100 and mu_disc(1) / mu_disc(16) > 100,1077      'regression rates %.4f and %.5f at k = 1, 16 (ratio 1/%.0f); strong-form rates %.4f and %.1f (ratio %.0f)' % (mu_disc(1), mu_disc(16), mu_disc(1) / mu_disc(16), rp1, rp16, rp16 / rp1))1078 1079xd = [0.5, 1.7, 3.0, 4.4, 5.6]1080ydat = [random.gauss(0, 1) for _ in xd]1081cc = solve([[Gper(a_ - b_) for b_ in xd] for a_ in xd], ydat)1082 1083 1084def uk(x):1085    return sum(c_ * Gper(x - b_) for c_, b_ in zip(cc, xd))1086 1087 1088r_int = max(abs(uk(a_) - y_) for a_, y_ in zip(xd, ydat))1089r_ode = max(abs(uk(x) - d2(uk, x)) for x in (0.1, 1.1, 2.3, 3.7, 5.0, 6.0))1090dl = 1e-51091r_jump = max(abs((uk(a_ + dl) - 2 * uk(a_) + uk(a_ - dl)) / dl + c_) for a_, c_ in zip(xd, cc))1092check('K5 the kernel interpolant solves u - u\'\' = sum c_i delta_{x_i}: the representer theorem is a fundamental solution',1093      r_int < 1e-12 and r_ode < 1e-6 and r_jump < 1e-3, 'interpolation %.0e; equation residual off the data %.1e; jump of u\' plus c_i %.1e' % (r_int, r_ode, r_jump))1094 1095sec('8. Langevin dynamics and stochastic gradient descent')1096 1097 1098def fp_gap(V, lo, hi, n, beta, k=2):1099    h = (hi - lo) / n1100    xs = [lo + i * h for i in range(n + 1)]1101    Vs = [V(x) for x in xs]1102    D = 1 / beta1103    d = []1104    for i in range(n + 1):1105        s = 0.01106        if i < n:1107            s += math.exp(-beta * (Vs[i + 1] - Vs[i]) / 2)1108        if i > 0:1109            s += math.exp(-beta * (Vs[i - 1] - Vs[i]) / 2)1110        d.append(D / h ** 2 * s)1111    e = [-D / h ** 2] * n1112    psi = [math.exp(-beta * v / 2) for v in Vs]1113    res = 0.01114    for i in range(n + 1):1115        s = d[i] * psi[i] + (e[i - 1] * psi[i - 1] if i > 0 else 0) + (e[i] * psi[i + 1] if i < n else 0)1116        res = max(res, abs(s) / (D / h ** 2 * max(psi)))1117    return tri_eigs(d, e, k), res1118 1119 1120(ev1, rs1), (ev4, rs4) = fp_gap(lambda x: x * x / 2, -6, 6, 240, 1.0), fp_gap(lambda x: x * x / 2, -4, 4, 320, 4.0)1121check('L1 the Gibbs density is stationary for the Fokker-Planck operator, and for a quadratic loss the spectral gap is the curvature at every temperature',1122      rs1 < 1e-12 and rs4 < 1e-12 and abs(ev1[0]) < 1e-8 and close(ev1[1], 1, 5e-3) and close(ev4[1], 1, 5e-3),1123      'gap %.4f at beta = 1 and %.4f at beta = 4' % (ev1[1], ev4[1]))1124 1125kr = {}1126for beta in (4.0, 6.0, 8.0, 10.0):1127    ev, _ = fp_gap(lambda x: (x * x - 1) ** 2, -2.2, 2.2, 880, beta)1128    kr[beta] = (ev[1], math.sqrt(32) / PI * math.exp(-beta))1129rt = {b_: kr[b_][0] / kr[b_][1] for b_ in kr}1130check('L2 in a double well the gap is the Kramers rate (sqrt(32)/pi) e^{-beta*barrier}, exponentially small in the inverse temperature',1131      all(0.7 < v < 1.05 for v in rt.values()) and abs(rt[10.0] - 1) < abs(rt[4.0] - 1) and abs(rt[10.0] - 1) < 0.12,1132      'gap / Kramers = %s at beta = 4, 6, 8, 10; gap at beta = 10 is %.2e' % (', '.join('%.3f' % rt[b_] for b_ in sorted(rt)), kr[10.0][0]))1133 1134eta, lam_, sig = 0.1, 2.0, 1.51135v = 0.01136for _ in range(2000):1137    v = (1 - eta * lam_) ** 2 * v + eta ** 2 * sig ** 21138check('L3 constant-step SGD on a quadratic has stationary variance eta*sigma^2/(lambda(2-eta*lambda)): a temperature eta*sigma^2/2 to first order',1139      close(v, eta * sig ** 2 / (lam_ * (2 - eta * lam_)), 1e-12), 'variance %.5f; first-order value %.5f' % (v, eta * sig ** 2 / (2 * lam_)))1140 1141sec('9. Control: the Bellman operator and viscosity solutions')1142nS, nA, gam = 6, 3, 0.91143Pm = []