{"about":"Lesson F5, optimisers. Computed by tools/school/fill_optimisers.mjs from system/school/optimisers.mjs, backprop.mjs and the attention lab's tinyformer. Do not edit by hand.","modelJsonSha256":"67600098679443bf87c78f0f3ef1a1399ae51642b3e9608517105d7831219d5e","rules":{"about":"Two parameters starting at (1, 0.1). Bias correction: a constant gradient (0.5, -0.002), Adam at the paper defaults (alpha 1e-3). With correction each coordinate moves by alpha on the first step whatever the gradient's size; without it, by alpha (1 - beta1) / sqrt(1 - beta2) = 3.162 alpha. Decoupled decay: zero gradient, lambda 0.1 (AdamW, lr 1e-3, eta 1) shrinks each weight by the factor 1 - eta lambda = 0.9 per step; Adam with the same lambda as L2 (folded into the gradient) moves each weight by about alpha per step, whatever its size, because Adam divides the decay by its own magnitude.","biasCorrection":{"gradient":[0.5,-0.002],"withCorrection":[[1,0.1],[0.999,0.101],[0.998,0.102],[0.997,0.103],[0.996,0.104],[0.995,0.105]],"withoutCorrection":[[1,0.1],[0.996838,0.103162],[0.992588,0.107411],[0.987638,0.112361],[0.982196,0.117802],[0.976399,0.123599]],"predictedFirstStepRatio":3.1623},"decoupledDecay":{"lambda":0.1,"adamw":[[1,0.1],[0.9,0.09],[0.81,0.081],[0.729,0.0729],[0.6561,0.06561],[0.59049,0.059049]],"adamL2":[[1,0.1],[0.999,0.099],[0.998,0.0980003],[0.997,0.097001],[0.996,0.0960024],[0.995,0.0950046]]}},"schedules":{"about":"Multipliers on the peak learning rate at update t. Linear warmup over 200 updates, then cosine (SGDR eq. 5, one cycle) or linear decay to zero at 4000; the attention lab's trainer uses warmup over 30 updates then flat.","warmup":200,"total":4000,"cosine":[[1,0.005],[50,0.25],[100,0.5],[150,0.75],[200,1],[250,0.9996],[300,0.9983],[350,0.9962],[400,0.9932],[450,0.9894],[500,0.9847],[550,0.9792],[600,0.9729],[650,0.9658],[700,0.9579],[750,0.9492],[800,0.9397],[850,0.9295],[900,0.9186],[950,0.9069],[1000,0.8946],[1050,0.8815],[1100,0.8679],[1150,0.8536],[1200,0.8386],[1250,0.8231],[1300,0.8071],[1350,0.7905],[1400,0.7735],[1450,0.7559],[1500,0.738],[1550,0.7196],[1600,0.7008],[1650,0.6818],[1700,0.6623],[1750,0.6427],[1800,0.6227],[1850,0.6026],[1900,0.5823],[1950,0.5618],[2000,0.5413],[2050,0.5207],[2100,0.5],[2150,0.4793],[2200,0.4587],[2250,0.4382],[2300,0.4177],[2350,0.3974],[2400,0.3773],[2450,0.3573],[2500,0.3377],[2550,0.3182],[2600,0.2992],[2650,0.2804],[2700,0.262],[2750,0.2441],[2800,0.2265],[2850,0.2095],[2900,0.1929],[2950,0.1769],[3000,0.1614],[3050,0.1464],[3100,0.1321],[3150,0.1185],[3200,0.1054],[3250,0.09308],[3300,0.08142],[3350,0.07047],[3400,0.06026],[3450,0.0508],[3500,0.04211],[3550,0.0342],[3600,0.02709],[3650,0.02079],[3700,0.0153],[3750,0.01064],[3800,0.006819],[3850,0.00384],[3900,0.001708],[3950,0.0004271],[4000,0]],"linear":[[1,0.005],[50,0.25],[100,0.5],[150,0.75],[200,1],[250,0.9868],[300,0.9737],[350,0.9605],[400,0.9474],[450,0.9342],[500,0.9211],[550,0.9079],[600,0.8947],[650,0.8816],[700,0.8684],[750,0.8553],[800,0.8421],[850,0.8289],[900,0.8158],[950,0.8026],[1000,0.7895],[1050,0.7763],[1100,0.7632],[1150,0.75],[1200,0.7368],[1250,0.7237],[1300,0.7105],[1350,0.6974],[1400,0.6842],[1450,0.6711],[1500,0.6579],[1550,0.6447],[1600,0.6316],[1650,0.6184],[1700,0.6053],[1750,0.5921],[1800,0.5789],[1850,0.5658],[1900,0.5526],[1950,0.5395],[2000,0.5263],[2050,0.5132],[2100,0.5],[2150,0.4868],[2200,0.4737],[2250,0.4605],[2300,0.4474],[2350,0.4342],[2400,0.4211],[2450,0.4079],[2500,0.3947],[2550,0.3816],[2600,0.3684],[2650,0.3553],[2700,0.3421],[2750,0.3289],[2800,0.3158],[2850,0.3026],[2900,0.2895],[2950,0.2763],[3000,0.2632],[3050,0.25],[3100,0.2368],[3150,0.2237],[3200,0.2105],[3250,0.1974],[3300,0.1842],[3350,0.1711],[3400,0.1579],[3450,0.1447],[3500,0.1316],[3550,0.1184],[3600,0.1053],[3650,0.09211],[3700,0.07895],[3750,0.06579],[3800,0.05263],[3850,0.03947],[3900,0.02632],[3950,0.01316],[4000,0]],"labTrainer":[[1,0.03333],[10,0.3333],[20,0.6667],[30,1],[40,1],[50,1],[60,1],[70,1],[80,1],[90,1],[100,1],[110,1],[120,1],[130,1],[140,1],[150,1],[160,1],[170,1],[180,1],[190,1],[200,1],[210,1],[220,1],[230,1],[240,1],[250,1],[260,1],[270,1],[280,1],[290,1],[300,1],[310,1],[320,1],[330,1],[340,1],[350,1],[360,1],[370,1],[380,1],[390,1],[400,1]]},"races":{"about":"Same start, each optimiser at its best step from the stated grid: fewest updates until the loss stays under the threshold to the end of the run ('stepsToThreshold'); 'firstBelow' is the first dip under it. Heavy ball beta 0.9; Adam at the paper defaults except alpha. Paths are thinned and stop at the settling update.","bowl":{"start":[-2,2],"maxSteps":2000,"threshold":0.000001,"min":[1,0.5],"settings":{"sgd":{},"momentum":{"beta":0.9},"adam":{"lr":0.001,"beta1":0.9,"beta2":0.999,"eps":1e-8}},"entrants":{"sgd":{"grid":[0.05,0.1,0.2,0.3,0.4,0.45],"chosenLr":0.4,"atGridEdge":false,"stepsToThreshold":33,"firstBelow":33,"finalLoss":1.5795e-32,"diverged":false,"tried":[{"lr":0.05,"stepsToThreshold":287,"firstBelow":287,"finalLoss":1.8921e-30,"diverged":false},{"lr":0.1,"stepsToThreshold":142,"firstBelow":142,"finalLoss":4.1991e-31,"diverged":false},{"lr":0.2,"stepsToThreshold":69,"firstBelow":69,"finalLoss":1.0498e-31,"diverged":false},{"lr":0.3,"stepsToThreshold":45,"firstBelow":45,"finalLoss":1.5795e-32,"diverged":false},{"lr":0.4,"stepsToThreshold":33,"firstBelow":33,"finalLoss":1.5795e-32,"diverged":false},{"lr":0.45,"stepsToThreshold":36,"firstBelow":36,"finalLoss":6.6347e-32,"diverged":false}],"path":[[-2,2,8.7894],[0.84067,2.9937,3.7126],[-0.47187,1.7187,1.6875],[0.62915,1.9407,0.83215],[0.21933,1.3731,0.44334],[0.66586,1.3662,0.25161],[0.55845,1.0923,0.14947],[0.7513,1.0342,0.091495],[0.73832,0.89117,0.057058],[0.82829,0.83466,0.035978],[0.84005,0.75469,0.022831],[0.88559,0.71158,0.014542],[0.90034,0.66457,0.0092818],[0.92515,0.63447,0.0059313],[0.93719,0.60589,0.0037928],[0.95151,0.58572,0.0024262],[0.96016,0.56797,0.0015523],[0.96876,0.55474,0.00099335],[0.97463,0.54357,0.00063569],[0.97993,0.53499,0.00040682],[0.98381,0.52791,0.00026036],[0.98713,0.52238,0.00016663],[0.98965,0.51787,0.00010664],[0.99175,0.51432,0.00006825],[0.99338,0.51144,0.00004368],[0.99472,0.50916,0.000027955],[0.99577,0.50732,0.000017891],[0.99662,0.50586,0.00001145],[0.99729,0.50469,0.0000073282],[0.99783,0.50375,0.00000469],[0.99827,0.503,0.0000030016],[0.99861,0.5024,0.000001921],[0.99889,0.50192,0.0000012295],[0.99911,0.50154,7.8686e-7]]},"momentum":{"grid":[0.01,0.02,0.05,0.1,0.2,0.3,0.5],"chosenLr":0.02,"atGridEdge":false,"stepsToThreshold":146,"firstBelow":126,"finalLoss":4.3335e-31,"diverged":false,"tried":[{"lr":0.01,"stepsToThreshold":148,"firstBelow":148,"finalLoss":9.3503e-30,"diverged":false},{"lr":0.02,"stepsToThreshold":146,"firstBelow":126,"finalLoss":4.3335e-31,"diverged":false},{"lr":0.05,"stepsToThreshold":151,"firstBelow":133,"finalLoss":7.6267e-32,"diverged":false},{"lr":0.1,"stepsToThreshold":149,"firstBelow":121,"finalLoss":2.3785e-31,"diverged":false},{"lr":0.2,"stepsToThreshold":149,"firstBelow":130,"finalLoss":0,"diverged":false},{"lr":0.3,"stepsToThreshold":153,"firstBelow":122,"finalLoss":1.7431e-31,"diverged":false},{"lr":0.5,"stepsToThreshold":153,"firstBelow":130,"finalLoss":6.6347e-32,"diverged":false}],"path":[[-2,2,8.7894],[-1.5985,2.1384,5.9435],[-0.8529,2.3743,2.5163],[-0.02862,2.5809,1.3863],[0.64437,2.6501,2.2169],[1.0305,2.5249,2.914],[1.1084,2.212,2.3145],[0.95378,1.7702,1.0236],[0.69646,1.287,0.20775],[0.47015,0.84882,0.24221],[0.3709,0.51759,0.60183],[0.4349,0.31824,0.67735],[0.63867,0.239,0.39377],[0.91714,0.24228,0.088752],[1.191,0.28076,0.026575],[1.393,0.313,0.15398],[1.4862,0.31414,0.25617],[1.4695,0.27951,0.22095],[1.3715,0.22133,0.11214],[1.237,0.16108,0.04501],[1.1111,0.12047,0.054417],[1.0259,0.11411,0.088285],[0.99425,0.14573,0.089427],[1.0098,0.20842,0.054272],[1.0529,0.28807,0.018259],[1.0993,0.36843,0.0075126],[1.1286,0.43588,0.016183],[1.1296,0.48265,0.023061],[1.1023,0.50773,0.017575],[1.0554,0.51575,0.0062882],[1.0028,0.5144,0.00021551],[0.9579,0.51151,0.0021256],[0.93005,0.51272,0.0064082],[0.92233,0.52026,0.007323],[0.93187,0.53302,0.0045924],[0.95174,0.54762,0.0017152],[0.97371,0.55997,0.0011632],[0.99087,0.56677,0.0022718],[0.99939,0.56644,0.0029742],[0.99896,0.55941,0.0023346],[0.99218,0.54772,0.0010953],[0.98313,0.53422,0.00037491],[0.97588,0.52167,0.00043954],[0.97319,0.51199,0.00073505],[0.97589,0.50592,0.00071573],[0.98303,0.50312,0.00037661],[0.99241,0.50243,0.000066213],[1.0015,0.50248,0.000013331],[1.0082,0.50209,0.00013501],[1.0115,0.50064,0.00021948],[1.0115,0.4981,0.00017466],[1.0089,0.49495,0.000074031],[1.0054,0.49191,0.000024291],[1.0022,0.48968,0.000046667],[1.0002,0.48869,0.000084398],[0.99986,0.48902,0.000085327],[1.0008,0.49044,0.000051903],[1.0025,0.49247,0.000020178],[1.0041,0.49462,0.000012855],[1.0051,0.49644,0.000021643],[1.0051,0.49771,0.000026245],[1.0041,0.49841,0.000018596],[1.0026,0.49867,0.0000065822],[1.0009,0.49872,6.4839e-7],[0.99946,0.4988,0.0000024388],[0.99855,0.49904,0.0000060327],[0.99826,0.4995,0.0000062385],[0.99848,0.50012,0.0000033332],[0.999,0.50078,7.9088e-7],[0.99957,0.50135,6.6063e-7],[0.99998,0.50172,0.0000019699],[1.0001,0.50184,0.0000026883],[1,0.50174,0.0000020799],[0.99971,0.50148,9.7967e-7]]},"adam":{"grid":[0.01,0.03,0.1,0.3,1],"chosenLr":0.3,"atGridEdge":false,"stepsToThreshold":146,"firstBelow":131,"finalLoss":0,"diverged":false,"tried":[{"lr":0.01,"stepsToThreshold":1372,"firstBelow":1372,"finalLoss":1.4606e-13,"diverged":false},{"lr":0.03,"stepsToThreshold":453,"firstBelow":453,"finalLoss":8.9837e-31,"diverged":false},{"lr":0.1,"stepsToThreshold":164,"firstBelow":164,"finalLoss":0,"diverged":false},{"lr":0.3,"stepsToThreshold":146,"firstBelow":131,"finalLoss":0,"diverged":false},{"lr":1,"stepsToThreshold":150,"firstBelow":134,"finalLoss":0,"diverged":false}],"path":[[-2,2,8.7894],[-1.4034,2.5903,4.4158],[-0.83676,3.0709,2.6589],[-0.34299,3.2567,2.4319],[0.034934,3.1304,2.3648],[0.28204,2.8164,1.974],[0.41558,2.4034,1.3386],[0.47132,1.9448,0.71422],[0.4905,1.4826,0.31067],[0.51181,1.0566,0.17357],[0.56448,0.70258,0.19087],[0.66246,0.44354,0.2091],[0.80277,0.28238,0.15839],[0.96883,0.2034,0.076009],[1.136,0.17987,0.033372],[1.2779,0.1824,0.056263],[1.3732,0.18618,0.10786],[1.4116,0.1764,0.13483],[1.3958,0.15067,0.11911],[1.3397,0.11744,0.083977],[1.2634,0.091477,0.060063],[1.1872,0.087805,0.054623],[1.1265,0.11608,0.052735],[1.0884,0.17714,0.040614],[1.0711,0.26288,0.021003],[1.066,0.35901,0.0063692],[1.0623,0.44951,0.0030446],[1.0507,0.52124,0.0059592],[1.0274,0.56747,0.007103],[0.9944,0.58889,0.0047272],[0.95852,0.59214,0.0027329],[0.92829,0.58655,0.0037784],[0.91066,0.5805,0.0060269],[0.90855,0.57866,0.0064199],[0.92013,0.58109,0.0046732],[0.93993,0.58424,0.0028477],[0.96119,0.58328,0.0022232],[0.9783,0.57458,0.002107],[0.98861,0.55747,0.001481],[0.99273,0.53448,0.00052015],[0.99359,0.51029,0.00003702],[0.99471,0.48978,0.00019766],[0.99845,0.47625,0.00044728],[1.0052,0.47047,0.00041025],[1.0132,0.47078,0.00027468],[1.0201,0.47418,0.00030223],[1.0235,0.47782,0.00040959],[1.0225,0.48004,0.00038414],[1.0179,0.48086,0.00023293],[1.0115,0.48144,0.00012013],[1.0055,0.48327,0.000099994],[1.0014,0.48721,0.000088542],[0.99948,0.49302,0.00003943],[0.99912,0.49952,0.0000019883],[0.99918,0.50518,0.00001305],[0.99869,0.50879,0.000038372],[0.99736,0.50998,0.000039463],[0.99557,0.50925,0.000027379],[0.9941,0.50757,0.000026093],[0.99365,0.50592,0.000030161],[0.99444,0.50482,0.000023672],[0.99617,0.50423,0.000010663],[0.99819,0.50373,0.0000044618],[0.99985,0.50288,0.0000050651],[1.0008,0.50149,0.000004205],[1.001,0.49975,0.0000012762],[1.0009,0.49811,0.0000011494],[1.0008,0.497,0.0000035132],[1.001,0.49666,0.0000042443],[1.0013,0.49699,0.0000029314],[1.0016,0.49768,0.0000021424],[1.0017,0.49839,0.0000022216],[1.0014,0.4989,0.0000016734],[1.0009,0.49918,5.7309e-7]]}}},"rosenbrock":{"start":[-1.2,1],"maxSteps":20000,"threshold":0.0001,"min":[1,1],"settings":{"sgd":{},"momentum":{"beta":0.9},"adam":{"lr":0.001,"beta1":0.9,"beta2":0.999,"eps":1e-8}},"entrants":{"sgd":{"grid":[0.00025,0.0005,0.001,0.0015,0.002],"chosenLr":0.0015,"atGridEdge":false,"stepsToThreshold":6217,"firstBelow":6217,"finalLoss":6.4913e-12,"diverged":false,"tried":[{"lr":0.00025,"stepsToThreshold":null,"firstBelow":null,"finalLoss":0.0037798,"diverged":false},{"lr":0.0005,"stepsToThreshold":18688,"firstBelow":18688,"finalLoss":0.000058766,"diverged":false},{"lr":0.001,"stepsToThreshold":9339,"firstBelow":9339,"finalLoss":1.9385e-8,"diverged":false},{"lr":0.0015,"stepsToThreshold":6217,"firstBelow":6217,"finalLoss":6.4913e-12,"diverged":false},{"lr":0.002,"stepsToThreshold":null,"firstBelow":4630,"finalLoss":0.00071992,"diverged":false}],"path":[[-1.2,1,24.2],[-0.95637,0.92267,3.8338],[-0.88826,0.79708,3.572],[-0.8142,0.67101,3.2979],[-0.73272,0.54494,3.0088],[-0.64173,0.41978,2.7016],[-0.53831,0.29745,2.3723],[-0.41845,0.1821,2.0169],[-0.27797,0.082778,1.6362],[-0.11836,0.016692,1.2514],[0.041086,0.0010547,0.91956],[0.17365,0.027638,0.68349],[0.27428,0.07217,0.52761],[0.35119,0.12027,0.4219],[0.41201,0.16685,0.34658],[0.4617,0.21047,0.29049],[0.50338,0.25091,0.24725],[0.53908,0.2883,0.21298],[0.57015,0.32294,0.18523],[0.59755,0.35508,0.16236],[0.62196,0.38499,0.14326],[0.64391,0.41289,0.1271],[0.66379,0.439,0.1133],[0.6819,0.46347,0.10142],[0.6985,0.48647,0.091109],[0.71377,0.50812,0.082109],[0.72789,0.52855,0.074207],[0.74098,0.54785,0.067237],[0.75316,0.56611,0.06106],[0.76452,0.58342,0.055566],[0.77515,0.59984,0.050661],[0.78512,0.61544,0.046268],[0.79448,0.63028,0.042323],[0.8033,0.6444,0.03877],[0.81161,0.65786,0.035563],[0.81945,0.6707,0.032662],[0.82687,0.68295,0.030032],[0.8339,0.69466,0.027643],[0.84056,0.70585,0.025469],[0.84689,0.71655,0.023488],[0.8529,0.72679,0.02168],[0.85861,0.7366,0.020028],[0.86406,0.74601,0.018515],[0.86924,0.75502,0.017129],[0.87419,0.76366,0.015858],[0.87891,0.77196,0.01469],[0.88342,0.77992,0.013617],[0.88772,0.78757,0.012629],[0.89184,0.79492,0.011719],[0.89578,0.80199,0.010881],[0.89956,0.80878,0.010107],[0.90317,0.81531,0.0093924],[0.90664,0.8216,0.0087323],[0.90996,0.82764,0.008122],[0.91314,0.83347,0.0075573],[0.9162,0.83907,0.0070346],[0.91914,0.84447,0.0065503],[0.92196,0.84968,0.0061015],[0.92466,0.85469,0.0056853],[0.92727,0.85952,0.0052992],[0.92977,0.86418,0.0049408],[0.93218,0.86867,0.0046079],[0.93449,0.873,0.0042986],[0.93672,0.87718,0.0040111],[0.93887,0.88122,0.0037438],[0.94093,0.88511,0.0034952],[0.94292,0.88886,0.0032638],[0.94483,0.89248,0.0030484],[0.94668,0.89598,0.0028478],[0.94846,0.89936,0.002661],[0.95017,0.90262,0.0024869],[0.95183,0.90578,0.0023246],[0.95342,0.90882,0.0021733],[0.95496,0.91176,0.0020323],[0.95644,0.9146,0.0019006],[0.95787,0.91734,0.0017778],[0.95925,0.92,0.0016632],[0.96058,0.92256,0.0015562],[0.96187,0.92504,0.0014563],[0.96311,0.92743,0.001363],[0.96431,0.92975,0.0012758],[0.96547,0.93199,0.0011944],[0.96659,0.93415,0.0011183],[0.96767,0.93625,0.0010472],[0.96871,0.93827,0.00098069],[0.96972,0.94023,0.00091852],[0.97069,0.94212,0.00086038],[0.97163,0.94396,0.000806],[0.97254,0.94573,0.00075514],[0.97342,0.94744,0.00070755],[0.97427,0.9491,0.00066302],[0.97509,0.95071,0.00062135],[0.97589,0.95226,0.00058235],[0.97666,0.95376,0.00054584],[0.9774,0.95522,0.00051166],[0.97812,0.95662,0.00047965],[0.97881,0.95799,0.00044969],[0.97948,0.9593,0.00042162],[0.98013,0.96058,0.00039533],[0.98076,0.96182,0.00037071],[0.98137,0.96301,0.00034764],[0.98196,0.96417,0.00032603],[0.98253,0.96529,0.00030578],[0.98308,0.96637,0.0002868],[0.98361,0.96743,0.00026902],[0.98413,0.96844,0.00025236],[0.98463,0.96943,0.00023673],[0.98511,0.97038,0.00022209],[0.98558,0.9713,0.00020836],[0.98603,0.9722,0.00019549],[0.98647,0.97306,0.00018343],[0.98689,0.9739,0.00017211],[0.9873,0.97471,0.0001615],[0.9877,0.9755,0.00015155],[0.98808,0.97626,0.00014222],[0.98846,0.977,0.00013347],[0.98882,0.97771,0.00012527],[0.98917,0.97841,0.00011757],[0.9895,0.97908,0.00011034],[0.98983,0.97973,0.00010357],[0.99001,0.98008,0.000099976]]},"momentum":{"grid":[0.0002,0.0005,0.001,0.002,0.003,0.0035,0.004],"chosenLr":0.003,"atGridEdge":false,"stepsToThreshold":285,"firstBelow":285,"finalLoss":1.3332e-28,"diverged":false,"tried":[{"lr":0.0002,"stepsToThreshold":4592,"firstBelow":4592,"finalLoss":1.6373e-15,"diverged":false},{"lr":0.0005,"stepsToThreshold":1782,"firstBelow":1782,"finalLoss":6.7053e-30,"diverged":false},{"lr":0.001,"stepsToThreshold":836,"firstBelow":836,"finalLoss":3.826e-29,"diverged":false},{"lr":0.002,"stepsToThreshold":502,"firstBelow":502,"finalLoss":0,"diverged":false},{"lr":0.003,"stepsToThreshold":285,"firstBelow":285,"finalLoss":1.3332e-28,"diverged":false},{"lr":0.0035,"stepsToThreshold":null,"firstBelow":null,"finalLoss":3.3422e+26,"diverged":true},{"lr":0.004,"stepsToThreshold":null,"firstBelow":null,"finalLoss":1.5614e+23,"diverged":true}],"path":[[-1.2,1,24.2],[-1.0367,0.28161,67.049],[0.64257,-0.034326,20.129],[0.35708,0.71797,35.278],[0.61115,0.21104,2.7904],[-0.062564,0.075651,1.6437],[-0.22186,-0.13315,4.8191],[0.027064,0.22471,5.9632],[0.24449,-0.15889,5.352],[0.14924,0.16747,2.832],[0.23193,-0.0059484,0.94681],[0.21175,0.038933,0.62482],[0.19231,0.096166,1.0026],[0.25807,-0.019405,1.2902],[0.21919,0.13088,1.2959],[0.32134,0.023936,1.0898],[0.29921,0.14841,0.83785],[0.39872,0.10313,0.67345],[0.39389,0.19919,0.56135],[0.48412,0.18677,0.49267],[0.48397,0.2705,0.3979],[0.55552,0.28018,0.27833],[0.57671,0.33511,0.1798],[0.60304,0.37972,0.18337],[0.65349,0.40187,0.18346],[0.66709,0.44858,0.1121],[0.68693,0.48591,0.11773],[0.72138,0.50919,0.090177],[0.74185,0.53889,0.079731],[0.75217,0.57187,0.065167],[0.76807,0.59792,0.060187],[0.78828,0.61844,0.045694],[0.80467,0.6389,0.045523],[0.81539,0.66054,0.035942],[0.82411,0.68123,0.031369],[0.83377,0.6994,0.029422],[0.84446,0.71521,0.024628],[0.8549,0.72961,0.02121],[0.86406,0.74334,0.01954],[0.87173,0.75664,0.017531],[0.87831,0.76938,0.01523],[0.88428,0.78138,0.013424],[0.88998,0.79255,0.012129],[0.89553,0.8029,0.011001],[0.90092,0.81252,0.0098933],[0.90608,0.82151,0.0088477],[0.91097,0.82995,0.0079276],[0.91554,0.83793,0.0071413],[0.9198,0.84549,0.0064619],[0.92377,0.85266,0.0058596],[0.92747,0.85946,0.0053159],[0.93093,0.86592,0.0048223],[0.93419,0.87204,0.0043749],[0.93725,0.87786,0.0039709],[0.94015,0.88338,0.0036068],[0.94289,0.88863,0.0032789],[0.9455,0.89361,0.0029833],[0.94797,0.89834,0.0027165],[0.95032,0.90284,0.0024752],[0.95255,0.90712,0.0022567],[0.95467,0.9112,0.0020587],[0.95669,0.91508,0.0018789],[0.95861,0.91878,0.0017155],[0.96044,0.9223,0.001567],[0.96219,0.92566,0.0014319],[0.96385,0.92886,0.001309],[0.96543,0.93192,0.0011969],[0.96694,0.93484,0.0010949],[0.96837,0.93762,0.0010018],[0.96974,0.94028,0.00091691],[0.97105,0.94282,0.00083944],[0.9723,0.94525,0.00076872],[0.97349,0.94757,0.00070412],[0.97462,0.94979,0.0006451],[0.97571,0.9519,0.00059116],[0.97674,0.95393,0.00054184],[0.97773,0.95587,0.00049674],[0.97868,0.95772,0.00045547],[0.97958,0.95949,0.00041771],[0.98044,0.96119,0.00038314],[0.98127,0.96281,0.0003515],[0.98206,0.96436,0.00032251],[0.98281,0.96585,0.00029596],[0.98353,0.96727,0.00027163],[0.98422,0.96863,0.00024934],[0.98488,0.96993,0.0002289],[0.98552,0.97118,0.00021017],[0.98612,0.97237,0.00019299],[0.9867,0.97352,0.00017723],[0.98725,0.97461,0.00016278],[0.98778,0.97566,0.00014952],[0.98829,0.97667,0.00013735],[0.98878,0.97763,0.00012619],[0.98924,0.97855,0.00011594],[0.98969,0.97944,0.00010654],[0.99011,0.98028,0.000097904]]},"adam":{"grid":[0.0003,0.001,0.003,0.01,0.03,0.1,0.3],"chosenLr":0.001,"atGridEdge":false,"stepsToThreshold":16790,"firstBelow":10346,"finalLoss":2.1448e-11,"diverged":false,"tried":[{"lr":0.0003,"stepsToThreshold":17395,"firstBelow":17395,"finalLoss":4.462e-13,"diverged":false},{"lr":0.001,"stepsToThreshold":16790,"firstBelow":10346,"finalLoss":2.1448e-11,"diverged":false},{"lr":0.003,"stepsToThreshold":19683,"firstBelow":6623,"finalLoss":5.9695e-16,"diverged":false},{"lr":0.01,"stepsToThreshold":19589,"firstBelow":3518,"finalLoss":3.26e-11,"diverged":false},{"lr":0.03,"stepsToThreshold":19801,"firstBelow":1804,"finalLoss":2.4527e-10,"diverged":false},{"lr":0.1,"stepsToThreshold":19838,"firstBelow":873,"finalLoss":1.1696e-8,"diverged":false},{"lr":0.3,"stepsToThreshold":null,"firstBelow":511,"finalLoss":0.012621,"diverged":false}],"path":[[-1.2,1,24.2],[-1.0962,1.1048,5.3295],[-1.0659,1.132,4.2697],[-1.0602,1.1299,4.2479],[-1.0568,1.1232,4.2345],[-1.053,1.1151,4.2187],[-1.0486,1.1058,4.2006],[-1.0436,1.0953,4.1801],[-1.0379,1.0836,4.1571],[-1.0316,1.0706,4.1316],[-1.0247,1.0563,4.1034],[-1.017,1.0407,4.0724],[-1.0086,1.0236,4.0385],[-0.99941,1.0051,4.0016],[-0.98933,0.98501,3.9613],[-0.97831,0.96332,3.9176],[-0.96629,0.93994,3.8702],[-0.9532,0.91478,3.8188],[-0.93893,0.88776,3.7633],[-0.92339,0.8588,3.7032],[-0.90647,0.82781,3.6384],[-0.88805,0.79472,3.5684],[-0.86799,0.75945,3.493],[-0.84613,0.72193,3.4118],[-0.82229,0.68212,3.3243],[-0.7963,0.63998,3.2302],[-0.76792,0.59551,3.1289],[-0.73691,0.54876,3.0201],[-0.703,0.49981,2.9033],[-0.66587,0.44885,2.7781],[-0.62518,0.39615,2.644],[-0.58057,0.34214,2.5008],[-0.53164,0.28744,2.3482],[-0.47803,0.23296,2.1865],[-0.41942,0.17992,2.0163],[-0.35567,0.12997,1.839],[-0.28696,0.085155,1.6571],[-0.21393,0.047827,1.474],[-0.13778,0.020258,1.2947],[-0.060241,0.0041455,1.1241],[0.016759,0.00013735,0.96677],[0.091484,0.0077067,0.82544],[0.16268,0.025432,0.70121],[0.22965,0.051465,0.59361],[0.29212,0.083927,0.50129],[0.35014,0.12114,0.42253],[0.40394,0.1617,0.3555],[0.45379,0.2045,0.29855],[0.50002,0.24865,0.25017],[0.54292,0.29347,0.20909],[0.58279,0.33842,0.17422],[0.61986,0.3831,0.14463],[0.65437,0.42716,0.11957],[0.68651,0.47034,0.098369],[0.71644,0.51241,0.080483],[0.74431,0.55321,0.06544],[0.77024,0.59256,0.052839],[0.79435,0.63035,0.042334],[0.81671,0.66645,0.033626],[0.83743,0.70078,0.026455],[0.85656,0.73324,0.020596],[0.87416,0.76377,0.01585],[0.89031,0.79231,0.012044],[0.90505,0.81882,0.0090248],[0.91843,0.84326,0.0066602],[0.93051,0.86563,0.0048337],[0.94134,0.88593,0.0034447],[0.95097,0.90419,0.0024064],[0.95946,0.92044,0.0016449],[0.96688,0.93475,0.001098],[0.97329,0.94721,0.00071426],[0.97876,0.9579,0.00045174],[0.98336,0.96695,0.0002771],[0.98718,0.97449,0.00016441],[0.9903,0.98067,0.000094092],[0.99281,0.98565,0.000051778],[0.99478,0.98956,0.000027308],[0.99629,0.99259,0.000013755],[0.99743,0.99487,0.0000065916],[0.99827,0.99654,0.0000029931],[0.99887,0.99773,0.0000012822],[0.99928,0.99856,5.1579e-7],[0.99956,0.99912,1.9385e-7],[0.99974,0.99948,6.7699e-8],[0.99985,0.9997,2.1841e-8],[0.99992,0.99984,6.4685e-9],[0.99996,0.99992,1.7465e-9],[0.99998,0.99996,4.2674e-10],[0.99999,0.99998,9.3592e-11],[1,0.99999,1.8263e-11],[1,1,3.1403e-12],[1,1,4.7082e-13],[1,1,6.084e-14],[1,1,6.6904e-15],[1,1,6.1738e-16],[1,1,4.7071e-17],[1,1,2.9146e-18],[1,1,1.4379e-19],[1,1,5.5319e-21],[1,1,1.6202e-22],[1,1,3.5156e-24],[1,1,5.4765e-26],[1,1,6.0014e-28],[1,1,9.38e-30],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[1,1,1.9722e-31],[0.99986,1.0001,0.000015682]]}}}},"aboveEdge":{"about":"The tilted bowl (curvatures 4 and 0.5) at 1.2 times plain descent's edge (step 0.6). Edges measured by bisection on the per-step growth of real runs: plain descent 2 / k_max, heavy ball (beta 0.9) 2 (1 + beta) / k_max. Each optimiser runs 400 updates from (-2, 2); Adam at its paper defaults with alpha 0.6.","kMax":4,"beta":0.9,"lr":0.6,"startLoss":8.7894,"plainBound":{"predicted":0.5,"measured":0.5},"momentumBound":{"predicted":0.95,"measured":0.949999999999},"runs":{"sgd":{"outcome":"diverges","finalLoss":1707900000000,"bestLoss":8.7894,"stepsToThreshold":null,"path":[[-2,2,8.7894],[2.261,3.4905,14.348],[-2.8227,-0.12334,26.711],[4.9117,3.867,51.663],[-5.4844,-2.4678,100.92],[9.3726,5.8771,197.64],[-11.216,-6.1724,387.29],[17.756,10.44,759.05],[-22.7,-12.997,1487.7],[34.011,19.689,2915.9],[-45.334,-26.16,5715.2],[65.785,37.967,11202],[-89.757,-51.854,21955],[128.02,73.866,43033],[-176.86,-102.16,84344],[249.98,144.26,165310],[-347.58,-200.74,324020],[489,282.26,635070],[-682.21,-393.95,1244700],[957.49,552.74,2439700],[-1338.1,-772.62,4781800],[1875.7,1082.9,9372300],[-2623.6,-1514.8,18370000],[3675.5,2122,36005000],[-5143.3,-2969.5,70569000],[7203,4158.6,138320000],[-10082,-5820.8,271100000],[14117,8150.3,531350000],[-19761,-11409,1041500000],[27668,15974,2041200000],[-38733,-22363,4000800000],[54229,31309,7841700000],[-75918,-43831,15370000000],[106290,61365,30125000000],[-148800,-85909,59044000000],[208320,120270,115730000000],[-291650,-168380,226820000000],[408310,235740,444570000000],[-571630,-330030,871370000000],[800280,462040,1707900000000]]},"momentum":{"outcome":"converges","finalLoss":4.9786e-18,"bestLoss":8.0654e-19,"stepsToThreshold":158,"path":[[-2,2,8.7894],[1.0122,1.2181,0.36803],[2.6999,-0.59615,2.517],[0.43345,-1.982,6.8679],[2.6567,1.5016,7.4933],[-1.1541,1.0361,5.698],[1.2168,1.9412,1.9749],[1.0925,0.112,0.062485],[1.1226,-0.83068,0.9936],[2.1369,0.44879,1.9332],[-0.15373,0.43131,2.2032],[1.2698,1.683,1.5596],[0.48885,0.50165,0.40697],[1.2769,0.053381,0.069511],[1.5915,0.14019,0.31311],[0.5604,0.20446,0.55891],[1.244,1.1772,0.65882],[0.44136,0.61889,0.39669],[1.2267,0.53193,0.09197],[1.2001,0.18277,0.035545],[0.93909,0.21609,0.087421],[1.1811,0.76607,0.17298],[0.60704,0.59734,0.18982],[1.1381,0.69531,0.096874],[0.98867,0.3187,0.025912],[1.0805,0.32005,0.010434],[1.1109,0.53346,0.025593],[0.79349,0.54098,0.054961],[1.07,0.68694,0.051535],[0.91217,0.43086,0.024544],[1.0978,0.42382,0.0076509],[1.0522,0.44423,0.0019837],[0.92577,0.49932,0.0086868],[1.0306,0.6193,0.016777],[0.91105,0.49172,0.013525],[1.07,0.49083,0.0067421],[1.0131,0.43501,0.0018794],[0.99553,0.48242,0.00036299],[1.0113,0.55391,0.0031199],[0.93872,0.51278,0.0047928],[1.0377,0.51996,0.0036297],[0.99304,0.45562,0.0018979],[1.0207,0.48269,0.00033071],[1.0029,0.51213,0.00016667],[0.96835,0.51337,0.0010468],[1.0151,0.52427,0.0013156],[0.98669,0.47851,0.001028],[1.0222,0.48973,0.00049833],[0.99944,0.4933,0.00003698],[0.98924,0.50765,0.000096361],[1.0033,0.51758,0.00031605],[0.98796,0.49375,0.00036762],[1.0153,0.49666,0.00029711],[0.99819,0.48913,0.00011624],[1.0002,0.50253,0.0000053354],[0.99872,0.50904,0.000041248],[0.99202,0.50088,0.000089416],[1.0079,0.50082,0.00010824],[0.99795,0.49151,0.000082495],[1.0041,0.49984,0.00002494],[0.99786,0.50277,0.0000034423],[0.99601,0.50277,0.000013406],[1.0028,0.50225,0.000025792],[0.99822,0.49532,0.000032624],[1.0041,0.4991,0.000021365],[0.99834,0.49951,0.0000056785],[0.99877,0.50229,0.0000017053],[1.0002,0.50204,0.000003719],[0.99872,0.49827,0.0000080029],[1.0028,0.49931,0.0000095234],[0.99902,0.49849,0.0000052929],[1.0002,0.50126,0.0000015493],[0.99933,0.50124,5.0179e-7],[0.99924,0.49987,0.0000010654],[1.0014,0.49973,0.0000025747],[0.99952,0.49864,0.0000026205],[1.0007,0.50045,0.0000013455],[0.99927,0.50049,4.5064e-7],[0.99966,0.50045,8.8935e-8],[1.0005,0.50002,3.6339e-7],[1.0006,0.49978,4.3628e-7]]},"adam":{"outcome":"converges","finalLoss":1.4472e-18,"bestLoss":4.2397e-19,"stepsToThreshold":145,"path":[[-2,2,8.7894],[-0.82449,3.1139,2.6709],[0.099576,3.3449,2.9488],[0.54959,2.7993,2.382],[0.62779,1.9821,0.89054],[0.56423,1.1177,0.15109],[0.56241,0.39379,0.37739],[0.7306,-0.048551,0.54424],[1.0471,-0.20615,0.29587],[1.4058,-0.18321,0.15803],[1.6742,-0.10837,0.34304],[1.7588,-0.076456,0.46517],[1.6572,-0.11058,0.32304],[1.4439,-0.16474,0.16448],[1.222,-0.1634,0.15637],[1.0744,-0.053541,0.15687],[1.0255,0.16041,0.067186],[1.0385,0.42022,0.0020367],[1.0491,0.64574,0.029209],[1.0092,0.77686,0.056702],[0.91717,0.80378,0.03603],[0.81435,0.76427,0.027512],[0.75488,0.71318,0.045931],[0.76883,0.6869,0.042033],[0.84387,0.68478,0.017839],[0.93662,0.67812,0.010979],[1.0028,0.63769,0.013641],[1.0248,0.55872,0.0055392],[1.0174,0.46543,0.00038318],[1.011,0.39386,0.006168],[1.0256,0.36707,0.0080185],[1.0569,0.38144,0.0044989],[1.0825,0.41326,0.0049662],[1.0822,0.43851,0.0055017],[1.054,0.44933,0.0021789],[1.0149,0.4554,0.00070678],[0.98629,0.47102,0.0014737],[0.97798,0.50037,0.00074505],[0.98335,0.53341,0.00035748],[0.98812,0.55453,0.0012829],[0.98389,0.55551,0.0011686],[0.97439,0.54119,0.00059256],[0.97009,0.52402,0.00070589],[0.97751,0.5133,0.00045883],[0.99327,0.50901,0.000034692],[1.0076,0.50476,0.00016203],[1.0133,0.49581,0.00020527],[1.011,0.48416,0.000097231],[1.0069,0.47636,0.00021134],[1.0064,0.47699,0.00020545],[1.0089,0.48459,0.000079007],[1.0099,0.4934,0.000084584],[1.0064,0.49878,0.000052956],[0.99968,0.50064,1.3301e-7],[0.9942,0.50204,0.000037476],[0.99296,0.5051,0.000040887],[0.99506,0.50875,0.000025263],[0.99729,0.51012,0.000040357],[0.99767,0.50784,0.000023064],[0.997,0.50345,0.0000065612],[0.99741,0.49975,0.000011539],[0.99965,0.49819,0.0000033881],[1.0023,0.49798,0.0000042059],[1.0035,0.49757,0.000010662],[1.0028,0.49655,0.0000058277],[1.0014,0.49593,0.0000058736],[1.0006,0.49674,0.000004894],[1.0007,0.49873,5.2243e-7],[1.0007,0.50063,0.0000016596],[0.99996,0.50147,0.0000014018],[0.99887,0.50143,9.5737e-7],[0.99832,0.50128,0.0000022735],[0.9987,0.50143,0.0000012375],[0.99948,0.50153,8.3327e-7],[0.99996,0.5011,7.6355e-7],[0.99997,0.50018,1.5763e-8],[0.99994,0.49935,3.5763e-7],[1.0002,0.49906,3.8175e-7],[1.0006,0.49922,2.8177e-7],[1.0008,0.49942,4.5363e-7],[1.0007,0.49945,3.3759e-7]]}}},"valleys":{"about":"Two valleys (the left one deeper). Which valley each start ends in after 3000 updates, on a 21 x 21 grid over [-2, 2]^2 (rows are y, -1 left, 1 right). SGD step 0.05, heavy ball step 0.005 with beta 0.9 (the same terminal speed on a constant slope, 0.005 / (1 - 0.9) = 0.05), Adam alpha 0.05. The demo start (2, 0) is one where momentum crosses the ridge.","settings":{"sgd":{"lr":0.05},"momentum":{"lr":0.005,"beta":0.9},"adam":{"lr":0.05,"beta1":0.9,"beta2":0.999,"eps":1e-8}},"basins":{"sgd":{"values":[[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1]],"fractionLeft":0.5125},"momentum":{"values":[[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,-1,-1,-1,-1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,-1,-1,-1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,-1,-1,-1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,-1,-1,-1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,-1,-1,-1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,-1,-1,-1],[1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,-1,-1],[1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,-1,-1],[1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,-1,-1],[1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,-1,-1],[1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,-1,-1],[1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,-1,-1],[1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,-1],[1,1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,-1],[1,1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,-1],[1,1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,-1],[1,1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,-1],[1,1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,-1],[1,1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,-1],[1,1,1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1]],"fractionLeft":0.5465},"adam":{"values":[[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],[-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]],"fractionLeft":0.5193}},"demo":{"start":[2,0],"runs":{"sgd":{"valley":"right","end":[0.97399,0.3896],"f":0.19743,"path":[[2,0,10.36],[0.972,0.3809,0.1975],[0.9739,0.3894,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974]]},"momentum":{"valley":"left","end":[-1.0241,-0.40965],"f":-0.20244,"path":[[2,0,10.36],[-1.117,-0.1128,0.005114],[-1.057,-0.4959,-0.1897],[-1.015,-0.3835,-0.2013],[-1.023,-0.4173,-0.2023],[-1.023,-0.4081,-0.2024],[-1.024,-0.41,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4097,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024],[-1.024,-0.4096,-0.2024]]},"adam":{"valley":"right","end":[0.97399,0.3896],"f":0.19743,"path":[[2,0,10.36],[1.042,0.6577,0.3028],[0.9036,0.29,0.222],[0.9526,0.3932,0.1993],[0.978,0.3912,0.1975],[0.9751,0.3891,0.1974],[0.9736,0.3895,0.1974],[0.974,0.3897,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974],[0.974,0.3896,0.1974]]}}}},"spiral":{"about":"The F4 spiral net (2, 8, 8, 2, tanh, 114 parameters) retrained from the same initial weights (seed 3) on the same 400 points (seed 1) with the same minibatches (32, seed 5) for 4000 updates. Each optimiser's step is chosen from its grid by the fewest updates to 95 percent accuracy on a separate validation draw (seed 4), so the held-out set (seed 2) is never used to choose. Heavy ball beta 0.9; Adam at the paper defaults except alpha. 'adamCosine' reruns Adam's chosen alpha with linear warmup over 200 updates and cosine decay to zero at 4000.","netSeed":3,"trainSeed":1,"heldSeed":2,"validSeed":4,"batchSeed":5,"points":400,"steps":4000,"batch":32,"every":100,"target":0.95,"beta":0.9,"grids":{"sgd":[0.1,0.3,1,3],"momentum":[0.01,0.03,0.1,0.3],"adam":[0.003,0.01,0.03,0.1,0.3]},"arms":{"sgd":{"chosenLr":0.3,"atGridEdge":false,"tried":[{"lr":0.1,"firstValidAtTarget":null,"finalValidAccuracy":0.7625,"finalHeldAccuracy":0.7125},{"lr":0.3,"firstValidAtTarget":2000,"finalValidAccuracy":0.9975,"finalHeldAccuracy":0.9975},{"lr":1,"firstValidAtTarget":2000,"finalValidAccuracy":0.9775,"finalHeldAccuracy":0.99},{"lr":3,"firstValidAtTarget":null,"finalValidAccuracy":0.5,"finalHeldAccuracy":0.5}],"firstHeldAtTarget":2000,"finalHeldAccuracy":0.9975,"finalHeldLoss":0.02252,"curve":[{"step":0,"loss":0.7413,"heldLoss":0.7396,"accuracy":0.375,"heldAccuracy":0.355},{"step":100,"loss":0.6614,"heldLoss":0.6492,"accuracy":0.605,"heldAccuracy":0.6075},{"step":200,"loss":0.651,"heldLoss":0.6407,"accuracy":0.6175,"heldAccuracy":0.6775},{"step":300,"loss":0.6547,"heldLoss":0.6473,"accuracy":0.6,"heldAccuracy":0.645},{"step":400,"loss":0.6551,"heldLoss":0.6504,"accuracy":0.6125,"heldAccuracy":0.6525},{"step":500,"loss":0.6468,"heldLoss":0.6395,"accuracy":0.62,"heldAccuracy":0.6725},{"step":600,"loss":0.6601,"heldLoss":0.6604,"accuracy":0.5775,"heldAccuracy":0.55},{"step":700,"loss":0.6427,"heldLoss":0.6381,"accuracy":0.6325,"heldAccuracy":0.675},{"step":800,"loss":0.6391,"heldLoss":0.6411,"accuracy":0.63,"heldAccuracy":0.65},{"step":900,"loss":0.6341,"heldLoss":0.6478,"accuracy":0.575,"heldAccuracy":0.54},{"step":1000,"loss":0.6176,"heldLoss":0.6401,"accuracy":0.5725,"heldAccuracy":0.545},{"step":1100,"loss":0.5846,"heldLoss":0.6102,"accuracy":0.665,"heldAccuracy":0.65},{"step":1200,"loss":0.5493,"heldLoss":0.5935,"accuracy":0.7175,"heldAccuracy":0.655},{"step":1300,"loss":0.504,"heldLoss":0.5755,"accuracy":0.735,"heldAccuracy":0.675},{"step":1400,"loss":0.464,"heldLoss":0.5396,"accuracy":0.745,"heldAccuracy":0.6575},{"step":1500,"loss":0.4325,"heldLoss":0.4981,"accuracy":0.7375,"heldAccuracy":0.6725},{"step":1600,"loss":0.3675,"heldLoss":0.4389,"accuracy":0.795,"heldAccuracy":0.7325},{"step":1700,"loss":0.3156,"heldLoss":0.3789,"accuracy":0.85,"heldAccuracy":0.805},{"step":1800,"loss":0.2585,"heldLoss":0.292,"accuracy":0.9225,"heldAccuracy":0.905},{"step":1900,"loss":0.2038,"heldLoss":0.2137,"accuracy":0.9375,"heldAccuracy":0.9425},{"step":2000,"loss":0.1647,"heldLoss":0.1708,"accuracy":0.9575,"heldAccuracy":0.965},{"step":2100,"loss":0.1367,"heldLoss":0.1293,"accuracy":0.9725,"heldAccuracy":0.9675},{"step":2200,"loss":0.1082,"heldLoss":0.1001,"accuracy":0.98,"heldAccuracy":0.9925},{"step":2300,"loss":0.08251,"heldLoss":0.08041,"accuracy":0.9825,"heldAccuracy":0.9925},{"step":2400,"loss":0.07725,"heldLoss":0.07065,"accuracy":0.9875,"heldAccuracy":0.9925},{"step":2500,"loss":0.06066,"heldLoss":0.06186,"accuracy":0.99,"heldAccuracy":0.9875},{"step":2600,"loss":0.05069,"heldLoss":0.04975,"accuracy":0.9925,"heldAccuracy":0.9975},{"step":2700,"loss":0.04524,"heldLoss":0.04793,"accuracy":0.9925,"heldAccuracy":0.9925},{"step":2800,"loss":0.04643,"heldLoss":0.03834,"accuracy":0.9925,"heldAccuracy":1},{"step":2900,"loss":0.03707,"heldLoss":0.0377,"accuracy":0.995,"heldAccuracy":0.995},{"step":3000,"loss":0.03422,"heldLoss":0.03573,"accuracy":0.9925,"heldAccuracy":0.995},{"step":3100,"loss":0.03881,"heldLoss":0.03431,"accuracy":0.9925,"heldAccuracy":0.9975},{"step":3200,"loss":0.04411,"heldLoss":0.04475,"accuracy":0.99,"heldAccuracy":0.99},{"step":3300,"loss":0.06082,"heldLoss":0.07797,"accuracy":0.9775,"heldAccuracy":0.975},{"step":3400,"loss":0.02585,"heldLoss":0.03311,"accuracy":0.995,"heldAccuracy":0.9925},{"step":3500,"loss":0.02628,"heldLoss":0.03746,"accuracy":0.995,"heldAccuracy":0.9925},{"step":3600,"loss":0.02624,"heldLoss":0.03689,"accuracy":0.995,"heldAccuracy":0.9925},{"step":3700,"loss":0.05409,"heldLoss":0.06817,"accuracy":0.9775,"heldAccuracy":0.98},{"step":3800,"loss":0.02251,"heldLoss":0.03469,"accuracy":0.9975,"heldAccuracy":0.9925},{"step":3900,"loss":0.01999,"heldLoss":0.02336,"accuracy":0.9975,"heldAccuracy":0.995},{"step":4000,"loss":0.02003,"heldLoss":0.02252,"accuracy":0.9975,"heldAccuracy":0.9975}]},"momentum":{"chosenLr":0.1,"atGridEdge":false,"tried":[{"lr":0.01,"firstValidAtTarget":null,"finalValidAccuracy":0.74,"finalHeldAccuracy":0.6925},{"lr":0.03,"firstValidAtTarget":2100,"finalValidAccuracy":0.9875,"finalHeldAccuracy":0.9925},{"lr":0.1,"firstValidAtTarget":1300,"finalValidAccuracy":0.995,"finalHeldAccuracy":0.995},{"lr":0.3,"firstValidAtTarget":null,"finalValidAccuracy":0.67,"finalHeldAccuracy":0.6875}],"firstHeldAtTarget":1300,"finalHeldAccuracy":0.995,"finalHeldLoss":0.02341,"curve":[{"step":0,"loss":0.7413,"heldLoss":0.7396,"accuracy":0.375,"heldAccuracy":0.355},{"step":100,"loss":0.6641,"heldLoss":0.6543,"accuracy":0.56,"heldAccuracy":0.555},{"step":200,"loss":0.6722,"heldLoss":0.6616,"accuracy":0.535,"heldAccuracy":0.555},{"step":300,"loss":0.6514,"heldLoss":0.6571,"accuracy":0.5625,"heldAccuracy":0.52},{"step":400,"loss":0.6617,"heldLoss":0.6859,"accuracy":0.605,"heldAccuracy":0.58},{"step":500,"loss":0.6095,"heldLoss":0.6413,"accuracy":0.61,"heldAccuracy":0.585},{"step":600,"loss":0.5662,"heldLoss":0.607,"accuracy":0.6825,"heldAccuracy":0.675},{"step":700,"loss":0.5248,"heldLoss":0.6224,"accuracy":0.7,"heldAccuracy":0.6575},{"step":800,"loss":0.3632,"heldLoss":0.4346,"accuracy":0.805,"heldAccuracy":0.745},{"step":900,"loss":0.2966,"heldLoss":0.3479,"accuracy":0.8525,"heldAccuracy":0.815},{"step":1000,"loss":0.1952,"heldLoss":0.2379,"accuracy":0.94,"heldAccuracy":0.9125},{"step":1100,"loss":0.2364,"heldLoss":0.2327,"accuracy":0.905,"heldAccuracy":0.895},{"step":1200,"loss":0.1695,"heldLoss":0.1784,"accuracy":0.9275,"heldAccuracy":0.925},{"step":1300,"loss":0.1397,"heldLoss":0.1271,"accuracy":0.955,"heldAccuracy":0.97},{"step":1400,"loss":0.1097,"heldLoss":0.1203,"accuracy":0.9625,"heldAccuracy":0.9725},{"step":1500,"loss":0.03697,"heldLoss":0.04588,"accuracy":0.9925,"heldAccuracy":0.9925},{"step":1600,"loss":0.03215,"heldLoss":0.04107,"accuracy":0.9875,"heldAccuracy":0.98},{"step":1700,"loss":0.02003,"heldLoss":0.02996,"accuracy":0.9975,"heldAccuracy":0.995},{"step":1800,"loss":0.033,"heldLoss":0.03546,"accuracy":0.99,"heldAccuracy":0.9925},{"step":1900,"loss":0.0129,"heldLoss":0.02728,"accuracy":1,"heldAccuracy":0.9925},{"step":2000,"loss":0.009112,"heldLoss":0.02526,"accuracy":1,"heldAccuracy":0.995},{"step":2100,"loss":0.007347,"heldLoss":0.01809,"accuracy":1,"heldAccuracy":0.995},{"step":2200,"loss":0.007178,"heldLoss":0.02022,"accuracy":1,"heldAccuracy":0.995},{"step":2300,"loss":0.004936,"heldLoss":0.02115,"accuracy":1,"heldAccuracy":0.995},{"step":2400,"loss":0.004188,"heldLoss":0.0207,"accuracy":1,"heldAccuracy":0.995},{"step":2500,"loss":0.003811,"heldLoss":0.01902,"accuracy":1,"heldAccuracy":0.995},{"step":2600,"loss":0.003311,"heldLoss":0.02253,"accuracy":1,"heldAccuracy":0.995},{"step":2700,"loss":0.002943,"heldLoss":0.02006,"accuracy":1,"heldAccuracy":0.995},{"step":2800,"loss":0.002772,"heldLoss":0.02173,"accuracy":1,"heldAccuracy":0.995},{"step":2900,"loss":0.002576,"heldLoss":0.02196,"accuracy":1,"heldAccuracy":0.995},{"step":3000,"loss":0.002375,"heldLoss":0.02222,"accuracy":1,"heldAccuracy":0.995},{"step":3100,"loss":0.002228,"heldLoss":0.02126,"accuracy":1,"heldAccuracy":0.995},{"step":3200,"loss":0.002107,"heldLoss":0.02223,"accuracy":1,"heldAccuracy":0.995},{"step":3300,"loss":0.00204,"heldLoss":0.02216,"accuracy":1,"heldAccuracy":0.995},{"step":3400,"loss":0.001887,"heldLoss":0.02302,"accuracy":1,"heldAccuracy":0.995},{"step":3500,"loss":0.001947,"heldLoss":0.02357,"accuracy":1,"heldAccuracy":0.995},{"step":3600,"loss":0.001913,"heldLoss":0.02499,"accuracy":1,"heldAccuracy":0.995},{"step":3700,"loss":0.00166,"heldLoss":0.02233,"accuracy":1,"heldAccuracy":0.995},{"step":3800,"loss":0.001612,"heldLoss":0.02243,"accuracy":1,"heldAccuracy":0.995},{"step":3900,"loss":0.001533,"heldLoss":0.02223,"accuracy":1,"heldAccuracy":0.995},{"step":4000,"loss":0.001482,"heldLoss":0.02341,"accuracy":1,"heldAccuracy":0.995}]},"adam":{"chosenLr":0.1,"atGridEdge":false,"tried":[{"lr":0.003,"firstValidAtTarget":null,"finalValidAccuracy":0.79,"finalHeldAccuracy":0.765},{"lr":0.01,"firstValidAtTarget":null,"finalValidAccuracy":0.7,"finalHeldAccuracy":0.655},{"lr":0.03,"firstValidAtTarget":1600,"finalValidAccuracy":0.9975,"finalHeldAccuracy":0.995},{"lr":0.1,"firstValidAtTarget":800,"finalValidAccuracy":0.9975,"finalHeldAccuracy":0.9925},{"lr":0.3,"firstValidAtTarget":null,"finalValidAccuracy":0.76,"finalHeldAccuracy":0.7975}],"firstHeldAtTarget":900,"finalHeldAccuracy":0.9925,"finalHeldLoss":0.04026,"curve":[{"step":0,"loss":0.7413,"heldLoss":0.7396,"accuracy":0.375,"heldAccuracy":0.355},{"step":100,"loss":0.6698,"heldLoss":0.6662,"accuracy":0.5875,"heldAccuracy":0.5725},{"step":200,"loss":0.6628,"heldLoss":0.6629,"accuracy":0.6775,"heldAccuracy":0.67},{"step":300,"loss":0.3813,"heldLoss":0.4128,"accuracy":0.8425,"heldAccuracy":0.815},{"step":400,"loss":0.2996,"heldLoss":0.3272,"accuracy":0.87,"heldAccuracy":0.8625},{"step":500,"loss":0.3259,"heldLoss":0.3274,"accuracy":0.865,"heldAccuracy":0.8775},{"step":600,"loss":0.2554,"heldLoss":0.3141,"accuracy":0.9025,"heldAccuracy":0.8875},{"step":700,"loss":0.2681,"heldLoss":0.2974,"accuracy":0.895,"heldAccuracy":0.885},{"step":800,"loss":0.1314,"heldLoss":0.1771,"accuracy":0.95,"heldAccuracy":0.9175},{"step":900,"loss":0.08619,"heldLoss":0.1437,"accuracy":0.9675,"heldAccuracy":0.95},{"step":1000,"loss":0.06487,"heldLoss":0.1117,"accuracy":0.9775,"heldAccuracy":0.955},{"step":1100,"loss":0.03382,"heldLoss":0.06417,"accuracy":0.9925,"heldAccuracy":0.98},{"step":1200,"loss":0.009106,"heldLoss":0.03173,"accuracy":1,"heldAccuracy":0.99},{"step":1300,"loss":0.08238,"heldLoss":0.07429,"accuracy":0.965,"heldAccuracy":0.97},{"step":1400,"loss":0.004245,"heldLoss":0.03112,"accuracy":1,"heldAccuracy":0.9925},{"step":1500,"loss":0.002706,"heldLoss":0.03136,"accuracy":1,"heldAccuracy":0.9925},{"step":1600,"loss":0.00219,"heldLoss":0.03072,"accuracy":1,"heldAccuracy":0.9925},{"step":1700,"loss":0.001851,"heldLoss":0.03199,"accuracy":1,"heldAccuracy":0.9925},{"step":1800,"loss":0.00159,"heldLoss":0.03118,"accuracy":1,"heldAccuracy":0.9925},{"step":1900,"loss":0.001399,"heldLoss":0.03218,"accuracy":1,"heldAccuracy":0.9925},{"step":2000,"loss":0.001228,"heldLoss":0.0315,"accuracy":1,"heldAccuracy":0.9925},{"step":2100,"loss":0.001101,"heldLoss":0.03377,"accuracy":1,"heldAccuracy":0.9925},{"step":2200,"loss":0.0009793,"heldLoss":0.03413,"accuracy":1,"heldAccuracy":0.9925},{"step":2300,"loss":0.0008875,"heldLoss":0.0344,"accuracy":1,"heldAccuracy":0.9925},{"step":2400,"loss":0.0008083,"heldLoss":0.03357,"accuracy":1,"heldAccuracy":0.9925},{"step":2500,"loss":0.0007414,"heldLoss":0.03504,"accuracy":1,"heldAccuracy":0.9925},{"step":2600,"loss":0.0006735,"heldLoss":0.0364,"accuracy":1,"heldAccuracy":0.9925},{"step":2700,"loss":0.0006215,"heldLoss":0.0347,"accuracy":1,"heldAccuracy":0.9925},{"step":2800,"loss":0.0005723,"heldLoss":0.03595,"accuracy":1,"heldAccuracy":0.9925},{"step":2900,"loss":0.0005264,"heldLoss":0.03605,"accuracy":1,"heldAccuracy":0.9925},{"step":3000,"loss":0.0004866,"heldLoss":0.03667,"accuracy":1,"heldAccuracy":0.9925},{"step":3100,"loss":0.0004522,"heldLoss":0.03705,"accuracy":1,"heldAccuracy":0.9925},{"step":3200,"loss":0.00042,"heldLoss":0.03678,"accuracy":1,"heldAccuracy":0.9925},{"step":3300,"loss":0.0003933,"heldLoss":0.03814,"accuracy":1,"heldAccuracy":0.9925},{"step":3400,"loss":0.000364,"heldLoss":0.03812,"accuracy":1,"heldAccuracy":0.9925},{"step":3500,"loss":0.0003398,"heldLoss":0.03911,"accuracy":1,"heldAccuracy":0.9925},{"step":3600,"loss":0.0003217,"heldLoss":0.04059,"accuracy":1,"heldAccuracy":0.9925},{"step":3700,"loss":0.000297,"heldLoss":0.03924,"accuracy":1,"heldAccuracy":0.9925},{"step":3800,"loss":0.000279,"heldLoss":0.03865,"accuracy":1,"heldAccuracy":0.9925},{"step":3900,"loss":0.0002606,"heldLoss":0.03957,"accuracy":1,"heldAccuracy":0.9925},{"step":4000,"loss":0.000244,"heldLoss":0.04026,"accuracy":1,"heldAccuracy":0.9925}]},"adamCosine":{"chosenLr":0.1,"schedule":"warmupCosine","warmup":200,"total":4000,"firstHeldAtTarget":800,"finalHeldAccuracy":0.99,"finalHeldLoss":0.0653,"curve":[{"step":0,"loss":0.7413,"heldLoss":0.7396,"accuracy":0.375,"heldAccuracy":0.355},{"step":100,"loss":0.6615,"heldLoss":0.6511,"accuracy":0.5925,"heldAccuracy":0.575},{"step":200,"loss":0.6919,"heldLoss":0.6849,"accuracy":0.58,"heldAccuracy":0.5775},{"step":300,"loss":0.6594,"heldLoss":0.6635,"accuracy":0.61,"heldAccuracy":0.605},{"step":400,"loss":0.5097,"heldLoss":0.5384,"accuracy":0.7775,"heldAccuracy":0.755},{"step":500,"loss":0.3763,"heldLoss":0.4146,"accuracy":0.8375,"heldAccuracy":0.82},{"step":600,"loss":0.3118,"heldLoss":0.4172,"accuracy":0.8625,"heldAccuracy":0.84},{"step":700,"loss":0.1501,"heldLoss":0.1928,"accuracy":0.95,"heldAccuracy":0.9175},{"step":800,"loss":0.03787,"heldLoss":0.0804,"accuracy":0.9925,"heldAccuracy":0.975},{"step":900,"loss":0.01876,"heldLoss":0.06975,"accuracy":0.9975,"heldAccuracy":0.9825},{"step":1000,"loss":0.02035,"heldLoss":0.06566,"accuracy":1,"heldAccuracy":0.9725},{"step":1100,"loss":0.07515,"heldLoss":0.04916,"accuracy":0.975,"heldAccuracy":0.985},{"step":1200,"loss":0.02314,"heldLoss":0.03668,"accuracy":0.995,"heldAccuracy":0.985},{"step":1300,"loss":0.03041,"heldLoss":0.06684,"accuracy":0.9925,"heldAccuracy":0.975},{"step":1400,"loss":0.04815,"heldLoss":0.08528,"accuracy":0.9825,"heldAccuracy":0.975},{"step":1500,"loss":0.0104,"heldLoss":0.0514,"accuracy":1,"heldAccuracy":0.9825},{"step":1600,"loss":0.004952,"heldLoss":0.08164,"accuracy":1,"heldAccuracy":0.98},{"step":1700,"loss":0.002718,"heldLoss":0.05915,"accuracy":1,"heldAccuracy":0.9875},{"step":1800,"loss":0.002064,"heldLoss":0.06129,"accuracy":1,"heldAccuracy":0.9875},{"step":1900,"loss":0.001691,"heldLoss":0.06189,"accuracy":1,"heldAccuracy":0.9875},{"step":2000,"loss":0.001442,"heldLoss":0.06197,"accuracy":1,"heldAccuracy":0.9875},{"step":2100,"loss":0.001309,"heldLoss":0.06318,"accuracy":1,"heldAccuracy":0.9875},{"step":2200,"loss":0.001196,"heldLoss":0.06222,"accuracy":1,"heldAccuracy":0.99},{"step":2300,"loss":0.001115,"heldLoss":0.0622,"accuracy":1,"heldAccuracy":0.99},{"step":2400,"loss":0.001047,"heldLoss":0.0625,"accuracy":1,"heldAccuracy":0.99},{"step":2500,"loss":0.0009802,"heldLoss":0.06363,"accuracy":1,"heldAccuracy":0.99},{"step":2600,"loss":0.0009332,"heldLoss":0.06371,"accuracy":1,"heldAccuracy":0.99},{"step":2700,"loss":0.0008986,"heldLoss":0.06325,"accuracy":1,"heldAccuracy":0.99},{"step":2800,"loss":0.0008602,"heldLoss":0.06365,"accuracy":1,"heldAccuracy":0.99},{"step":2900,"loss":0.0008298,"heldLoss":0.06411,"accuracy":1,"heldAccuracy":0.99},{"step":3000,"loss":0.0008057,"heldLoss":0.06427,"accuracy":1,"heldAccuracy":0.99},{"step":3100,"loss":0.0007903,"heldLoss":0.06455,"accuracy":1,"heldAccuracy":0.99},{"step":3200,"loss":0.0007711,"heldLoss":0.06475,"accuracy":1,"heldAccuracy":0.99},{"step":3300,"loss":0.0007593,"heldLoss":0.06497,"accuracy":1,"heldAccuracy":0.99},{"step":3400,"loss":0.0007488,"heldLoss":0.06511,"accuracy":1,"heldAccuracy":0.99},{"step":3500,"loss":0.000741,"heldLoss":0.06524,"accuracy":1,"heldAccuracy":0.99},{"step":3600,"loss":0.0007356,"heldLoss":0.06526,"accuracy":1,"heldAccuracy":0.99},{"step":3700,"loss":0.0007326,"heldLoss":0.06529,"accuracy":1,"heldAccuracy":0.99},{"step":3800,"loss":0.000731,"heldLoss":0.06529,"accuracy":1,"heldAccuracy":0.99},{"step":3900,"loss":0.0007304,"heldLoss":0.0653,"accuracy":1,"heldAccuracy":0.99},{"step":4000,"loss":0.0007303,"heldLoss":0.0653,"accuracy":1,"heldAccuracy":0.99}]}},"seedSpread":{"about":"Each arm at its chosen step, rerun with five (net seed, batch seed) pairs; data fixed. Updates to 95 percent held-out accuracy (null if never within 4000) and final held-out accuracy.","seeds":[[3,5],[13,15],[23,25],[33,35],[43,45]],"arms":{"sgd":{"rows":[{"firstHeldAtTarget":2000,"finalHeldAccuracy":0.9975},{"firstHeldAtTarget":1900,"finalHeldAccuracy":0.9975},{"firstHeldAtTarget":2400,"finalHeldAccuracy":0.9975},{"firstHeldAtTarget":1700,"finalHeldAccuracy":0.995},{"firstHeldAtTarget":2500,"finalHeldAccuracy":0.99}],"finalHeldAccuracy":{"min":0.99,"max":0.9975,"mean":0.9955}},"momentum":{"rows":[{"firstHeldAtTarget":1300,"finalHeldAccuracy":0.995},{"firstHeldAtTarget":1000,"finalHeldAccuracy":0.9975},{"firstHeldAtTarget":1100,"finalHeldAccuracy":0.995},{"firstHeldAtTarget":1100,"finalHeldAccuracy":0.995},{"firstHeldAtTarget":1100,"finalHeldAccuracy":0.9975}],"finalHeldAccuracy":{"min":0.995,"max":0.9975,"mean":0.996}},"adam":{"rows":[{"firstHeldAtTarget":900,"finalHeldAccuracy":0.9925},{"firstHeldAtTarget":400,"finalHeldAccuracy":0.9875},{"firstHeldAtTarget":800,"finalHeldAccuracy":0.9925},{"firstHeldAtTarget":900,"finalHeldAccuracy":0.99},{"firstHeldAtTarget":3100,"finalHeldAccuracy":0.9175}],"finalHeldAccuracy":{"min":0.9175,"max":0.9925,"mean":0.976}}}}},"transformer":{"about":"The attention lab's transformer, 400 updates from the same initial weights (seed 7) on the same batches (16 rows, data seed 11), with the same linear warmup (30 updates) and global-norm clipping at 1.0. What differs: AdamW is the lab trainer's own (moments kept in float32, decoupled weight decay 0.01 times lr on matrices and embeddings only); SGD and heavy ball (beta 0.9) use the pure steps of optimisers.mjs with no weight decay. Each arm's peak step is chosen from its grid by loss on 64 validation rows (seed 4242) after 400 updates; held-out curves are on the lab's 256 held-out rows. 400 updates is early: the copy skill appears near update 750 in the full run, so this compares the first stretch of training only. 'train' is the mean loss of the training batches since the previous point.","modelSeed":7,"dataSeed":11,"batch":16,"warmup":30,"clip":1,"steps":400,"every":50,"heldRows":256,"validSeed":4242,"validRows":64,"grids":{"adamw":[0.003,0.01,0.03],"sgd":[0.1,0.3,1,3],"momentum":[0.03,0.1,0.3,1]},"arms":{"adamw":{"chosenLr":0.01,"atGridEdge":false,"finalHeldOut":0.8456,"finalCopyAccuracy":0.3221,"tried":[{"lr":0.003,"validLoss":0.8514,"finalHeldOut":0.8583,"finalCopyAccuracy":0.2628},{"lr":0.01,"validLoss":0.8469,"finalHeldOut":0.8456,"finalCopyAccuracy":0.3221},{"lr":0.03,"validLoss":0.8611,"finalHeldOut":0.863,"finalCopyAccuracy":0.2589}],"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":50,"train":2.1519,"heldOut":1.1656,"copyAccuracy":0.2134,"localAccuracy":0.6078},{"step":100,"train":1.0402,"heldOut":0.9654,"copyAccuracy":0.1561,"localAccuracy":0.6744},{"step":150,"train":0.9162,"heldOut":0.9239,"copyAccuracy":0.2016,"localAccuracy":0.6907},{"step":200,"train":0.8889,"heldOut":0.8812,"copyAccuracy":0.2134,"localAccuracy":0.6976},{"step":250,"train":0.8445,"heldOut":0.878,"copyAccuracy":0.2826,"localAccuracy":0.6965},{"step":300,"train":0.8392,"heldOut":0.8436,"copyAccuracy":0.3399,"localAccuracy":0.7026},{"step":350,"train":0.8262,"heldOut":0.8397,"copyAccuracy":0.3261,"localAccuracy":0.6969},{"step":400,"train":0.8193,"heldOut":0.8456,"copyAccuracy":0.3221,"localAccuracy":0.6971}]},"sgd":{"chosenLr":1,"atGridEdge":false,"finalHeldOut":0.9608,"finalCopyAccuracy":0.2352,"tried":[{"lr":0.1,"validLoss":1.1458,"finalHeldOut":1.1636,"finalCopyAccuracy":0.1798},{"lr":0.3,"validLoss":1.0039,"finalHeldOut":1.0089,"finalCopyAccuracy":0.1996},{"lr":1,"validLoss":0.9573,"finalHeldOut":0.9608,"finalCopyAccuracy":0.2352},{"lr":3,"validLoss":1.3388,"finalHeldOut":1.3289,"finalCopyAccuracy":0.1107}],"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":50,"train":2.8223,"heldOut":1.9609,"copyAccuracy":0.1008,"localAccuracy":0.4333},{"step":100,"train":1.5726,"heldOut":1.3579,"copyAccuracy":0.1957,"localAccuracy":0.5646},{"step":150,"train":1.2481,"heldOut":1.1681,"copyAccuracy":0.1502,"localAccuracy":0.6109},{"step":200,"train":1.1572,"heldOut":1.1138,"copyAccuracy":0.164,"localAccuracy":0.6231},{"step":250,"train":1.094,"heldOut":1.1452,"copyAccuracy":0.1937,"localAccuracy":0.6231},{"step":300,"train":1.1114,"heldOut":1.0584,"copyAccuracy":0.1739,"localAccuracy":0.6474},{"step":350,"train":1.0204,"heldOut":1.0608,"copyAccuracy":0.1443,"localAccuracy":0.6486},{"step":400,"train":0.9866,"heldOut":0.9608,"copyAccuracy":0.2352,"localAccuracy":0.674}]},"momentum":{"chosenLr":0.3,"atGridEdge":false,"finalHeldOut":0.8925,"finalCopyAccuracy":0.2648,"tried":[{"lr":0.03,"validLoss":0.92,"finalHeldOut":0.9228,"finalCopyAccuracy":0.166},{"lr":0.1,"validLoss":0.8824,"finalHeldOut":0.8924,"finalCopyAccuracy":0.2154},{"lr":0.3,"validLoss":0.8802,"finalHeldOut":0.8925,"finalCopyAccuracy":0.2648},{"lr":1,"validLoss":0.8906,"finalHeldOut":0.9014,"finalCopyAccuracy":0.1818}],"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":50,"train":2.2085,"heldOut":1.2872,"copyAccuracy":0.2055,"localAccuracy":0.5623},{"step":100,"train":1.1828,"heldOut":1.1064,"copyAccuracy":0.1917,"localAccuracy":0.6195},{"step":150,"train":1.0185,"heldOut":1.0132,"copyAccuracy":0.1779,"localAccuracy":0.6587},{"step":200,"train":0.9693,"heldOut":0.9487,"copyAccuracy":0.1739,"localAccuracy":0.6857},{"step":250,"train":0.9053,"heldOut":0.9068,"copyAccuracy":0.1877,"localAccuracy":0.6884},{"step":300,"train":0.8998,"heldOut":0.9228,"copyAccuracy":0.1937,"localAccuracy":0.6811},{"step":350,"train":0.869,"heldOut":0.8734,"copyAccuracy":0.2332,"localAccuracy":0.6946},{"step":400,"train":0.849,"heldOut":0.8925,"copyAccuracy":0.2648,"localAccuracy":0.6873}]}},"reproducesModelJson":true,"modelJsonAt400":{"heldOut":0.8456,"copyAccuracy":0.3221}},"seconds":{"surfaces":0.6,"spiral":9.4,"transformer":80.3,"total":90}}