{"about":"Lesson L13, data. Computed by tools/school/fill_data.mjs from system/school/data.mjs and the attention lab's own model and trainer. Do not edit by hand. Losses are in nats (natural log); every run starts from the same model seed and is scored on the same held-out rows.","modelSha256":"67600098679443bf87c78f0f3ef1a1399ae51642b3e9608517105d7831219d5e","modelFile":"media/school/attention/model.json","recipe":{"modelSeed":7,"dataSeed":11,"steps":2000,"batch":16,"lr":0.01,"heldOutRows":256,"config":{"vocab":46,"ctx":24,"dModel":32,"nHeads":2,"nLayers":2,"dFF":64,"attnMode":"normal"}},"reproduce":{"about":"The unchanged recipe through this lesson's own batch code: model seed 7, data seed 11, 2000 steps, scored every 25 steps on the same 256 held-out rows, compared with the curve stored with the trained model after rounding to 4 decimals. The rewritten batch code draws the same rows as the corpus generator (the tests check bit for bit), so a match here shows every other run in this lesson differs only in its data. Bit-identical on one engine (V8 in Node); another engine was not tested.","points":81,"pointsEqual":81,"worstAbsDiff":0,"summary":{"endStep":2000,"heldOut":0.6874,"copyAccuracy":0.9921,"localAccuracy":0.7002,"bestHeldOut":0.6775,"bestStep":1850,"firstCopyStep":700,"diverged":false},"split":{"lookback":{"n":506,"accuracy":0.9921,"loss":0.056},"local":{"n":5638,"accuracy":0.7002,"loss":0.7441},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.0033},"whoEpisodes":{"n":399,"accuracy":0.99,"loss":0.0702},"guideAnswer":{"n":124,"accuracy":0.9677,"loss":0.2109}}},"noise":{"about":"Seed spread: the reference recipe with data seeds 12 and 13 (model seed unchanged), and the 5 percent look-back run with the same three data seeds. Three runs per recipe: a difference between recipes smaller than these ranges is not evidence of anything.","reference":{"heldOut":{"values":[0.6874,0.6844,0.6865],"mean":0.6861,"range":0.003},"copyAccuracy":{"values":[0.9921,0.9941,0.998],"mean":0.9947,"range":0.0059}},"lookback5":{"heldOut":{"values":[1.0663,1.0959,1.0968],"mean":1.0863,"range":0.0305},"copyAccuracy":{"values":[0.1917,0.17,0.1937],"mean":0.1851,"range":0.0237}},"runs":[{"name":"seed12","summary":{"endStep":2000,"heldOut":0.6844,"copyAccuracy":0.9941,"localAccuracy":0.7042,"bestHeldOut":0.6794,"bestStep":1700,"firstCopyStep":700,"diverged":false},"lookbackFed":65158,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.9941,"loss":0.0299},"local":{"n":5638,"accuracy":0.7042,"loss":0.7432},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.0051},"whoEpisodes":{"n":399,"accuracy":0.9925,"loss":0.0365},"guideAnswer":{"n":124,"accuracy":0.9758,"loss":0.1004}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.5737,"heldOut":0.9501,"copyAccuracy":0.1779,"localAccuracy":0.6859},{"step":200,"train":0.8889,"heldOut":0.896,"copyAccuracy":0.1502,"localAccuracy":0.696},{"step":300,"train":0.8529,"heldOut":0.8608,"copyAccuracy":0.334,"localAccuracy":0.6949},{"step":400,"train":0.8252,"heldOut":0.8265,"copyAccuracy":0.3814,"localAccuracy":0.696},{"step":500,"train":0.7943,"heldOut":0.7666,"copyAccuracy":0.6759,"localAccuracy":0.6983},{"step":600,"train":0.7461,"heldOut":0.7404,"copyAccuracy":0.747,"localAccuracy":0.6979},{"step":700,"train":0.7019,"heldOut":0.7081,"copyAccuracy":0.9822,"localAccuracy":0.6987},{"step":800,"train":0.6801,"heldOut":0.6975,"copyAccuracy":0.9921,"localAccuracy":0.6873},{"step":900,"train":0.6659,"heldOut":0.6911,"copyAccuracy":0.998,"localAccuracy":0.699},{"step":1000,"train":0.6758,"heldOut":0.7,"copyAccuracy":0.9802,"localAccuracy":0.7031},{"step":1100,"train":0.6735,"heldOut":0.6876,"copyAccuracy":0.9941,"localAccuracy":0.7056},{"step":1200,"train":0.6719,"heldOut":0.7076,"copyAccuracy":0.9842,"localAccuracy":0.7029},{"step":1300,"train":0.6677,"heldOut":0.6919,"copyAccuracy":0.9941,"localAccuracy":0.7066},{"step":1400,"train":0.6672,"heldOut":0.6928,"copyAccuracy":1,"localAccuracy":0.701},{"step":1500,"train":0.6583,"heldOut":0.6881,"copyAccuracy":0.996,"localAccuracy":0.6946},{"step":1600,"train":0.6692,"heldOut":0.6883,"copyAccuracy":0.9842,"localAccuracy":0.699},{"step":1700,"train":0.6631,"heldOut":0.6794,"copyAccuracy":1,"localAccuracy":0.7015},{"step":1800,"train":0.6585,"heldOut":0.6883,"copyAccuracy":0.9941,"localAccuracy":0.6979},{"step":1900,"train":0.6646,"heldOut":0.691,"copyAccuracy":0.996,"localAccuracy":0.7059},{"step":2000,"train":0.6657,"heldOut":0.6844,"copyAccuracy":0.9941,"localAccuracy":0.7042}]},{"name":"seed13","summary":{"endStep":2000,"heldOut":0.6865,"copyAccuracy":0.998,"localAccuracy":0.7027,"bestHeldOut":0.6816,"bestStep":1200,"firstCopyStep":800,"diverged":false},"lookbackFed":65535,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.998,"loss":0.0122},"local":{"n":5638,"accuracy":0.7027,"loss":0.7471},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.0053},"whoEpisodes":{"n":399,"accuracy":0.9975,"loss":0.0141},"guideAnswer":{"n":124,"accuracy":1,"loss":0.0045}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.5875,"heldOut":0.9956,"copyAccuracy":0.2134,"localAccuracy":0.668},{"step":200,"train":0.9011,"heldOut":0.8831,"copyAccuracy":0.1937,"localAccuracy":0.6919},{"step":300,"train":0.8572,"heldOut":0.8515,"copyAccuracy":0.3419,"localAccuracy":0.6962},{"step":400,"train":0.8115,"heldOut":0.8157,"copyAccuracy":0.4447,"localAccuracy":0.7026},{"step":500,"train":0.7562,"heldOut":0.7465,"copyAccuracy":0.747,"localAccuracy":0.704},{"step":600,"train":0.7328,"heldOut":0.7633,"copyAccuracy":0.6976,"localAccuracy":0.6992},{"step":700,"train":0.7229,"heldOut":0.7526,"copyAccuracy":0.8162,"localAccuracy":0.6923},{"step":800,"train":0.6868,"heldOut":0.6982,"copyAccuracy":0.9822,"localAccuracy":0.6978},{"step":900,"train":0.6835,"heldOut":0.7027,"copyAccuracy":0.9901,"localAccuracy":0.6978},{"step":1000,"train":0.6715,"heldOut":0.7019,"copyAccuracy":0.9921,"localAccuracy":0.7013},{"step":1100,"train":0.6696,"heldOut":0.7041,"copyAccuracy":0.9506,"localAccuracy":0.6999},{"step":1200,"train":0.6797,"heldOut":0.6816,"copyAccuracy":0.996,"localAccuracy":0.7047},{"step":1300,"train":0.663,"heldOut":0.6824,"copyAccuracy":0.998,"localAccuracy":0.7042},{"step":1400,"train":0.6665,"heldOut":0.7007,"copyAccuracy":0.9723,"localAccuracy":0.7073},{"step":1500,"train":0.6784,"heldOut":0.6849,"copyAccuracy":0.996,"localAccuracy":0.7024},{"step":1600,"train":0.6667,"heldOut":0.6963,"copyAccuracy":0.9783,"localAccuracy":0.6963},{"step":1700,"train":0.672,"heldOut":0.6953,"copyAccuracy":0.998,"localAccuracy":0.6914},{"step":1800,"train":0.6578,"heldOut":0.6823,"copyAccuracy":0.998,"localAccuracy":0.7001},{"step":1900,"train":0.6679,"heldOut":0.6854,"copyAccuracy":1,"localAccuracy":0.6988},{"step":2000,"train":0.6586,"heldOut":0.6865,"copyAccuracy":0.998,"localAccuracy":0.7027}]},{"name":"lb05seed12","summary":{"endStep":2000,"heldOut":1.0959,"copyAccuracy":0.17,"localAccuracy":0.6646,"bestHeldOut":1.0665,"bestStep":1500,"firstCopyStep":null,"diverged":false},"lookbackFed":6071,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.17,"loss":1.9854},"local":{"n":5638,"accuracy":0.6646,"loss":1.0161},"copyEpisodes":{"n":107,"accuracy":0.0841,"loss":2.4886},"whoEpisodes":{"n":399,"accuracy":0.193,"loss":1.8504},"guideAnswer":{"n":124,"accuracy":0.2903,"loss":1.1419}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.1931,"heldOut":1.4985,"copyAccuracy":0.166,"localAccuracy":0.5562},{"step":200,"train":0.5237,"heldOut":1.2945,"copyAccuracy":0.1937,"localAccuracy":0.6},{"step":300,"train":0.4954,"heldOut":1.197,"copyAccuracy":0.1759,"localAccuracy":0.635},{"step":400,"train":0.4909,"heldOut":1.1715,"copyAccuracy":0.1858,"localAccuracy":0.6529},{"step":500,"train":0.4833,"heldOut":1.1357,"copyAccuracy":0.1502,"localAccuracy":0.6607},{"step":600,"train":0.4807,"heldOut":1.131,"copyAccuracy":0.1976,"localAccuracy":0.6605},{"step":700,"train":0.4789,"heldOut":1.1084,"copyAccuracy":0.1324,"localAccuracy":0.6674},{"step":800,"train":0.4803,"heldOut":1.1666,"copyAccuracy":0.1798,"localAccuracy":0.6311},{"step":900,"train":0.4781,"heldOut":1.1236,"copyAccuracy":0.1937,"localAccuracy":0.6586},{"step":1000,"train":0.483,"heldOut":1.1136,"copyAccuracy":0.17,"localAccuracy":0.6618},{"step":1100,"train":0.4757,"heldOut":1.1165,"copyAccuracy":0.1739,"localAccuracy":0.6665},{"step":1200,"train":0.4748,"heldOut":1.0691,"copyAccuracy":0.1621,"localAccuracy":0.6619},{"step":1300,"train":0.4741,"heldOut":1.1056,"copyAccuracy":0.1443,"localAccuracy":0.6715},{"step":1400,"train":0.4815,"heldOut":1.0799,"copyAccuracy":0.1502,"localAccuracy":0.652},{"step":1500,"train":0.4674,"heldOut":1.0665,"copyAccuracy":0.1798,"localAccuracy":0.6752},{"step":1600,"train":0.4694,"heldOut":1.0749,"copyAccuracy":0.1561,"localAccuracy":0.6678},{"step":1700,"train":0.4816,"heldOut":1.0769,"copyAccuracy":0.1719,"localAccuracy":0.6685},{"step":1800,"train":0.4741,"heldOut":1.0825,"copyAccuracy":0.1522,"localAccuracy":0.6722},{"step":1900,"train":0.4631,"heldOut":1.0811,"copyAccuracy":0.166,"localAccuracy":0.6706},{"step":2000,"train":0.4635,"heldOut":1.0959,"copyAccuracy":0.17,"localAccuracy":0.6646}]},{"name":"lb05seed13","summary":{"endStep":2000,"heldOut":1.0968,"copyAccuracy":0.1937,"localAccuracy":0.6749,"bestHeldOut":1.0366,"bestStep":1600,"firstCopyStep":null,"diverged":false},"lookbackFed":6029,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.1937,"loss":1.9931},"local":{"n":5638,"accuracy":0.6749,"loss":1.0164},"copyEpisodes":{"n":107,"accuracy":0.1028,"loss":2.4354},"whoEpisodes":{"n":399,"accuracy":0.218,"loss":1.8745},"guideAnswer":{"n":124,"accuracy":0.3952,"loss":1.1025}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.1877,"heldOut":1.6748,"copyAccuracy":0.1601,"localAccuracy":0.5408},{"step":200,"train":0.5198,"heldOut":1.2763,"copyAccuracy":0.2036,"localAccuracy":0.6089},{"step":300,"train":0.4964,"heldOut":1.1581,"copyAccuracy":0.1759,"localAccuracy":0.6398},{"step":400,"train":0.4843,"heldOut":1.2252,"copyAccuracy":0.1601,"localAccuracy":0.6272},{"step":500,"train":0.4824,"heldOut":1.1579,"copyAccuracy":0.1621,"localAccuracy":0.6368},{"step":600,"train":0.4821,"heldOut":1.1323,"copyAccuracy":0.1897,"localAccuracy":0.6465},{"step":700,"train":0.4881,"heldOut":1.1426,"copyAccuracy":0.164,"localAccuracy":0.6344},{"step":800,"train":0.4757,"heldOut":1.1303,"copyAccuracy":0.1621,"localAccuracy":0.674},{"step":900,"train":0.4732,"heldOut":1.1351,"copyAccuracy":0.1976,"localAccuracy":0.6385},{"step":1000,"train":0.487,"heldOut":1.1025,"copyAccuracy":0.1462,"localAccuracy":0.6501},{"step":1100,"train":0.4788,"heldOut":1.0951,"copyAccuracy":0.1621,"localAccuracy":0.6729},{"step":1200,"train":0.4797,"heldOut":1.1204,"copyAccuracy":0.166,"localAccuracy":0.6462},{"step":1300,"train":0.4681,"heldOut":1.0842,"copyAccuracy":0.1739,"localAccuracy":0.6742},{"step":1400,"train":0.4795,"heldOut":1.0956,"copyAccuracy":0.1621,"localAccuracy":0.6696},{"step":1500,"train":0.4725,"heldOut":1.1206,"copyAccuracy":0.164,"localAccuracy":0.6531},{"step":1600,"train":0.4822,"heldOut":1.0366,"copyAccuracy":0.1838,"localAccuracy":0.6749},{"step":1700,"train":0.4751,"heldOut":1.0722,"copyAccuracy":0.1542,"localAccuracy":0.6564},{"step":1800,"train":0.4694,"heldOut":1.0527,"copyAccuracy":0.2036,"localAccuracy":0.6768},{"step":1900,"train":0.4775,"heldOut":1.0474,"copyAccuracy":0.1818,"localAccuracy":0.6768},{"step":2000,"train":0.4736,"heldOut":1.0968,"copyAccuracy":0.1937,"localAccuracy":0.6749}]}]},"lookback":{"about":"How much of a skill's data is needed. The corpus default writes 40 percent copy episodes, 40 percent who-did-it episodes (the two kinds whose targets need a look back) and 20 percent plain ones, so the default look-back share is 80 percent of episodes. Each run keeps copy and who episodes equal and fills the rest with plain episodes: 0, 1, 5, 20, 40 and 80 percent. lookbackFed counts the look-back target words the model was trained on out of targetsFed. Every run is scored on the same default-mix held-out rows (506 look-back targets), so the plain-heavy runs also face a mix they rarely trained on. One seed each except 5 and 80 percent (see noise). This shows the amount needed for this skill in this model at this budget; it does not give a general rule.","runs":[{"lookbackShare":0,"name":"lb0","summary":{"endStep":2000,"heldOut":8.2495,"copyAccuracy":0.0316,"localAccuracy":0.3187,"bestHeldOut":3.8414,"bestStep":0,"firstCopyStep":null,"diverged":false},"lookbackFed":0,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.0316,"loss":14.2448},"local":{"n":5638,"accuracy":0.3187,"loss":7.7114},"copyEpisodes":{"n":107,"accuracy":0.0374,"loss":15.4056},"whoEpisodes":{"n":399,"accuracy":0.0301,"loss":13.9336},"guideAnswer":{"n":124,"accuracy":0,"loss":10.2262}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.0472,"heldOut":5.006,"copyAccuracy":0.0158,"localAccuracy":0.3538},{"step":200,"train":0.4057,"heldOut":5.4921,"copyAccuracy":0.0277,"localAccuracy":0.3411},{"step":300,"train":0.4027,"heldOut":5.6143,"copyAccuracy":0.0119,"localAccuracy":0.331},{"step":400,"train":0.3985,"heldOut":5.6902,"copyAccuracy":0,"localAccuracy":0.3295},{"step":500,"train":0.4097,"heldOut":6.1842,"copyAccuracy":0.0494,"localAccuracy":0.3212},{"step":600,"train":0.4009,"heldOut":5.755,"copyAccuracy":0.0435,"localAccuracy":0.3411},{"step":700,"train":0.4003,"heldOut":6.156,"copyAccuracy":0.0099,"localAccuracy":0.3292},{"step":800,"train":0.3973,"heldOut":6.4511,"copyAccuracy":0.0119,"localAccuracy":0.3292},{"step":900,"train":0.397,"heldOut":6.5179,"copyAccuracy":0.002,"localAccuracy":0.3203},{"step":1000,"train":0.3964,"heldOut":6.591,"copyAccuracy":0.0514,"localAccuracy":0.3008},{"step":1100,"train":0.3966,"heldOut":6.5985,"copyAccuracy":0.0415,"localAccuracy":0.3031},{"step":1200,"train":0.3996,"heldOut":6.4025,"copyAccuracy":0.0277,"localAccuracy":0.342},{"step":1300,"train":0.4048,"heldOut":7.0466,"copyAccuracy":0.0514,"localAccuracy":0.2911},{"step":1400,"train":0.3956,"heldOut":7.173,"copyAccuracy":0.0474,"localAccuracy":0.3092},{"step":1500,"train":0.3962,"heldOut":7.2632,"copyAccuracy":0.0494,"localAccuracy":0.3194},{"step":1600,"train":0.3954,"heldOut":7.4706,"copyAccuracy":0.0099,"localAccuracy":0.32},{"step":1700,"train":0.3967,"heldOut":7.7793,"copyAccuracy":0.0059,"localAccuracy":0.2893},{"step":1800,"train":0.3964,"heldOut":7.5705,"copyAccuracy":0.0415,"localAccuracy":0.2895},{"step":1900,"train":0.4044,"heldOut":8.2465,"copyAccuracy":0.0455,"localAccuracy":0.3097},{"step":2000,"train":0.3939,"heldOut":8.2495,"copyAccuracy":0.0316,"localAccuracy":0.3187}]},{"lookbackShare":0.01,"name":"lb01","summary":{"endStep":2000,"heldOut":1.3365,"copyAccuracy":0.1798,"localAccuracy":0.6259,"bestHeldOut":1.2693,"bestStep":1600,"firstCopyStep":null,"diverged":false},"lookbackFed":1183,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.1798,"loss":2.1845},"local":{"n":5638,"accuracy":0.6259,"loss":1.2603},"copyEpisodes":{"n":107,"accuracy":0.0561,"loss":2.6434},"whoEpisodes":{"n":399,"accuracy":0.213,"loss":2.0614},"guideAnswer":{"n":124,"accuracy":0.3548,"loss":1.2786}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.092,"heldOut":2.3572,"copyAccuracy":0.0178,"localAccuracy":0.4824},{"step":200,"train":0.4576,"heldOut":1.8632,"copyAccuracy":0.1225,"localAccuracy":0.519},{"step":300,"train":0.4428,"heldOut":1.6813,"copyAccuracy":0.1344,"localAccuracy":0.5598},{"step":400,"train":0.4383,"heldOut":1.5746,"copyAccuracy":0.1561,"localAccuracy":0.5438},{"step":500,"train":0.4285,"heldOut":1.4771,"copyAccuracy":0.1739,"localAccuracy":0.5999},{"step":600,"train":0.4363,"heldOut":1.5119,"copyAccuracy":0.168,"localAccuracy":0.5809},{"step":700,"train":0.4319,"heldOut":1.4329,"copyAccuracy":0.1403,"localAccuracy":0.6289},{"step":800,"train":0.4224,"heldOut":1.4835,"copyAccuracy":0.1858,"localAccuracy":0.6041},{"step":900,"train":0.4235,"heldOut":1.5015,"copyAccuracy":0.164,"localAccuracy":0.5842},{"step":1000,"train":0.4209,"heldOut":1.6386,"copyAccuracy":0.1186,"localAccuracy":0.5763},{"step":1100,"train":0.4238,"heldOut":1.4215,"copyAccuracy":0.1719,"localAccuracy":0.6101},{"step":1200,"train":0.4184,"heldOut":1.2927,"copyAccuracy":0.168,"localAccuracy":0.6265},{"step":1300,"train":0.421,"heldOut":1.3256,"copyAccuracy":0.1561,"localAccuracy":0.635},{"step":1400,"train":0.4216,"heldOut":1.3846,"copyAccuracy":0.1779,"localAccuracy":0.6121},{"step":1500,"train":0.4264,"heldOut":1.2792,"copyAccuracy":0.1265,"localAccuracy":0.6424},{"step":1600,"train":0.4153,"heldOut":1.2693,"copyAccuracy":0.1522,"localAccuracy":0.6355},{"step":1700,"train":0.4187,"heldOut":1.3839,"copyAccuracy":0.1897,"localAccuracy":0.6527},{"step":1800,"train":0.4122,"heldOut":1.3132,"copyAccuracy":0.1462,"localAccuracy":0.643},{"step":1900,"train":0.4146,"heldOut":1.2966,"copyAccuracy":0.1502,"localAccuracy":0.6373},{"step":2000,"train":0.4191,"heldOut":1.3365,"copyAccuracy":0.1798,"localAccuracy":0.6259}]},{"lookbackShare":0.05,"name":"lb05","summary":{"endStep":2000,"heldOut":1.0663,"copyAccuracy":0.1917,"localAccuracy":0.6708,"bestHeldOut":1.0503,"bestStep":1400,"firstCopyStep":null,"diverged":false},"lookbackFed":6118,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.1917,"loss":1.9543},"local":{"n":5638,"accuracy":0.6708,"loss":0.9866},"copyEpisodes":{"n":107,"accuracy":0.0748,"loss":2.3876},"whoEpisodes":{"n":399,"accuracy":0.2231,"loss":1.8381},"guideAnswer":{"n":124,"accuracy":0.3952,"loss":1.142}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.1921,"heldOut":1.5158,"copyAccuracy":0.1462,"localAccuracy":0.5584},{"step":200,"train":0.5148,"heldOut":1.2935,"copyAccuracy":0.1858,"localAccuracy":0.5892},{"step":300,"train":0.5018,"heldOut":1.2694,"copyAccuracy":0.1225,"localAccuracy":0.6424},{"step":400,"train":0.5019,"heldOut":1.2057,"copyAccuracy":0.1561,"localAccuracy":0.6286},{"step":500,"train":0.4854,"heldOut":1.1299,"copyAccuracy":0.1542,"localAccuracy":0.6506},{"step":600,"train":0.4803,"heldOut":1.1303,"copyAccuracy":0.1621,"localAccuracy":0.6527},{"step":700,"train":0.4778,"heldOut":1.084,"copyAccuracy":0.1601,"localAccuracy":0.6438},{"step":800,"train":0.4845,"heldOut":1.1316,"copyAccuracy":0.17,"localAccuracy":0.6559},{"step":900,"train":0.4782,"heldOut":1.1335,"copyAccuracy":0.1601,"localAccuracy":0.6699},{"step":1000,"train":0.4788,"heldOut":1.1262,"copyAccuracy":0.1719,"localAccuracy":0.6584},{"step":1100,"train":0.4735,"heldOut":1.1414,"copyAccuracy":0.164,"localAccuracy":0.6591},{"step":1200,"train":0.4729,"heldOut":1.1068,"copyAccuracy":0.1798,"localAccuracy":0.6449},{"step":1300,"train":0.4803,"heldOut":1.1,"copyAccuracy":0.1601,"localAccuracy":0.6712},{"step":1400,"train":0.4775,"heldOut":1.0503,"copyAccuracy":0.1779,"localAccuracy":0.6678},{"step":1500,"train":0.4781,"heldOut":1.0581,"copyAccuracy":0.164,"localAccuracy":0.674},{"step":1600,"train":0.4681,"heldOut":1.0928,"copyAccuracy":0.1542,"localAccuracy":0.6584},{"step":1700,"train":0.4698,"heldOut":1.0829,"copyAccuracy":0.2075,"localAccuracy":0.6717},{"step":1800,"train":0.4767,"heldOut":1.0699,"copyAccuracy":0.1719,"localAccuracy":0.6639},{"step":1900,"train":0.4631,"heldOut":1.0936,"copyAccuracy":0.1838,"localAccuracy":0.6689},{"step":2000,"train":0.4744,"heldOut":1.0663,"copyAccuracy":0.1917,"localAccuracy":0.6708}]},{"lookbackShare":0.2,"name":"lb20","summary":{"endStep":2000,"heldOut":0.8648,"copyAccuracy":0.5593,"localAccuracy":0.6759,"bestHeldOut":0.8565,"bestStep":1400,"firstCopyStep":null,"diverged":false},"lookbackFed":22418,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.5593,"loss":1.1525},"local":{"n":5638,"accuracy":0.6759,"loss":0.8389},"copyEpisodes":{"n":107,"accuracy":0.0841,"loss":2.3049},"whoEpisodes":{"n":399,"accuracy":0.6867,"loss":0.8434},"guideAnswer":{"n":124,"accuracy":0.9435,"loss":0.2087}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.3703,"heldOut":1.1482,"copyAccuracy":0.1423,"localAccuracy":0.6405},{"step":200,"train":0.6652,"heldOut":1.0063,"copyAccuracy":0.1877,"localAccuracy":0.6696},{"step":300,"train":0.6487,"heldOut":0.9863,"copyAccuracy":0.1739,"localAccuracy":0.6674},{"step":400,"train":0.6335,"heldOut":0.9709,"copyAccuracy":0.1759,"localAccuracy":0.6758},{"step":500,"train":0.6316,"heldOut":0.974,"copyAccuracy":0.1581,"localAccuracy":0.6745},{"step":600,"train":0.63,"heldOut":0.9674,"copyAccuracy":0.166,"localAccuracy":0.6634},{"step":700,"train":0.6347,"heldOut":0.9455,"copyAccuracy":0.1798,"localAccuracy":0.6783},{"step":800,"train":0.6113,"heldOut":0.9312,"copyAccuracy":0.1561,"localAccuracy":0.6846},{"step":900,"train":0.6238,"heldOut":0.9227,"copyAccuracy":0.2273,"localAccuracy":0.6774},{"step":1000,"train":0.6155,"heldOut":0.9397,"copyAccuracy":0.2945,"localAccuracy":0.6871},{"step":1100,"train":0.5993,"heldOut":0.8855,"copyAccuracy":0.4348,"localAccuracy":0.677},{"step":1200,"train":0.6123,"heldOut":0.8814,"copyAccuracy":0.4842,"localAccuracy":0.6777},{"step":1300,"train":0.6019,"heldOut":0.8994,"copyAccuracy":0.5435,"localAccuracy":0.682},{"step":1400,"train":0.6015,"heldOut":0.8565,"copyAccuracy":0.5217,"localAccuracy":0.6797},{"step":1500,"train":0.5951,"heldOut":0.8757,"copyAccuracy":0.5553,"localAccuracy":0.671},{"step":1600,"train":0.5901,"heldOut":0.8604,"copyAccuracy":0.5415,"localAccuracy":0.6827},{"step":1700,"train":0.5884,"heldOut":0.8854,"copyAccuracy":0.5474,"localAccuracy":0.6763},{"step":1800,"train":0.5951,"heldOut":0.8573,"copyAccuracy":0.5613,"localAccuracy":0.6823},{"step":1900,"train":0.5863,"heldOut":0.858,"copyAccuracy":0.5435,"localAccuracy":0.6838},{"step":2000,"train":0.585,"heldOut":0.8648,"copyAccuracy":0.5593,"localAccuracy":0.6759}]},{"lookbackShare":0.4,"name":"lb40","summary":{"endStep":2000,"heldOut":0.7268,"copyAccuracy":0.9862,"localAccuracy":0.6806,"bestHeldOut":0.7181,"bestStep":1100,"firstCopyStep":900,"diverged":false},"lookbackFed":40120,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.9862,"loss":0.0646},"local":{"n":5638,"accuracy":0.6806,"loss":0.7863},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.0245},"whoEpisodes":{"n":399,"accuracy":0.9825,"loss":0.0754},"guideAnswer":{"n":124,"accuracy":0.9919,"loss":0.0362}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.5293,"heldOut":1.0402,"copyAccuracy":0.1957,"localAccuracy":0.6486},{"step":200,"train":0.7985,"heldOut":0.9255,"copyAccuracy":0.2194,"localAccuracy":0.6791},{"step":300,"train":0.7494,"heldOut":0.9063,"copyAccuracy":0.3221,"localAccuracy":0.6745},{"step":400,"train":0.7513,"heldOut":0.8657,"copyAccuracy":0.336,"localAccuracy":0.6862},{"step":500,"train":0.7367,"heldOut":0.8728,"copyAccuracy":0.3399,"localAccuracy":0.6838},{"step":600,"train":0.7377,"heldOut":0.9019,"copyAccuracy":0.3281,"localAccuracy":0.6756},{"step":700,"train":0.7265,"heldOut":0.8606,"copyAccuracy":0.3874,"localAccuracy":0.683},{"step":800,"train":0.7132,"heldOut":0.8191,"copyAccuracy":0.6601,"localAccuracy":0.6882},{"step":900,"train":0.6718,"heldOut":0.7544,"copyAccuracy":0.9763,"localAccuracy":0.6795},{"step":1000,"train":0.6499,"heldOut":0.7497,"copyAccuracy":0.9447,"localAccuracy":0.6829},{"step":1100,"train":0.6368,"heldOut":0.7181,"copyAccuracy":0.9881,"localAccuracy":0.6958},{"step":1200,"train":0.6407,"heldOut":0.7275,"copyAccuracy":0.9842,"localAccuracy":0.6848},{"step":1300,"train":0.6355,"heldOut":0.7286,"copyAccuracy":0.9901,"localAccuracy":0.6797},{"step":1400,"train":0.6374,"heldOut":0.7441,"copyAccuracy":0.9387,"localAccuracy":0.672},{"step":1500,"train":0.6307,"heldOut":0.7186,"copyAccuracy":0.9901,"localAccuracy":0.6843},{"step":1600,"train":0.6289,"heldOut":0.7397,"copyAccuracy":0.9723,"localAccuracy":0.688},{"step":1700,"train":0.6301,"heldOut":0.7188,"copyAccuracy":0.9941,"localAccuracy":0.6784},{"step":1800,"train":0.624,"heldOut":0.7256,"copyAccuracy":0.998,"localAccuracy":0.6777},{"step":1900,"train":0.6298,"heldOut":0.7328,"copyAccuracy":0.998,"localAccuracy":0.6781},{"step":2000,"train":0.6256,"heldOut":0.7268,"copyAccuracy":0.9862,"localAccuracy":0.6806}]},{"lookbackShare":0.8,"name":"reference","summary":{"endStep":2000,"heldOut":0.6874,"copyAccuracy":0.9921,"localAccuracy":0.7002,"bestHeldOut":0.6775,"bestStep":1850,"firstCopyStep":700,"diverged":false},"lookbackFed":65237,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.9921,"loss":0.056},"local":{"n":5638,"accuracy":0.7002,"loss":0.7441},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.0033},"whoEpisodes":{"n":399,"accuracy":0.99,"loss":0.0702},"guideAnswer":{"n":124,"accuracy":0.9677,"loss":0.2109}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":25,"train":2.9816,"heldOut":1.7252,"copyAccuracy":0.1522,"localAccuracy":0.5488},{"step":50,"train":1.3223,"heldOut":1.1656,"copyAccuracy":0.2134,"localAccuracy":0.6078},{"step":75,"train":1.0861,"heldOut":1.0454,"copyAccuracy":0.1976,"localAccuracy":0.6483},{"step":100,"train":0.9942,"heldOut":0.9654,"copyAccuracy":0.1561,"localAccuracy":0.6744},{"step":125,"train":0.9327,"heldOut":0.9599,"copyAccuracy":0.1798,"localAccuracy":0.6823},{"step":150,"train":0.8998,"heldOut":0.9239,"copyAccuracy":0.2016,"localAccuracy":0.6907},{"step":175,"train":0.8934,"heldOut":0.9101,"copyAccuracy":0.166,"localAccuracy":0.693},{"step":200,"train":0.8844,"heldOut":0.8812,"copyAccuracy":0.2134,"localAccuracy":0.6976},{"step":225,"train":0.8422,"heldOut":0.8643,"copyAccuracy":0.2431,"localAccuracy":0.7001},{"step":250,"train":0.8468,"heldOut":0.878,"copyAccuracy":0.2826,"localAccuracy":0.6965},{"step":275,"train":0.8422,"heldOut":0.8454,"copyAccuracy":0.3024,"localAccuracy":0.7008},{"step":300,"train":0.8362,"heldOut":0.8436,"copyAccuracy":0.3399,"localAccuracy":0.7026},{"step":325,"train":0.8286,"heldOut":0.8536,"copyAccuracy":0.3281,"localAccuracy":0.694},{"step":350,"train":0.8238,"heldOut":0.8397,"copyAccuracy":0.3261,"localAccuracy":0.6969},{"step":375,"train":0.8355,"heldOut":0.8523,"copyAccuracy":0.3142,"localAccuracy":0.7017},{"step":400,"train":0.8032,"heldOut":0.8456,"copyAccuracy":0.3221,"localAccuracy":0.6971},{"step":425,"train":0.841,"heldOut":0.8448,"copyAccuracy":0.3162,"localAccuracy":0.6908},{"step":450,"train":0.8254,"heldOut":0.832,"copyAccuracy":0.3458,"localAccuracy":0.6997},{"step":475,"train":0.8116,"heldOut":0.824,"copyAccuracy":0.4229,"localAccuracy":0.7077},{"step":500,"train":0.7949,"heldOut":0.8182,"copyAccuracy":0.4289,"localAccuracy":0.7057},{"step":525,"train":0.7991,"heldOut":0.8344,"copyAccuracy":0.4723,"localAccuracy":0.6836},{"step":550,"train":0.7913,"heldOut":0.8073,"copyAccuracy":0.5119,"localAccuracy":0.6919},{"step":575,"train":0.7913,"heldOut":0.8039,"copyAccuracy":0.5277,"localAccuracy":0.696},{"step":600,"train":0.7993,"heldOut":0.7937,"copyAccuracy":0.5751,"localAccuracy":0.702},{"step":625,"train":0.7871,"heldOut":0.7775,"copyAccuracy":0.6028,"localAccuracy":0.7015},{"step":650,"train":0.7516,"heldOut":0.7728,"copyAccuracy":0.6601,"localAccuracy":0.6995},{"step":675,"train":0.7525,"heldOut":0.7455,"copyAccuracy":0.7569,"localAccuracy":0.7022},{"step":700,"train":0.7395,"heldOut":0.7289,"copyAccuracy":0.9348,"localAccuracy":0.6898},{"step":725,"train":0.6994,"heldOut":0.7114,"copyAccuracy":0.9763,"localAccuracy":0.6994},{"step":750,"train":0.6929,"heldOut":0.715,"copyAccuracy":0.9684,"localAccuracy":0.6992},{"step":775,"train":0.6882,"heldOut":0.7218,"copyAccuracy":0.9684,"localAccuracy":0.702},{"step":800,"train":0.6876,"heldOut":0.7033,"copyAccuracy":0.9763,"localAccuracy":0.7045},{"step":825,"train":0.6905,"heldOut":0.7058,"copyAccuracy":0.9822,"localAccuracy":0.7001},{"step":850,"train":0.6853,"heldOut":0.7002,"copyAccuracy":0.9941,"localAccuracy":0.7018},{"step":875,"train":0.6766,"heldOut":0.7192,"copyAccuracy":0.9605,"localAccuracy":0.7081},{"step":900,"train":0.6752,"heldOut":0.7048,"copyAccuracy":0.9921,"localAccuracy":0.6999},{"step":925,"train":0.6712,"heldOut":0.6969,"copyAccuracy":0.9901,"localAccuracy":0.6994},{"step":950,"train":0.6799,"heldOut":0.7053,"copyAccuracy":0.9941,"localAccuracy":0.7026},{"step":975,"train":0.6835,"heldOut":0.7108,"copyAccuracy":0.9901,"localAccuracy":0.6958},{"step":1000,"train":0.6711,"heldOut":0.6981,"copyAccuracy":0.996,"localAccuracy":0.7013},{"step":1025,"train":0.6718,"heldOut":0.6933,"copyAccuracy":0.9941,"localAccuracy":0.7059},{"step":1050,"train":0.6785,"heldOut":0.6871,"copyAccuracy":1,"localAccuracy":0.7034},{"step":1075,"train":0.6685,"heldOut":0.6913,"copyAccuracy":0.9921,"localAccuracy":0.7088},{"step":1100,"train":0.663,"heldOut":0.6942,"copyAccuracy":0.998,"localAccuracy":0.699},{"step":1125,"train":0.6573,"heldOut":0.6939,"copyAccuracy":0.996,"localAccuracy":0.6937},{"step":1150,"train":0.677,"heldOut":0.7227,"copyAccuracy":0.9466,"localAccuracy":0.6962},{"step":1175,"train":0.6877,"heldOut":0.7006,"copyAccuracy":0.9822,"localAccuracy":0.6967},{"step":1200,"train":0.6915,"heldOut":0.6998,"copyAccuracy":0.9585,"localAccuracy":0.6999},{"step":1225,"train":0.6709,"heldOut":0.6928,"copyAccuracy":0.9881,"localAccuracy":0.702},{"step":1250,"train":0.6798,"heldOut":0.6964,"copyAccuracy":0.9941,"localAccuracy":0.701},{"step":1275,"train":0.6682,"heldOut":0.6867,"copyAccuracy":0.9941,"localAccuracy":0.7045},{"step":1300,"train":0.6593,"heldOut":0.6898,"copyAccuracy":0.9921,"localAccuracy":0.6971},{"step":1325,"train":0.6622,"heldOut":0.6923,"copyAccuracy":0.9941,"localAccuracy":0.6967},{"step":1350,"train":0.6736,"heldOut":0.6964,"copyAccuracy":0.996,"localAccuracy":0.7026},{"step":1375,"train":0.6611,"heldOut":0.6937,"copyAccuracy":0.996,"localAccuracy":0.6962},{"step":1400,"train":0.6596,"heldOut":0.6821,"copyAccuracy":0.998,"localAccuracy":0.699},{"step":1425,"train":0.6634,"heldOut":0.6795,"copyAccuracy":0.996,"localAccuracy":0.7027},{"step":1450,"train":0.6668,"heldOut":0.6836,"copyAccuracy":0.998,"localAccuracy":0.6969},{"step":1475,"train":0.6606,"heldOut":0.6828,"copyAccuracy":0.998,"localAccuracy":0.7068},{"step":1500,"train":0.665,"heldOut":0.6944,"copyAccuracy":0.9881,"localAccuracy":0.7004},{"step":1525,"train":0.6643,"heldOut":0.6946,"copyAccuracy":0.9881,"localAccuracy":0.6988},{"step":1550,"train":0.6637,"heldOut":0.6917,"copyAccuracy":0.9941,"localAccuracy":0.7061},{"step":1575,"train":0.6583,"heldOut":0.6895,"copyAccuracy":1,"localAccuracy":0.6981},{"step":1600,"train":0.66,"heldOut":0.6814,"copyAccuracy":0.9921,"localAccuracy":0.7054},{"step":1625,"train":0.6716,"heldOut":0.6936,"copyAccuracy":0.9941,"localAccuracy":0.6985},{"step":1650,"train":0.6793,"heldOut":0.7088,"copyAccuracy":0.9881,"localAccuracy":0.6967},{"step":1675,"train":0.6832,"heldOut":0.6867,"copyAccuracy":0.9901,"localAccuracy":0.702},{"step":1700,"train":0.6602,"heldOut":0.6875,"copyAccuracy":0.9881,"localAccuracy":0.6942},{"step":1725,"train":0.6564,"heldOut":0.6858,"copyAccuracy":0.998,"localAccuracy":0.7065},{"step":1750,"train":0.659,"heldOut":0.6884,"copyAccuracy":0.998,"localAccuracy":0.6946},{"step":1775,"train":0.6686,"heldOut":0.6938,"copyAccuracy":0.9802,"localAccuracy":0.7015},{"step":1800,"train":0.6638,"heldOut":0.6838,"copyAccuracy":0.9901,"localAccuracy":0.704},{"step":1825,"train":0.6577,"heldOut":0.6893,"copyAccuracy":0.998,"localAccuracy":0.6997},{"step":1850,"train":0.6666,"heldOut":0.6775,"copyAccuracy":0.998,"localAccuracy":0.6978},{"step":1875,"train":0.6578,"heldOut":0.6832,"copyAccuracy":0.998,"localAccuracy":0.7024},{"step":1900,"train":0.6651,"heldOut":0.6863,"copyAccuracy":0.996,"localAccuracy":0.6983},{"step":1925,"train":0.6695,"heldOut":0.6868,"copyAccuracy":0.9842,"localAccuracy":0.7065},{"step":1950,"train":0.6679,"heldOut":0.6829,"copyAccuracy":0.996,"localAccuracy":0.7034},{"step":1975,"train":0.6624,"heldOut":0.6799,"copyAccuracy":1,"localAccuracy":0.7056},{"step":2000,"train":0.6524,"heldOut":0.6874,"copyAccuracy":0.9921,"localAccuracy":0.7002}]}]},"noCopy":{"about":"Copy episodes removed, who-did-it episodes kept (copy 0, who 0.4, plain 0.2, renormalised). copyEpisodes is look-back accuracy on the held-out targets inside copy episodes (the thing after 'pylon put a cone on the'); whoEpisodes inside who episodes. The model still learns to look back, from who episodes; whether that carries over to a sentence frame it never saw is what copyEpisodes measures. One seed.","run":{"name":"noCopyEpisodes","summary":{"endStep":2000,"heldOut":3.2708,"copyAccuracy":0.7984,"localAccuracy":0.5277,"bestHeldOut":2.2423,"bestStep":200,"firstCopyStep":null,"diverged":false},"lookbackFed":90611,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.7984,"loss":2.1142},"local":{"n":5638,"accuracy":0.5277,"loss":3.3746},"copyEpisodes":{"n":107,"accuracy":0.0654,"loss":9.8537},"whoEpisodes":{"n":399,"accuracy":0.995,"loss":0.0386},"guideAnswer":{"n":124,"accuracy":0.9839,"loss":0.1113}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":1.6284,"heldOut":2.2908,"copyAccuracy":0.1581,"localAccuracy":0.5172},{"step":200,"train":0.9579,"heldOut":2.2423,"copyAccuracy":0.2648,"localAccuracy":0.5445},{"step":300,"train":0.8958,"heldOut":2.4498,"copyAccuracy":0.3162,"localAccuracy":0.5374},{"step":400,"train":0.875,"heldOut":2.2964,"copyAccuracy":0.3953,"localAccuracy":0.5639},{"step":500,"train":0.8187,"heldOut":2.2558,"copyAccuracy":0.6087,"localAccuracy":0.5264},{"step":600,"train":0.7587,"heldOut":2.3148,"copyAccuracy":0.7767,"localAccuracy":0.5259},{"step":700,"train":0.7118,"heldOut":2.6681,"copyAccuracy":0.7589,"localAccuracy":0.5257},{"step":800,"train":0.6997,"heldOut":2.6238,"copyAccuracy":0.7984,"localAccuracy":0.5229},{"step":900,"train":0.6965,"heldOut":2.5285,"copyAccuracy":0.7826,"localAccuracy":0.5355},{"step":1000,"train":0.6997,"heldOut":2.7379,"copyAccuracy":0.7945,"localAccuracy":0.5264},{"step":1100,"train":0.6956,"heldOut":2.9799,"copyAccuracy":0.8004,"localAccuracy":0.5158},{"step":1200,"train":0.6922,"heldOut":2.8666,"copyAccuracy":0.7885,"localAccuracy":0.527},{"step":1300,"train":0.6957,"heldOut":2.8646,"copyAccuracy":0.7964,"localAccuracy":0.5424},{"step":1400,"train":0.6928,"heldOut":2.9398,"copyAccuracy":0.7984,"localAccuracy":0.533},{"step":1500,"train":0.681,"heldOut":3.139,"copyAccuracy":0.8004,"localAccuracy":0.525},{"step":1600,"train":0.6888,"heldOut":2.9946,"copyAccuracy":0.7925,"localAccuracy":0.5403},{"step":1700,"train":0.6828,"heldOut":3.2303,"copyAccuracy":0.7925,"localAccuracy":0.5247},{"step":1800,"train":0.6822,"heldOut":2.9482,"copyAccuracy":0.7925,"localAccuracy":0.5248},{"step":1900,"train":0.6826,"heldOut":3.0374,"copyAccuracy":0.8024,"localAccuracy":0.5495},{"step":2000,"train":0.6777,"heldOut":3.2708,"copyAccuracy":0.7984,"localAccuracy":0.5277}]},"reference":{"lookback":{"n":506,"accuracy":0.9921,"loss":0.056},"local":{"n":5638,"accuracy":0.7002,"loss":0.7441},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.0033},"whoEpisodes":{"n":399,"accuracy":0.99,"loss":0.0702},"guideAnswer":{"n":124,"accuracy":0.9677,"loss":0.2109}}},"labelNoise":{"about":"A share of training target words replaced by a word drawn uniformly from all 46 (inputs untouched, a separate noise stream, so the clean rows are the reference run's rows). changedShare is the measured share of targets that changed (a replacement can draw the same word). idealLoss is what a model that learned the noisy targets perfectly would score on the clean held-out rows: the generator's own odds mixed with the uniform noise, computed exactly; the clean value is the generator floor. Uniform noise does not change which word is most likely, so a model that fitted the noisy targets perfectly would keep its accuracy and pay only in loss; the runs show local accuracy holding while the look-back skill, which needs a clean signal to form, arrives later or not at all within the budget. train is the loss on the noisy training targets. One seed each.","floor":0.6585,"runs":[{"share":0.1,"changedShare":0.0982,"idealLoss":0.7561,"name":"noise10","summary":{"endStep":2000,"heldOut":0.8506,"copyAccuracy":0.7609,"localAccuracy":0.7001,"bestHeldOut":0.8164,"bestStep":1900,"firstCopyStep":null,"diverged":false},"lookbackFed":65237,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.7609,"loss":0.6391},"local":{"n":5638,"accuracy":0.7001,"loss":0.8696},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.121},"whoEpisodes":{"n":399,"accuracy":0.6967,"loss":0.7781},"guideAnswer":{"n":124,"accuracy":1,"loss":0.1458}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":2.0755,"heldOut":1.1247,"copyAccuracy":0.1542,"localAccuracy":0.6596},{"step":200,"train":1.5236,"heldOut":0.9967,"copyAccuracy":0.17,"localAccuracy":0.6933},{"step":300,"train":1.4535,"heldOut":0.9683,"copyAccuracy":0.1818,"localAccuracy":0.7015},{"step":400,"train":1.4273,"heldOut":0.97,"copyAccuracy":0.1897,"localAccuracy":0.7011},{"step":500,"train":1.4411,"heldOut":0.961,"copyAccuracy":0.2134,"localAccuracy":0.7006},{"step":600,"train":1.4432,"heldOut":0.9423,"copyAccuracy":0.2668,"localAccuracy":0.6928},{"step":700,"train":1.4274,"heldOut":0.9254,"copyAccuracy":0.332,"localAccuracy":0.7049},{"step":800,"train":1.3636,"heldOut":0.8643,"copyAccuracy":0.664,"localAccuracy":0.6962},{"step":900,"train":1.3548,"heldOut":0.8478,"copyAccuracy":0.7273,"localAccuracy":0.7026},{"step":1000,"train":1.3134,"heldOut":0.827,"copyAccuracy":0.747,"localAccuracy":0.7038},{"step":1100,"train":1.3331,"heldOut":0.8344,"copyAccuracy":0.7589,"localAccuracy":0.7001},{"step":1200,"train":1.3298,"heldOut":0.8334,"copyAccuracy":0.7589,"localAccuracy":0.7029},{"step":1300,"train":1.2952,"heldOut":0.8348,"copyAccuracy":0.7589,"localAccuracy":0.6949},{"step":1400,"train":1.3238,"heldOut":0.823,"copyAccuracy":0.7628,"localAccuracy":0.6988},{"step":1500,"train":1.3214,"heldOut":0.8624,"copyAccuracy":0.7075,"localAccuracy":0.6987},{"step":1600,"train":1.3055,"heldOut":0.8378,"copyAccuracy":0.7194,"localAccuracy":0.7027},{"step":1700,"train":1.3124,"heldOut":0.8269,"copyAccuracy":0.7589,"localAccuracy":0.701},{"step":1800,"train":1.3056,"heldOut":0.8255,"copyAccuracy":0.7648,"localAccuracy":0.701},{"step":1900,"train":1.3077,"heldOut":0.8164,"copyAccuracy":0.7668,"localAccuracy":0.702},{"step":2000,"train":1.3115,"heldOut":0.8506,"copyAccuracy":0.7609,"localAccuracy":0.7001}]},{"share":0.3,"changedShare":0.2937,"idealLoss":0.9858,"name":"noise30","summary":{"endStep":2000,"heldOut":1.1606,"copyAccuracy":0.3281,"localAccuracy":0.6976,"bestHeldOut":1.1379,"bestStep":1700,"firstCopyStep":null,"diverged":false},"lookbackFed":65237,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.3281,"loss":1.9852},"local":{"n":5638,"accuracy":0.6976,"loss":1.0865},"copyEpisodes":{"n":107,"accuracy":0.0841,"loss":2.6014},"whoEpisodes":{"n":399,"accuracy":0.3935,"loss":1.82},"guideAnswer":{"n":124,"accuracy":0.9435,"loss":0.5979}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":100,"train":2.7253,"heldOut":1.4228,"copyAccuracy":0.1917,"localAccuracy":0.6362},{"step":200,"train":2.3693,"heldOut":1.2975,"copyAccuracy":0.1561,"localAccuracy":0.6852},{"step":300,"train":2.314,"heldOut":1.202,"copyAccuracy":0.1621,"localAccuracy":0.6985},{"step":400,"train":2.2575,"heldOut":1.1774,"copyAccuracy":0.1917,"localAccuracy":0.7011},{"step":500,"train":2.2848,"heldOut":1.2131,"copyAccuracy":0.1818,"localAccuracy":0.6995},{"step":600,"train":2.2682,"heldOut":1.2252,"copyAccuracy":0.2134,"localAccuracy":0.6903},{"step":700,"train":2.2567,"heldOut":1.2058,"copyAccuracy":0.1759,"localAccuracy":0.6962},{"step":800,"train":2.2599,"heldOut":1.1884,"copyAccuracy":0.1621,"localAccuracy":0.6981},{"step":900,"train":2.2701,"heldOut":1.1863,"copyAccuracy":0.1779,"localAccuracy":0.7002},{"step":1000,"train":2.2484,"heldOut":1.1921,"copyAccuracy":0.1818,"localAccuracy":0.7022},{"step":1100,"train":2.2575,"heldOut":1.193,"copyAccuracy":0.17,"localAccuracy":0.7006},{"step":1200,"train":2.2624,"heldOut":1.1755,"copyAccuracy":0.1858,"localAccuracy":0.6962},{"step":1300,"train":2.257,"heldOut":1.1658,"copyAccuracy":0.2095,"localAccuracy":0.7022},{"step":1400,"train":2.2353,"heldOut":1.1606,"copyAccuracy":0.2016,"localAccuracy":0.7011},{"step":1500,"train":2.2456,"heldOut":1.1476,"copyAccuracy":0.247,"localAccuracy":0.7011},{"step":1600,"train":2.2403,"heldOut":1.1558,"copyAccuracy":0.3083,"localAccuracy":0.7049},{"step":1700,"train":2.2408,"heldOut":1.1379,"copyAccuracy":0.336,"localAccuracy":0.7049},{"step":1800,"train":2.2262,"heldOut":1.1656,"copyAccuracy":0.334,"localAccuracy":0.7052},{"step":1900,"train":2.2394,"heldOut":1.1445,"copyAccuracy":0.3379,"localAccuracy":0.705},{"step":2000,"train":2.2222,"heldOut":1.1606,"copyAccuracy":0.3281,"localAccuracy":0.6976}]}]},"duplication":{"about":"The same 2000 steps of 16 rows, so the same number of training tokens, drawn from a fixed pool of 64, 256 or 1,024 rows served in shuffled passes, against fresh rows every step (the reference). poolLoss is the loss on the pool itself, scored every 100 steps (what the model was trained on); heldOut is on the untouched rows; the gap between them is memorising. passes: how many times each pool row was seen. The held-out rows are new rows of the same grammar, so a short sentence in them may also sit in a pool. One seed each.","fresh":{"name":"reference","summary":{"endStep":2000,"heldOut":0.6874,"copyAccuracy":0.9921,"localAccuracy":0.7002,"bestHeldOut":0.6775,"bestStep":1850,"firstCopyStep":700,"diverged":false},"lookbackFed":65237,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.9921,"loss":0.056},"local":{"n":5638,"accuracy":0.7002,"loss":0.7441},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.0033},"whoEpisodes":{"n":399,"accuracy":0.99,"loss":0.0702},"guideAnswer":{"n":124,"accuracy":0.9677,"loss":0.2109}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204},{"step":25,"train":2.9816,"heldOut":1.7252,"copyAccuracy":0.1522,"localAccuracy":0.5488},{"step":50,"train":1.3223,"heldOut":1.1656,"copyAccuracy":0.2134,"localAccuracy":0.6078},{"step":75,"train":1.0861,"heldOut":1.0454,"copyAccuracy":0.1976,"localAccuracy":0.6483},{"step":100,"train":0.9942,"heldOut":0.9654,"copyAccuracy":0.1561,"localAccuracy":0.6744},{"step":125,"train":0.9327,"heldOut":0.9599,"copyAccuracy":0.1798,"localAccuracy":0.6823},{"step":150,"train":0.8998,"heldOut":0.9239,"copyAccuracy":0.2016,"localAccuracy":0.6907},{"step":175,"train":0.8934,"heldOut":0.9101,"copyAccuracy":0.166,"localAccuracy":0.693},{"step":200,"train":0.8844,"heldOut":0.8812,"copyAccuracy":0.2134,"localAccuracy":0.6976},{"step":225,"train":0.8422,"heldOut":0.8643,"copyAccuracy":0.2431,"localAccuracy":0.7001},{"step":250,"train":0.8468,"heldOut":0.878,"copyAccuracy":0.2826,"localAccuracy":0.6965},{"step":275,"train":0.8422,"heldOut":0.8454,"copyAccuracy":0.3024,"localAccuracy":0.7008},{"step":300,"train":0.8362,"heldOut":0.8436,"copyAccuracy":0.3399,"localAccuracy":0.7026},{"step":325,"train":0.8286,"heldOut":0.8536,"copyAccuracy":0.3281,"localAccuracy":0.694},{"step":350,"train":0.8238,"heldOut":0.8397,"copyAccuracy":0.3261,"localAccuracy":0.6969},{"step":375,"train":0.8355,"heldOut":0.8523,"copyAccuracy":0.3142,"localAccuracy":0.7017},{"step":400,"train":0.8032,"heldOut":0.8456,"copyAccuracy":0.3221,"localAccuracy":0.6971},{"step":425,"train":0.841,"heldOut":0.8448,"copyAccuracy":0.3162,"localAccuracy":0.6908},{"step":450,"train":0.8254,"heldOut":0.832,"copyAccuracy":0.3458,"localAccuracy":0.6997},{"step":475,"train":0.8116,"heldOut":0.824,"copyAccuracy":0.4229,"localAccuracy":0.7077},{"step":500,"train":0.7949,"heldOut":0.8182,"copyAccuracy":0.4289,"localAccuracy":0.7057},{"step":525,"train":0.7991,"heldOut":0.8344,"copyAccuracy":0.4723,"localAccuracy":0.6836},{"step":550,"train":0.7913,"heldOut":0.8073,"copyAccuracy":0.5119,"localAccuracy":0.6919},{"step":575,"train":0.7913,"heldOut":0.8039,"copyAccuracy":0.5277,"localAccuracy":0.696},{"step":600,"train":0.7993,"heldOut":0.7937,"copyAccuracy":0.5751,"localAccuracy":0.702},{"step":625,"train":0.7871,"heldOut":0.7775,"copyAccuracy":0.6028,"localAccuracy":0.7015},{"step":650,"train":0.7516,"heldOut":0.7728,"copyAccuracy":0.6601,"localAccuracy":0.6995},{"step":675,"train":0.7525,"heldOut":0.7455,"copyAccuracy":0.7569,"localAccuracy":0.7022},{"step":700,"train":0.7395,"heldOut":0.7289,"copyAccuracy":0.9348,"localAccuracy":0.6898},{"step":725,"train":0.6994,"heldOut":0.7114,"copyAccuracy":0.9763,"localAccuracy":0.6994},{"step":750,"train":0.6929,"heldOut":0.715,"copyAccuracy":0.9684,"localAccuracy":0.6992},{"step":775,"train":0.6882,"heldOut":0.7218,"copyAccuracy":0.9684,"localAccuracy":0.702},{"step":800,"train":0.6876,"heldOut":0.7033,"copyAccuracy":0.9763,"localAccuracy":0.7045},{"step":825,"train":0.6905,"heldOut":0.7058,"copyAccuracy":0.9822,"localAccuracy":0.7001},{"step":850,"train":0.6853,"heldOut":0.7002,"copyAccuracy":0.9941,"localAccuracy":0.7018},{"step":875,"train":0.6766,"heldOut":0.7192,"copyAccuracy":0.9605,"localAccuracy":0.7081},{"step":900,"train":0.6752,"heldOut":0.7048,"copyAccuracy":0.9921,"localAccuracy":0.6999},{"step":925,"train":0.6712,"heldOut":0.6969,"copyAccuracy":0.9901,"localAccuracy":0.6994},{"step":950,"train":0.6799,"heldOut":0.7053,"copyAccuracy":0.9941,"localAccuracy":0.7026},{"step":975,"train":0.6835,"heldOut":0.7108,"copyAccuracy":0.9901,"localAccuracy":0.6958},{"step":1000,"train":0.6711,"heldOut":0.6981,"copyAccuracy":0.996,"localAccuracy":0.7013},{"step":1025,"train":0.6718,"heldOut":0.6933,"copyAccuracy":0.9941,"localAccuracy":0.7059},{"step":1050,"train":0.6785,"heldOut":0.6871,"copyAccuracy":1,"localAccuracy":0.7034},{"step":1075,"train":0.6685,"heldOut":0.6913,"copyAccuracy":0.9921,"localAccuracy":0.7088},{"step":1100,"train":0.663,"heldOut":0.6942,"copyAccuracy":0.998,"localAccuracy":0.699},{"step":1125,"train":0.6573,"heldOut":0.6939,"copyAccuracy":0.996,"localAccuracy":0.6937},{"step":1150,"train":0.677,"heldOut":0.7227,"copyAccuracy":0.9466,"localAccuracy":0.6962},{"step":1175,"train":0.6877,"heldOut":0.7006,"copyAccuracy":0.9822,"localAccuracy":0.6967},{"step":1200,"train":0.6915,"heldOut":0.6998,"copyAccuracy":0.9585,"localAccuracy":0.6999},{"step":1225,"train":0.6709,"heldOut":0.6928,"copyAccuracy":0.9881,"localAccuracy":0.702},{"step":1250,"train":0.6798,"heldOut":0.6964,"copyAccuracy":0.9941,"localAccuracy":0.701},{"step":1275,"train":0.6682,"heldOut":0.6867,"copyAccuracy":0.9941,"localAccuracy":0.7045},{"step":1300,"train":0.6593,"heldOut":0.6898,"copyAccuracy":0.9921,"localAccuracy":0.6971},{"step":1325,"train":0.6622,"heldOut":0.6923,"copyAccuracy":0.9941,"localAccuracy":0.6967},{"step":1350,"train":0.6736,"heldOut":0.6964,"copyAccuracy":0.996,"localAccuracy":0.7026},{"step":1375,"train":0.6611,"heldOut":0.6937,"copyAccuracy":0.996,"localAccuracy":0.6962},{"step":1400,"train":0.6596,"heldOut":0.6821,"copyAccuracy":0.998,"localAccuracy":0.699},{"step":1425,"train":0.6634,"heldOut":0.6795,"copyAccuracy":0.996,"localAccuracy":0.7027},{"step":1450,"train":0.6668,"heldOut":0.6836,"copyAccuracy":0.998,"localAccuracy":0.6969},{"step":1475,"train":0.6606,"heldOut":0.6828,"copyAccuracy":0.998,"localAccuracy":0.7068},{"step":1500,"train":0.665,"heldOut":0.6944,"copyAccuracy":0.9881,"localAccuracy":0.7004},{"step":1525,"train":0.6643,"heldOut":0.6946,"copyAccuracy":0.9881,"localAccuracy":0.6988},{"step":1550,"train":0.6637,"heldOut":0.6917,"copyAccuracy":0.9941,"localAccuracy":0.7061},{"step":1575,"train":0.6583,"heldOut":0.6895,"copyAccuracy":1,"localAccuracy":0.6981},{"step":1600,"train":0.66,"heldOut":0.6814,"copyAccuracy":0.9921,"localAccuracy":0.7054},{"step":1625,"train":0.6716,"heldOut":0.6936,"copyAccuracy":0.9941,"localAccuracy":0.6985},{"step":1650,"train":0.6793,"heldOut":0.7088,"copyAccuracy":0.9881,"localAccuracy":0.6967},{"step":1675,"train":0.6832,"heldOut":0.6867,"copyAccuracy":0.9901,"localAccuracy":0.702},{"step":1700,"train":0.6602,"heldOut":0.6875,"copyAccuracy":0.9881,"localAccuracy":0.6942},{"step":1725,"train":0.6564,"heldOut":0.6858,"copyAccuracy":0.998,"localAccuracy":0.7065},{"step":1750,"train":0.659,"heldOut":0.6884,"copyAccuracy":0.998,"localAccuracy":0.6946},{"step":1775,"train":0.6686,"heldOut":0.6938,"copyAccuracy":0.9802,"localAccuracy":0.7015},{"step":1800,"train":0.6638,"heldOut":0.6838,"copyAccuracy":0.9901,"localAccuracy":0.704},{"step":1825,"train":0.6577,"heldOut":0.6893,"copyAccuracy":0.998,"localAccuracy":0.6997},{"step":1850,"train":0.6666,"heldOut":0.6775,"copyAccuracy":0.998,"localAccuracy":0.6978},{"step":1875,"train":0.6578,"heldOut":0.6832,"copyAccuracy":0.998,"localAccuracy":0.7024},{"step":1900,"train":0.6651,"heldOut":0.6863,"copyAccuracy":0.996,"localAccuracy":0.6983},{"step":1925,"train":0.6695,"heldOut":0.6868,"copyAccuracy":0.9842,"localAccuracy":0.7065},{"step":1950,"train":0.6679,"heldOut":0.6829,"copyAccuracy":0.996,"localAccuracy":0.7034},{"step":1975,"train":0.6624,"heldOut":0.6799,"copyAccuracy":1,"localAccuracy":0.7056},{"step":2000,"train":0.6524,"heldOut":0.6874,"copyAccuracy":0.9921,"localAccuracy":0.7002}]},"runs":[{"rows":64,"passes":500,"name":"pool64","summary":{"endStep":2000,"heldOut":3.0157,"copyAccuracy":0.4783,"localAccuracy":0.611,"bestHeldOut":1.2955,"bestStep":100,"firstCopyStep":null,"diverged":false},"lookbackFed":62000,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.4783,"loss":3.9846},"local":{"n":5638,"accuracy":0.611,"loss":2.9288},"copyEpisodes":{"n":107,"accuracy":0.3551,"loss":5.914},"whoEpisodes":{"n":399,"accuracy":0.5113,"loss":3.4672},"guideAnswer":{"n":124,"accuracy":0.6855,"loss":1.8775}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204,"poolLoss":3.8434},{"step":100,"train":1.4238,"heldOut":1.2955,"copyAccuracy":0.247,"localAccuracy":0.6215,"poolLoss":0.5864},{"step":200,"train":0.4029,"heldOut":2.0923,"copyAccuracy":0.3814,"localAccuracy":0.6057,"poolLoss":0.2253},{"step":300,"train":0.2163,"heldOut":2.5725,"copyAccuracy":0.4012,"localAccuracy":0.6036,"poolLoss":0.1548},{"step":400,"train":0.1616,"heldOut":2.7172,"copyAccuracy":0.3755,"localAccuracy":0.5938,"poolLoss":0.1975},{"step":500,"train":0.1854,"heldOut":2.7809,"copyAccuracy":0.415,"localAccuracy":0.5947,"poolLoss":0.1638},{"step":600,"train":0.162,"heldOut":3.0348,"copyAccuracy":0.419,"localAccuracy":0.6078,"poolLoss":0.1297},{"step":700,"train":0.1372,"heldOut":2.9892,"copyAccuracy":0.4249,"localAccuracy":0.6093,"poolLoss":0.1308},{"step":800,"train":0.1353,"heldOut":2.995,"copyAccuracy":0.417,"localAccuracy":0.6096,"poolLoss":0.1271},{"step":900,"train":0.1992,"heldOut":2.7699,"copyAccuracy":0.417,"localAccuracy":0.5952,"poolLoss":0.1584},{"step":1000,"train":0.1455,"heldOut":2.996,"copyAccuracy":0.4466,"localAccuracy":0.6091,"poolLoss":0.1278},{"step":1100,"train":0.1343,"heldOut":2.9898,"copyAccuracy":0.4625,"localAccuracy":0.608,"poolLoss":0.1286},{"step":1200,"train":0.1334,"heldOut":3.0163,"copyAccuracy":0.4249,"localAccuracy":0.607,"poolLoss":0.1272},{"step":1300,"train":0.1832,"heldOut":2.9104,"copyAccuracy":0.4684,"localAccuracy":0.6013,"poolLoss":0.1435},{"step":1400,"train":0.1352,"heldOut":3.0528,"copyAccuracy":0.4822,"localAccuracy":0.611,"poolLoss":0.1274},{"step":1500,"train":0.1323,"heldOut":3.0265,"copyAccuracy":0.4625,"localAccuracy":0.6091,"poolLoss":0.1266},{"step":1600,"train":0.1315,"heldOut":3.037,"copyAccuracy":0.4625,"localAccuracy":0.6132,"poolLoss":0.1266},{"step":1700,"train":0.1575,"heldOut":2.4835,"copyAccuracy":0.3972,"localAccuracy":0.5889,"poolLoss":0.282},{"step":1800,"train":0.1611,"heldOut":3.0811,"copyAccuracy":0.4704,"localAccuracy":0.6029,"poolLoss":0.1267},{"step":1900,"train":0.1313,"heldOut":3.0368,"copyAccuracy":0.4862,"localAccuracy":0.6075,"poolLoss":0.1265},{"step":2000,"train":0.1309,"heldOut":3.0157,"copyAccuracy":0.4783,"localAccuracy":0.611,"poolLoss":0.1276}]},{"rows":256,"passes":125,"name":"pool256","summary":{"endStep":2000,"heldOut":1.7508,"copyAccuracy":0.7747,"localAccuracy":0.6634,"bestHeldOut":0.8826,"bestStep":400,"firstCopyStep":null,"diverged":false},"lookbackFed":63250,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.7747,"loss":1.209},"local":{"n":5638,"accuracy":0.6634,"loss":1.7995},"copyEpisodes":{"n":107,"accuracy":0.7664,"loss":1.1623},"whoEpisodes":{"n":399,"accuracy":0.7769,"loss":1.2215},"guideAnswer":{"n":124,"accuracy":0.7258,"loss":1.729}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204,"poolLoss":3.8439},{"step":100,"train":1.5592,"heldOut":1.0096,"copyAccuracy":0.17,"localAccuracy":0.6621,"poolLoss":0.9054},{"step":200,"train":0.8533,"heldOut":0.9251,"copyAccuracy":0.1957,"localAccuracy":0.6862,"poolLoss":0.7878},{"step":300,"train":0.7758,"heldOut":0.9146,"copyAccuracy":0.5534,"localAccuracy":0.6806,"poolLoss":0.7056},{"step":400,"train":0.7006,"heldOut":0.8826,"copyAccuracy":0.7134,"localAccuracy":0.6806,"poolLoss":0.6263},{"step":500,"train":0.6196,"heldOut":0.9411,"copyAccuracy":0.7688,"localAccuracy":0.6726,"poolLoss":0.5668},{"step":600,"train":0.5662,"heldOut":0.9902,"copyAccuracy":0.8083,"localAccuracy":0.6722,"poolLoss":0.5004},{"step":700,"train":0.5189,"heldOut":1.0659,"copyAccuracy":0.7826,"localAccuracy":0.6799,"poolLoss":0.4607},{"step":800,"train":0.4821,"heldOut":1.1246,"copyAccuracy":0.8478,"localAccuracy":0.6729,"poolLoss":0.4338},{"step":900,"train":0.4447,"heldOut":1.2663,"copyAccuracy":0.8636,"localAccuracy":0.6563,"poolLoss":0.3925},{"step":1000,"train":0.4151,"heldOut":1.3061,"copyAccuracy":0.8063,"localAccuracy":0.6616,"poolLoss":0.3432},{"step":1100,"train":0.3847,"heldOut":1.3999,"copyAccuracy":0.8142,"localAccuracy":0.6648,"poolLoss":0.3368},{"step":1200,"train":0.3722,"heldOut":1.4401,"copyAccuracy":0.836,"localAccuracy":0.6531,"poolLoss":0.3148},{"step":1300,"train":0.3393,"heldOut":1.6125,"copyAccuracy":0.83,"localAccuracy":0.6628,"poolLoss":0.3097},{"step":1400,"train":0.3423,"heldOut":1.5499,"copyAccuracy":0.834,"localAccuracy":0.6616,"poolLoss":0.281},{"step":1500,"train":0.3263,"heldOut":1.6075,"copyAccuracy":0.836,"localAccuracy":0.6564,"poolLoss":0.2919},{"step":1600,"train":0.3192,"heldOut":1.6614,"copyAccuracy":0.8261,"localAccuracy":0.6557,"poolLoss":0.2654},{"step":1700,"train":0.3016,"heldOut":1.7348,"copyAccuracy":0.7727,"localAccuracy":0.6605,"poolLoss":0.274},{"step":1800,"train":0.3108,"heldOut":1.6547,"copyAccuracy":0.8162,"localAccuracy":0.6575,"poolLoss":0.2843},{"step":1900,"train":0.3006,"heldOut":1.7338,"copyAccuracy":0.8024,"localAccuracy":0.6598,"poolLoss":0.2635},{"step":2000,"train":0.2876,"heldOut":1.7508,"copyAccuracy":0.7747,"localAccuracy":0.6634,"poolLoss":0.2635}]},{"rows":1024,"passes":32,"name":"pool1024","summary":{"endStep":2000,"heldOut":0.7062,"copyAccuracy":0.996,"localAccuracy":0.6978,"bestHeldOut":0.693,"bestStep":1200,"firstCopyStep":700,"diverged":false},"lookbackFed":64531.25,"targetsFed":768000,"split":{"lookback":{"n":506,"accuracy":0.996,"loss":0.0334},"local":{"n":5638,"accuracy":0.6978,"loss":0.7666},"copyEpisodes":{"n":107,"accuracy":1,"loss":0.0025},"whoEpisodes":{"n":399,"accuracy":0.995,"loss":0.0417},"guideAnswer":{"n":124,"accuracy":0.9919,"loss":0.1203}},"curve":[{"step":0,"train":null,"heldOut":3.8414,"copyAccuracy":0.0692,"localAccuracy":0.0204,"poolLoss":3.8436},{"step":100,"train":1.5919,"heldOut":0.9844,"copyAccuracy":0.2134,"localAccuracy":0.668,"poolLoss":0.9453},{"step":200,"train":0.9073,"heldOut":0.908,"copyAccuracy":0.1897,"localAccuracy":0.6908,"poolLoss":0.8728},{"step":300,"train":0.8665,"heldOut":0.8644,"copyAccuracy":0.3103,"localAccuracy":0.6983,"poolLoss":0.8351},{"step":400,"train":0.8246,"heldOut":0.8301,"copyAccuracy":0.4447,"localAccuracy":0.6971,"poolLoss":0.7952},{"step":500,"train":0.7879,"heldOut":0.8019,"copyAccuracy":0.6423,"localAccuracy":0.6962,"poolLoss":0.7652},{"step":600,"train":0.7552,"heldOut":0.7448,"copyAccuracy":0.8557,"localAccuracy":0.7034,"poolLoss":0.7099},{"step":700,"train":0.6984,"heldOut":0.7009,"copyAccuracy":0.9921,"localAccuracy":0.7036,"poolLoss":0.6571},{"step":800,"train":0.6656,"heldOut":0.7234,"copyAccuracy":0.9625,"localAccuracy":0.6994,"poolLoss":0.6672},{"step":900,"train":0.6757,"heldOut":0.695,"copyAccuracy":0.996,"localAccuracy":0.7056,"poolLoss":0.6539},{"step":1000,"train":0.6606,"heldOut":0.6984,"copyAccuracy":0.9941,"localAccuracy":0.6987,"poolLoss":0.6474},{"step":1100,"train":0.6641,"heldOut":0.7024,"copyAccuracy":0.9881,"localAccuracy":0.6972,"poolLoss":0.6641},{"step":1200,"train":0.6634,"heldOut":0.693,"copyAccuracy":0.996,"localAccuracy":0.699,"poolLoss":0.6455},{"step":1300,"train":0.659,"heldOut":0.7178,"copyAccuracy":0.996,"localAccuracy":0.7002,"poolLoss":0.662},{"step":1400,"train":0.6602,"heldOut":0.7039,"copyAccuracy":0.9881,"localAccuracy":0.6969,"poolLoss":0.6549},{"step":1500,"train":0.6563,"heldOut":0.6984,"copyAccuracy":0.9822,"localAccuracy":0.6995,"poolLoss":0.6416},{"step":1600,"train":0.6536,"heldOut":0.6955,"copyAccuracy":0.9941,"localAccuracy":0.6985,"poolLoss":0.6364},{"step":1700,"train":0.6524,"heldOut":0.7056,"copyAccuracy":0.9901,"localAccuracy":0.6967,"poolLoss":0.644},{"step":1800,"train":0.6527,"heldOut":0.7281,"copyAccuracy":0.9605,"localAccuracy":0.6878,"poolLoss":0.6664},{"step":1900,"train":0.6482,"heldOut":0.7108,"copyAccuracy":0.9921,"localAccuracy":0.7011,"poolLoss":0.639},{"step":2000,"train":0.6442,"heldOut":0.7062,"copyAccuracy":0.996,"localAccuracy":0.6978,"poolLoss":0.6341}]}]},"shift":{"about":"Held-out rows from another mix, 256 rows each from their own seeds: copy episodes only, who episodes only, plain episodes only, who episodes where rerun is always the doer, and who episodes where tallow or pylon is. generator is the corpus generator's own loss on those rows when it assumes the default mix: a perfect default-mix model pays that much on shifted rows too, so the gap that matters is model minus generator. On 256 rows that gap has sampling noise of about a hundredth of a nat or more (the reference model lands slightly under the generator on copy-only rows), so only large gaps are read as a shift effect. rerunNeverDoes was trained on data where rerun never does the deed in a who episode (so never answers 'who ... ?'); guideAnswer is the accuracy on the guide's name after the question mark. One seed each; 256 rows per set.","generator":{"copyOnly":0.5493,"whoOnly":0.7174,"plainOnly":0.6083,"whoRerunDoes":0.7327,"whoOthersDo":0.7214},"reference":{"copyOnly":{"loss":0.5433,"lookback":1,"local":0.7829,"guideAnswer":null,"nLookback":280},"whoOnly":{"loss":0.7591,"lookback":0.9968,"local":0.6426,"guideAnswer":1,"nLookback":946},"plainOnly":{"loss":0.6525,"lookback":null,"local":0.6606,"guideAnswer":null,"nLookback":0},"whoRerunDoes":{"loss":0.773,"lookback":0.9978,"local":0.6477,"guideAnswer":1,"nLookback":895},"whoOthersDo":{"loss":0.7707,"lookback":0.9946,"local":0.6416,"guideAnswer":1,"nLookback":927}},"rerunNeverDoes":{"copyOnly":{"loss":0.5549,"lookback":0.9857,"local":0.778,"guideAnswer":null,"nLookback":280},"whoOnly":{"loss":1.1285,"lookback":0.8446,"local":0.625,"guideAnswer":0.6712,"nLookback":946},"plainOnly":{"loss":0.6715,"lookback":null,"local":0.661,"guideAnswer":null,"nLookback":0},"whoRerunDoes":{"loss":1.8809,"lookback":0.6302,"local":0.5601,"guideAnswer":0,"nLookback":895},"whoOthersDo":{"loss":0.7754,"lookback":0.9428,"local":0.6465,"guideAnswer":1,"nLookback":927},"summary":{"endStep":2000,"heldOut":0.828,"copyAccuracy":0.8814,"localAccuracy":0.6903,"bestHeldOut":0.772,"bestStep":1200,"firstCopyStep":800,"diverged":false}},"lb0":{"copyOnly":{"loss":9.2097,"lookback":0.0607,"local":0.2236,"guideAnswer":null,"nLookback":280},"whoOnly":{"loss":9.8077,"lookback":0.0169,"local":0.2443,"guideAnswer":0,"nLookback":946},"plainOnly":{"loss":0.3915,"lookback":null,"local":0.7651,"guideAnswer":null,"nLookback":0},"whoRerunDoes":{"loss":10.5202,"lookback":0.0223,"local":0.246,"guideAnswer":0,"nLookback":895},"whoOthersDo":{"loss":9.4864,"lookback":0.0194,"local":0.2436,"guideAnswer":0,"nLookback":927}},"noCopyEpisodes":{"copyOnly":{"loss":6.9236,"lookback":0.0464,"local":0.3464,"guideAnswer":null,"nLookback":280},"whoOnly":{"loss":0.6859,"lookback":1,"local":0.6524,"guideAnswer":1,"nLookback":946},"plainOnly":{"loss":0.5902,"lookback":null,"local":0.743,"guideAnswer":null,"nLookback":0},"whoRerunDoes":{"loss":0.7013,"lookback":1,"local":0.6733,"guideAnswer":1,"nLookback":895},"whoOthersDo":{"loss":0.6912,"lookback":1,"local":0.6354,"guideAnswer":1,"nLookback":927}}},"runSeconds":{"lb05":39.8,"lb0":40,"lb01":40,"reference":45.8,"lb40":41.3,"lb20":41.7,"noCopyEpisodes":41.6,"noise10":41.2,"noise30":39.7,"pool64":40.9,"pool256":41.7,"pool1024":47.2,"rerunNeverDoes":35.8,"seed12":35.5,"seed13":35.6,"lb05seed12":35,"lb05seed13":33.7},"seconds":193.8}