Increase breakpoint decode window size slightly

Changes

4 changed files (+24/-24)

• README.md
  • @@ -28,7 +28,7 @@ ## Training
    I implemented this architecture using tch-rs (Rust bindings to PyTorch). The tricky part is that we need to mask the character that the model is trying to predict to prevent the model from cheating. My implementation applies two convolutions for the left and right of the target character and then adds them together. I used " " as a padding token (probably a bad idea). For more details, see my code at the end of this report. - I trained the model on WikiText-103 for 20 epochs using an AMD Radeon 7900XTX and fine-tuned for an additional epoch on the Gutenberg dataset. (Note to self: for training on ROCm, run using `LD_LIBRARY_PATH=.venv/lib/python3.13/site-packages/torch/lib:$LD_LIBRARY_PATH LIBTORCH_USE_PYTORCH=1 cargo r`.) It achieves a final loss of 0.37 which means it puts $e^{-0.37} \approx 0.69$ probability on the correct character on average. + I trained the model on WikiText-103 for 20 epochs using an AMD Radeon 7900XTX and fine-tuned for an additional epoch on the Gutenberg dataset. (Note to self: for training on ROCm, run using `LD_LIBRARY_PATH=.venv/lib/python3.13/site-packages/torch/lib:$LD_LIBRARY_PATH LIBTORCH_USE_PYTORCH=1 cargo r -r`.) It achieves a final loss of 0.37 which means it puts $e^{-0.37} \approx 0.69$ probability on the correct character on average. <!-- idk how to make Typst/Pandoc move the image after the text like in LaTeX --> 
• mkthingy.sh
  • @@ -1,5 +1,6 @@
    #!/bin/sh + rg TODO src if rg ' dbg!' src; then echo CONTAINS DEBUG OUTPUT! exit 1
• report.pdf
• src/main.rs
  • @@ -10,7 +10,7 @@ use std::fs::File;
    use std::io::{BufReader, prelude::*}; use std::thread; use tch::nn::{Module, OptimizerConfig}; - use tch::{CModule, Device, IndexOp, Kind, NewAxis, Reduction, Tensor, nn, no_grad}; + use tch::{Device, IndexOp, Kind, NewAxis, Reduction, Tensor, nn, no_grad}; const LR: f64 = 5e-4; const WD: f64 = 1e-4;
  • @@ -243,7 +243,7 @@ // Cleanup using CNN
    let mut rng = Xoshiro256PlusPlus::from_rng(&mut rand::rng()); // Use padding to properly handle the ends let (mut ys, mut loss) = probs_padded(net, &permute(s, &p)); - dbg!(loss, to_string(&permute(s, &p))); + // dbg!(loss, to_string(&permute(s, &p))); if loss > 3. { // Not worth trying return (p, loss);
  • @@ -251,7 +251,7 @@ }
    let mut pbest = *p2; let mut lossbest = loss; let mut w = weights(s, &p, &ys); - for _ in 0..400 * iters / s.len() { + for _ in 0..300 * iters / s.len() { let dist = WeightedIndex::new(&w).unwrap(); let sample = dist.sample(&mut rng); let a = p
  • @@ -358,14 +358,14 @@ let acc = f32::min(0., lp2 - lp);
    if rng.random::<f32>() < acc.exp() { p = q; lp = lp2; - if lp2 < lpbest { + if lp2 > lpbest { lpbest = lp2; pbest = q; } } } } - dbg!(k, to_string(&permute(s, &pbest)), lpbest); + // dbg!(k, to_string(&permute(s, &pbest)), lpbest); tmp.push((pbest, lpbest)); } tmp
  • @@ -457,7 +457,6 @@ let (pr, lossr) = decode(&net, &text[m..], &grams);
    // dbg!(to_string(&permute(&text[..m], &pl))); // dbg!(to_string(&permute(&text[m..], &pr))); // dbg!(lossl, lossr); - // TODO: should we do the sliding window from the other direction??? let mut reallossbest = 100.; let mut ans = vec![]; // This is spaghetti code yeah I know
  • @@ -480,9 +479,8 @@ sum -= deque.front().unwrap();
    deque.pop_front(); } if deque.len() == WINDOW && sum / WINDOW as f64 > 5. { - // Breakpoint probably in i - WINDOW to i - // TODO: make this larger - let pr2 = decode(&net, &text[i..], &grams).0; + // Breakpoint probably in i - WINDOW to i - 5 + let pr2 = decode(&net, &text[i - 5..], &grams).0; let mut jbest = 0; let mut lossbest = 100.; let mut pt = permute(
  • @@ -498,7 +496,7 @@ jbest = j + 1; // Off-by-1 error
    lossbest = loss; } } - dbg!(jbest, i); + // dbg!(jbest, i); (ans, reallossbest) = finish(&net, &text, &pl, &pr2, jbest); break; }
  • @@ -527,8 +525,8 @@ sum -= deque.front().unwrap();
    deque.pop_front(); } if deque.len() == WINDOW && sum / WINDOW as f64 > 5. { - // Breakpoint probably in i to i + WINDOW - let pl2 = decode(&net, &text[..i], &grams).0; + // Breakpoint probably in i + 5 to i + WINDOW + let pl2 = decode(&net, &text[..i + 5], &grams).0; let mut jbest = 0; let mut lossbest = 100.; let mut pt = permute(
  • @@ -544,6 +542,7 @@ jbest = j + 1; // Off-by-1 error
    lossbest = loss; } } + // dbg!(jbest, i); let (ans2, loss2) = finish(&net, &text, &pl2, &pr, jbest); if loss2 < reallossbest { ans = ans2;
  • @@ -604,16 +603,16 @@ opt.backward_step(&loss);
    } } vs.save("model.safetensors").unwrap(); - vs.freeze(); - let mut closure = |input: &[Tensor]| vec![net.forward(&input[0])]; - let model = CModule::create_by_tracing( - "MyModule", - "forward", - &[Tensor::zeros([1, 784], (tch::Kind::Int64, device))], - &mut closure, - ) - .unwrap(); - // I think this has the input size hardcoded though sad - model.save("model.pt").unwrap(); + // vs.freeze(); + // let mut closure = |input: &[Tensor]| vec![net.forward(&input[0])]; + // let model = CModule::create_by_tracing( + // "MyModule", + // "forward", + // &[Tensor::zeros([1, 784], (tch::Kind::Int64, device))], + // &mut closure, + // ) + // .unwrap(); + // // I think this has the input size hardcoded though sad + // model.save("model.pt").unwrap(); } }