1. 1
2. 2
3. 3
4. 4
5. 5
6. 6
7. 7
8. 8
9. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
16. 16
17. 17
18. 18
19. 19
20. 20
21. 21

# https://huggingface.co/datasets/Salesforce/wikitext
# https://shibamoulilahiri.github.io/gutenberg_dataset.html
# Use `parquet-tools csv` to mangle the dataset into a text file
# Run this with PyPy for SPEEED

from encode import _clean_text

with open('gutenberg-dirty') as f:
    lines = f.readlines()

print(len(lines))

with open('gutenberg', 'w') as f:
    for i, l in enumerate(lines):
        if len(l) < 200:
            # Skip junk
            continue
        cleaned = _clean_text(l)
        if len(cleaned) > 200:
            f.write(cleaned + '\n')
            print(i)