1. 1
2. 2
3. 3
4. 4
5. 5
6. 6
7. 7
8. 8
9. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
16. 16
17. 17
18. 18
19. 19
20. 20
21. 21
22. 22
23. 23
24. 24
25. 25
26. 26
27. 27
28. 28
29. 29

#!/usr/bin/python3

import re
import psycopg2


# Fetch messages from database since it's way faster than using the API
conn = psycopg2.connect(dbname="mastodon_production")
cur = conn.cursor()
cur.execute('SELECT * FROM statuses')
statuses = cur.fetchall()


# Extract all words from statuses
# Use regex to remove HTML stuff
text = [re.sub(r'<[^>]*>', ' ', status[2]) for status in statuses]
# print(text[0:100])


words = [word for message in text for word in message.split()]
# Remove URLs and special characters and convert to lowercase
words = [re.sub(r'[^a-z0-9]', '', word.lower()) for word in words if word.find('://') == -1]
# Remove empty strings
words = [word for word in words if word != '']


with open('/tmp/text', 'w') as f:
    for word in words:
        f.write(word + '\n')