1. 1
2. 2
3. 3
4. 4
5. 5
6. 6
7. 7
8. 8
9. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
16. 16
17. 17
18. 18
19. 19
20. 20
21. 21
22. 22
23. 23
24. 24
25. 25
26. 26
27. 27
28. 28
29. 29
30. 30
31. 31
32. 32
33. 33

from argparse import ArgumentParser
from html import unescape
from re import sub

from psycopg2 import connect


parser = ArgumentParser()
parser.add_argument('-d', '--database', help='database connection string')
parser.add_argument('-o', '--output', help='output file', default='data')
args = parser.parse_args()


# Fetch messages from database since it's way faster than using the API
conn = connect(args.database)
cur = conn.cursor()
cur.execute('SELECT text FROM statuses WHERE language=\'en\'')
statuses = cur.fetchall()


# Clean up statuses
for i in range(len(statuses)):
    # Remove HTML stuff
    statuses[i] = unescape(sub(r'<[^>]*>', '', statuses[i][0]))
    # Remove URLs and special characters and convert to lowercase
    # Uncomment for generating data for LSTMs
    #statuses[i] = ' '.join([sub(r'[^a-z0-9]', '', word.lower()) for word in statuses[i].split() if word.find('://') == -1])


# Save to output file
with open(args.output, 'w') as f:
    for status in statuses:
        print(status, file=f)