blob: 2ff8a59ccf70a50fcfdcd202997aac92c51d1144 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
from argparse import ArgumentParser
from html import unescape
from re import sub
from psycopg2 import connect
parser = ArgumentParser()
parser.add_argument('-d', '--database', help='database connection string')
parser.add_argument('-o', '--output', help='output file', default='data')
args = parser.parse_args()
# Fetch messages from database since it's way faster than using the API
conn = connect(args.database)
cur = conn.cursor()
cur.execute('SELECT text FROM statuses WHERE language=\'en\'')
statuses = cur.fetchall()
# Clean up statuses
for i in range(len(statuses)):
# Remove HTML stuff
statuses[i] = unescape(sub(r'<[^>]*>', '', statuses[i][0]))
# Remove URLs and special characters and convert to lowercase
# Uncomment for generating data for LSTMs
#statuses[i] = ' '.join([sub(r'[^a-z0-9]', '', word.lower()) for word in statuses[i].split() if word.find('://') == -1])
# Save to output file
with open(args.output, 'w') as f:
for status in statuses:
print(status, file=f)
|
