Create a db extraction script

Changes

1 changed files (+29/-0)

• db.py (new)
  • @@ -0,0 +1,29 @@
    + #!/usr/bin/python3 + + import re + import psycopg2 + + + # Fetch messages from database since it's way faster than using the API + conn = psycopg2.connect(dbname="mastodon_production") + cur = conn.cursor() + cur.execute('SELECT * FROM statuses') + statuses = cur.fetchall() + + + # Extract all words from statuses + # Use regex to remove HTML stuff + text = [re.sub(r'<[^>]*>', ' ', status[2]) for status in statuses] + # print(text[0:100]) + + + words = [word for message in text for word in message.split()] + # Remove URLs and special characters and convert to lowercase + words = [re.sub(r'[^a-z0-9]', '', word.lower()) for word in words if word.find('://') == -1] + # Remove empty strings + words = [word for word in words if word != ''] + + + with open('/tmp/text', 'w') as f: + for word in words: + f.write(word + '\n')