Clean up data by removing URLs and special characters and convert to lowercase

Changes

1 changed files (+7/-3)

• main.py
  • @@ -16,16 +16,20 @@ cur = conn.cursor()
    cur.execute('SELECT * FROM statuses') statuses = cur.fetchall() - # Get the messages as plain text + + # Extract all words from statuses # Use regex to remove HTML stuff - # TODO: Remove punctuation and other junk text = [re.sub(r'<[^>]*>', '', status[2]) for status in statuses] # print(text[0:100]) class Dataset(torch.utils.data.Dataset): def __init__(self): + # Flatten text into words self.words = [word for message in text for word in message.split()] + # Remove URLs and special characters and convert to lowercase + self.words = [re.sub(r'[^a-z0-9]', '', word.lower()) for word in self.words if word.find('://') == -1] + self.word_counts = Counter(self.words) self.uniq_words = sorted(self.word_counts, key=self.word_counts.get)
  • @@ -140,4 +144,4 @@ def predict(text, next_words=100):
    return words - print(predict('This is a test')) + print(predict('this is a test'))