| import re |
|
|
|
|
| def clean_corpus(chat_export_file): |
| """Prepare a WhatsApp chat export for training with chatterbot.""" |
| message_corpus = remove_chat_metadata(chat_export_file) |
| cleaned_corpus = remove_non_message_text(message_corpus) |
| return cleaned_corpus |
|
|
|
|
| def remove_chat_metadata(chat_export_file): |
| """Remove WhatsApp chat metadata. |
| |
| WhatsApp chat exports come with metadata about each message: |
| |
| date time username message |
| --------------------------------------- |
| 8/26/22, 17:47 - Jane Doe: Message text |
| |
| This function removes all the metadata up to the text of each message. |
| |
| Args: |
| chat_export_file (str): The name of the chat export file |
| |
| Returns: |
| tuple: The text of each message in the conversation |
| """ |
| date_time = r"(\d+\/\d+\/\d+,\s\d+:\d+)" |
| dash_whitespace = r"\s-\s" |
| username = r"([\w\s]+)" |
| metadata_end = r":\s" |
| pattern = date_time + dash_whitespace + username + metadata_end |
|
|
| with open(chat_export_file, "r") as corpus_file: |
| content = corpus_file.read() |
| cleaned_corpus = re.sub(pattern, "", content) |
| return tuple(cleaned_corpus.split("\n")) |
|
|
|
|
| def remove_non_message_text(export_text_lines): |
| """Remove conversation-irrelevant text from chat export. |
| |
| WhatsApp chat exports come with a standardized intro line, |
| and an empty line at the end of the file. |
| Text exports also replace media messages with text that isn't |
| relevant for the conversation. This function removes all that. |
| |
| Args: |
| export_text_lines (tuple): All lines from the export file |
| |
| Returns: |
| tuple: Messages that are a relevant part of the conversation |
| """ |
| messages = export_text_lines[1:-1] |
|
|
| filter_out_msgs = ("<Media omitted>",) |
| return tuple((msg for msg in messages if msg not in filter_out_msgs)) |
|
|
